@inproceedings{90d4791afc7e468986e38c774a0f22df,
title = "Shedding light on the automated essay scoring process",
abstract = "This paper explores in depth the suitability of the 2012 Automated Student Assessment Prize (ASAP) contest's essay datasets. It evaluates the potential of deep learning and state-of-the-art NLP tools in automated essay scoring (AES) to predict not only holistic scores but also the finer-grained rubric scores, an area underexplored but essential to provision formative feedback and uncover the AI reasoning behind AES. For comparison purpose, this paper advocates the need for transparency when sharing AES processes and outcomes. Finally, it reveals the insufficiency of ASAP essay datasets to train generalizable AES models by examining the distributions of holistic and rubric scores. Findings show that the strength of agreement between human and machine graders on holistic scores does not translate into similar strength on rubric scores and that the learning made by the machine barely exceeds the performance of a na{\"i}ve predictor.",
keywords = "Automated essay scoring, Automated student assessment prize, Deep learning, Natural language processing, Rubrics",
author = "David Boulanger and Vivekanandan Kumar",
note = "Publisher Copyright: {\textcopyright} EDM 2019 - Proceedings of the 12th International Conference on Educational Data Mining. All rights reserved.; 12th International Conference on Educational Data Mining, EDM 2019 ; Conference date: 02-07-2019 Through 05-07-2019",
year = "2019",
language = "English",
series = "EDM 2019 - Proceedings of the 12th International Conference on Educational Data Mining",
pages = "512--515",
editor = "Lynch, {Collin F.} and Agathe Merceron and Michel Desmarais and Roger Nkambou",
booktitle = "EDM 2019 - Proceedings of the 12th International Conference on Educational Data Mining",
}