@inproceedings{10.1145/3544549.3585880,
author = {Hassan, Saad and Ding, Yao and Kerure, Agneya Abhimanyu and Miller, Christi and Burnett, John and Biondo, Emily and Gilbert, Brenden},
title = {Exploring the Design Space of Automatically Generated Emotive Captions for Deaf or Hard of Hearing Users},
year = {2023},
isbn = {9781450394222},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3544549.3585880},
doi = {10.1145/3544549.3585880},
abstract = {Caption text conveys salient auditory information to deaf or hard-of-hearing (DHH) viewers. However, the emotional information within the speech is not captured. We developed three emotive captioning schemas that map the output of audio-based emotion detection models to expressive caption text that can convey underlying emotions. The three schemas used typographic changes to the text, color changes, or both. Next, we designed a Unity framework to implement these schemas and used it to generate stimuli videos. In an experimental evaluation with 28 DHH viewers, we compared DHH viewers’ ability to understand emotions and their subjective judgments across the three captioning schemas. We found no significant difference in participants’ ability to understand the emotion based on the captions or their subjective preference ratings. Open-ended feedback revealed factors contributing to individual differences in preferences among the participants and challenges with automatically generated emotive captions that motivate future work.},
booktitle = {Extended Abstracts of the 2023 CHI Conference on Human Factors in Computing Systems},
articleno = {125},
numpages = {10},
keywords = {emotive captions, captions, Deaf or hard of hearing users, Applications of emotion recognition},
location = {<conf-loc>, <city>Hamburg</city>, <country>Germany</country>, </conf-loc>},
series = {CHI EA '23}
}