@inproceedings{10.1145/3706598.3713304,
author = {de Lacerda Pataca, Calu\~{a} and Hassan, Saad and May, Lloyd and Olson, Michelle M and D'aurio, Toni and Peiris, Roshan L and Huenerfauth, Matt},
title = {Tactile Emotions: Multimodal Affective Captioning with Haptics Improves Narrative Engagement for d/Deaf and Hard-of-Hearing Viewers},
year = {2025},
isbn = {9798400713941},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3706598.3713304},
doi = {10.1145/3706598.3713304},
abstract = {This paper explores a multimodal approach for translating emotional cues present in speech, designed with Deaf and Hard-of-Hearing (dhh) individuals in mind. Prior work has focused on visual cues applied to captions, successfully conveying whether a speaker’s words have a negative or positive tone (valence), but with mixed results regarding the intensity (arousal) of these emotions. We propose a novel method using haptic feedback to communicate a speaker’s arousal levels through vibrations on a wrist-worn device. In a formative study with 16 dhh participants, we tested six haptic patterns and found that participants preferred single per-word vibrations at 75 Hz to encode arousal. In a follow-up study with 27 dhh participants, this pattern was paired with visual cues, and narrative engagement with audio-visual content was measured. Results indicate that combining haptics with visuals significantly increased engagement compared to a conventional captioning baseline and a visuals-only affective captioning style.},
booktitle = {Proceedings of the 2025 CHI Conference on Human Factors in Computing Systems},
articleno = {68},
numpages = {17},
keywords = {Accessibility, Emotion / Affective Computing, Individuals with Disabilities \& Assistive Technologies, Empirical study that tells us about how people use a system},
location = {
},
series = {CHI '25}
}