@inproceedings{10.1145/3797867.3828986,
author = {Kim, Chaelin and Crochet, Denise and Bohacek, Maty and Hassan, Saad},
title = {Design and Evaluation of an AI-Based American Sign Language Video Comprehension Tool: Exploring the Benefits and Use of Automatic Segmentation and Sign Lookup},
year = {2026},
isbn = {9798400725210},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3797867.3828986},
doi = {10.1145/3797867.3828986},
abstract = {Sign language learners often watch continuous signing videos to develop their receptive and expressive skills, yet looking up and identifying unfamiliar signs during these comprehension tasks remains a challenge. Prior HCI research has explored in-situ sign lookup tools using Wizard-of-Oz prototypes, enabling learners to search for signs by submitting video segments as queries. Findings showed that this approach improved comprehension and reduced cognitive load compared to feature-based dictionaries. Building on advances in AI-based sign language recognition and segmentation, we present an automatic video comprehension prototype system that segments continuous signing into individual signs and employs a fine-tuned recognition model to retrieve the closest matches for each segment. We report findings from an experimental study with thirteen intermediate ASL learners, combining quantitative and observational analyses to compare our system with a baseline that supports user-selected segment-based search but lacks AI-based automatic segmentation. Results show that automatic segmentation improved user experience, while observational data revealed distinct interaction patterns between the two prototype systems. We release both prototype systems and present design guidelines for video comprehension tools for sign language learners.},
booktitle = {Proceedings of the 28th International ACM SIGACCESS Conference on Computers and Accessibility},
articleno = {33},
numpages = {19},
keywords = {Sign Language, American Sign Language, Language Learning, Sign Language Learning, Artificial Intelligence, Sign Language Recognition, Sign Language Segmentation, Human-AI Interaction, Video Comprehension},
location = {Porto, Portugal},
series = {ASSETS '26}
}