@inproceedings{10.1145/3797867.3841227,
author = {Kim, Chaelin and Hassan, Saad},
title = {Supporting Sign Language Video Comprehension through In-Situ Automatic Segmentation and Sign Lookup Tool},
year = {2026},
isbn = {9798400725210},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3797867.3841227},
doi = {10.1145/3797867.3841227},
abstract = {Sign language learners often closely watch signing videos to develop receptive skills. However, they encounter unfamiliar signs, especially early on when they are still developing their vocabulary, and later due to challenges in understanding signs in continuous videos because of coarticulation. Existing tools, such as feature-based dictionaries, often require learners to identify the linguistic properties of a sign to search for it, which can be cumbersome. Prior work has proposed interfaces to support in-situ sign lookup, but end-to-end systems have not yet been disseminated or demonstrated. We demonstrate a fully automatic video comprehension prototype system that combines state-of-the-art sign language segmentation with a fine-tuned spatiotemporal recognition network. The underlying AI pipeline ingests a continuous signing video, applies a linguistically motivated BIO-tagging segmentation framework to locate boundaries, and routes individual segments into a fine-tuned Spatial-Temporal Graph Convolutional Network (ST-GCN) to retrieve the closest gloss matches. We present the system design and demonstration plan. We also discuss the system’s current limitations and future research directions.},
booktitle = {Proceedings of the 28th International ACM SIGACCESS Conference on Computers and Accessibility},
articleno = {169},
numpages = {5},
keywords = {American Sign Language, Language Learning, Sign Language, Sign Language Learning, Artificial Intelligence, Sign Language Recognition, Sign Language Segmentation, AI Tools},
location = {Porto, Portugal},
series = {ASSETS '26}
}