@inproceedings{7ef1b6862067475ab62a3a4a64e98ac7,
title = "Semantic Clinical Artificial Intelligence (SCAI) Usability Testing",
abstract = "We evaluated the performance of Semantic Clinical Artificial Intelligence (SCAI, pronounced Sky), a large language model (LLM) knowledge resource through usability testing. This pretest-intervention-posttest mixed-methods user interface (UI) design study investigates usability to determine whether the LLM provides a more comprehensive, efficient, and enhanced user-friendly means of delivering end user information as compared to using primary sources of information from the Internet (Web). Our analysis focused on assessing the LLM's efficiency and usability in helping users arrive at accurate and reliable outcomes, to ultimately determine its value as an innovative tool for packaging and presenting information. Usability test sessions were conducted using the cognitive walkthrough approach, via Zoom. Participants were asked to respond to scenarios using only the LLM, and then only the web, and vice versa. These sessions were followed by user feedback sessions where participants rated their experiences and responded to open-ended questions related to the overall usability and satisfaction with SCAI.",
keywords = "human-centered design, large language models, usability testing",
author = "Rachel Stephens and Gillian Franklin and Taylor, \{Kevin B.\} and Tung, \{Kaity H.\} and Vladyslav Ilchenko and Muhammad Piracha and Shmuel Tiosano and Kendria Hall and Irshad Ally and Elkin, \{Peter L.\}",
note = "Publisher Copyright: {\textcopyright} 2025 The Authors.; 7th International Conference on Context Sensitive Health Informatics: AI for Social Good, CSHI 2025 ; Conference date: 23-05-2025 Through 24-05-2025",
year = "2025",
month = may,
day = "12",
doi = "10.3233/SHTI25023",
language = "English",
series = "Studies in Health Technology and Informatics",
publisher = "IOS Press BV",
pages = "27--32",
editor = "Hadiza Ismaila and Linda Dusseljee-Peute and Romaric Marcilly and Elkin, \{Peter L.\} and Kuziemsky, \{Craig E.\} and Christian Nohr and \{Van Dort\}, \{Bethany A.\} and Rebecca Randell",
booktitle = "Context Sensitive Health Informatics",
address = "Netherlands",
}