<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article article-type="research-article" dtd-version="2.0" xmlns:xlink="http://www.w3.org/1999/xlink">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JMIR</journal-id>
      <journal-id journal-id-type="nlm-ta">JMIR AI</journal-id>
      <journal-title>JMIR AI</journal-title>
      <issn pub-type="epub">2817-1705</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v5i1e84699</article-id>
      <article-id pub-id-type="pmid"/>
      <article-id pub-id-type="doi">10.2196/84699</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Original Paper</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Original Paper</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Balancing Human and AI Features for Digital Autism Diagnostics from Home Videos: Human-In-The-Loop Machine Learning Study</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Coristine</surname>
            <given-names>Andrew</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Bao</surname>
            <given-names>Bokan</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Kurmashev</surname>
            <given-names>Ruslan</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author">
          <name name-style="western">
            <surname>Rasool</surname>
            <given-names>Abdur</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-5334-9001</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author">
          <name name-style="western">
            <surname>Dunlap</surname>
            <given-names>Kaitlyn</given-names>
          </name>
          <degrees>BA</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-4423-5269</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author">
          <name name-style="western">
            <surname>Husic</surname>
            <given-names>Arman</given-names>
          </name>
          <degrees>BSc</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-9180-5212</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author">
          <name name-style="western">
            <surname>Kline</surname>
            <given-names>Aaron</given-names>
          </name>
          <degrees>BS</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-0077-5485</ext-link>
        </contrib>
        <contrib id="contrib5" contrib-type="author">
          <name name-style="western">
            <surname>Azizian</surname>
            <given-names>Parnian</given-names>
          </name>
          <degrees>BSc</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0009-0006-6975-4953</ext-link>
        </contrib>
        <contrib id="contrib6" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Wall</surname>
            <given-names>Dennis P</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-7889-9146</ext-link>
        </contrib>
        <contrib id="contrib7" contrib-type="author" corresp="yes" equal-contrib="yes">
          <name name-style="western">
            <surname>Washington</surname>
            <given-names>Peter</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff3" ref-type="aff">3</xref>
          <address>
            <institution>Department of Medicine</institution>
            <institution>University of California, San Francisco</institution>
            <addr-line>500 Parnassus Avenue</addr-line>
            <addr-line>San Francisco, CA, 94143</addr-line>
            <country>United States</country>
            <phone>1 (415) 353 2067</phone>
            <email>peter.washington@ucsf.edu</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-3276-4411</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Department of Information and Computer Sciences</institution>
        <institution>University of Hawaii at Manoa</institution>
        <addr-line>Honolulu, HI</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Department of Psychiatry and Behavioral Sciences (by courtesy)</institution>
        <institution>Stanford University</institution>
        <addr-line>California, CA</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff3">
        <label>3</label>
        <institution>Department of Medicine</institution>
        <institution>University of California, San Francisco</institution>
        <addr-line>San Francisco, CA</addr-line>
        <country>United States</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Peter Washington <email>peter.washington@ucsf.edu</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <year>2026</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>20</day>
        <month>11</month>
        <year>2026</year>
      </pub-date>
      <volume>5</volume>
      <elocation-id>e84699</elocation-id>
      <history>
        <date date-type="received">
          <day>29</day>
          <month>9</month>
          <year>2025</year>
        </date>
        <date date-type="rev-request">
          <day>15</day>
          <month>1</month>
          <year>2026</year>
        </date>
        <date date-type="rev-recd">
          <day>3</day>
          <month>9</month>
          <year>2026</year>
        </date>
        <date date-type="accepted">
          <day>9</day>
          <month>9</month>
          <year>2026</year>
        </date>
      </history>
      <copyright-statement>©Abdur Rasool, Kaitlyn Dunlap, Arman Husic, Aaron Kline, Parnian Azizian, Dennis P Wall, Peter Washington. Originally published in JMIR AI (https://ai.jmir.org), 20.11.2026.</copyright-statement>
      <copyright-year>2026</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR AI, is properly cited. The complete bibliographic information, a link to the original publication on https://www.ai.jmir.org/, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="https://ai.jmir.org/2026/1/e84699" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>Autism spectrum disorder (ASD or autism) diagnosis faces significant delays due to specialist shortages and subjective, clinic-based assessments. Although AI offers a potentially scalable solution via automated video analysis, current state-of-the-art deep learning models often fail to capture subtle, clinically nuanced social behaviors essential for reliable detection.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>This study aimed to develop and evaluate a human-in-the-loop (HITL) framework that combines automated AI feature extraction with crowdsourced human annotation for autism classification on naturalistic home videos. We targeted low-confidence AI facial feature extraction outputs to be replaced by human features. We also sought to quantify the relationship between the confidence threshold used for swapping AI features with human features, the volume of human replacement of AI features, and the predictive diagnostic performance.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>We implemented a HITL pipeline integrating deep learning models for facial emotion (7-class) feature extraction. A confidence threshold applied to the emitted probability of the AI feature extractor model was used to identify low-confidence facial features or those that the AI model was unable to make a prediction for (henceforth referred to as “unresolved”), which were then replaced by human “feature extraction” (ie, annotation) conducted by rigorously vetted crowd workers. We evaluated this approach on 100 high-quality videos (50 autism and 50 neurotypical) from the GuessWhat? digital therapeutic app. Performance was assessed across decision thresholds (0.1-0.9 in increments of 0.1) using sensitivity, specificity, <italic>F</italic><sub>1</sub>-score, and area under the receiver operating characteristic curve (AUROC), with 5-fold cross-validation.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>Across decision thresholds, performance improved overall but was nonmonotonic at intermediate thresholds. At a threshold of 0.9, 59.7% (597/1000) of frames were replaced by human annotations, and mean sensitivity increased from 0.51 (SD 0.24) at a threshold of 0.1 to 0.66 (SD 0.10) at a threshold of 0.9, an absolute increase of approximately 15 percentage points. The HITL-enhanced multimodal model achieved an aggregate out-of-fold AUROC of 0.76, compared with 0.54 for the AI-only model.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>In this proof-of-concept study, replacing AI features with human features tended to be associated with higher observed autism classification performance in this particular dataset. We targeted low-confidence and unresolved frames for human annotation. We note that these findings are exploratory and thus do not isolate the independent causal effect of human replacement or establish clinical diagnostic validity.</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>autism spectrum disorder</kwd>
        <kwd>human-in-the-loop AI</kwd>
        <kwd>artificial intelligence</kwd>
        <kwd>digital phenotyping</kwd>
        <kwd>video analysis</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <p>Autism spectrum disorder (ASD; also known as autism) is a neurodevelopmental condition characterized by social interaction challenges, communication difficulties, and restricted and repetitive behaviors, typically identifiable in early childhood [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref2">2</xref>]. Many children do not receive a formal diagnosis until the age of 4 years or later due to limited specialist availability, long wait times, and reliance on subjective clinical observations and caregiver reports [<xref ref-type="bibr" rid="ref3">3</xref>]. In the United States, autism prevalence has risen from 1 in 150 children in 2000 to 1 in 36 by 2020, and yet more than 83% of counties lack sufficient diagnostic services [<xref ref-type="bibr" rid="ref3">3</xref>-<xref ref-type="bibr" rid="ref5">5</xref>]. These delays hinder early intervention, which improves long-term developmental outcomes and quality of life for individuals with autism [<xref ref-type="bibr" rid="ref2">2</xref>,<xref ref-type="bibr" rid="ref6">6</xref>-<xref ref-type="bibr" rid="ref9">9</xref>].</p>
      <p>With increasing autism prevalence and diagnostic resource shortages, scalable and accessible diagnostic tools are urgently needed [<xref ref-type="bibr" rid="ref1">1</xref>]. Digital phenotyping, which uses mobile health technologies and wearables to quantify human behavior, enables real-world behavioral data collection and AI-driven analysis [<xref ref-type="bibr" rid="ref2">2</xref>,<xref ref-type="bibr" rid="ref10">10</xref>,<xref ref-type="bibr" rid="ref11">11</xref>]. Mobile apps allow structured video capture of children engaging in interactive tasks while AI models analyze behavioral patterns associated with autism [<xref ref-type="bibr" rid="ref10">10</xref>,<xref ref-type="bibr" rid="ref12">12</xref>]. A notable example is a Google Glass–based system that provides real-time feedback on emotional expressions, aiding both data collection and social engagement. These digital tools demonstrate how scalable alternatives to in-clinic evaluations can support early diagnosis and treatment [<xref ref-type="bibr" rid="ref2">2</xref>,<xref ref-type="bibr" rid="ref7">7</xref>].</p>
      <p>AI-based digital phenotyping approaches can and will benefit from a sharper focus on improving their ability to capture nuanced social behaviors critical for autism diagnosis [<xref ref-type="bibr" rid="ref2">2</xref>,<xref ref-type="bibr" rid="ref13">13</xref>-<xref ref-type="bibr" rid="ref15">15</xref>]. Key autism-related indicators, such as fleeting eye contact or microexpressions, are frequently misclassified by AI [<xref ref-type="bibr" rid="ref2">2</xref>,<xref ref-type="bibr" rid="ref16">16</xref>,<xref ref-type="bibr" rid="ref17">17</xref>]. Subtle social cues vary across children and contexts, making them difficult to classify. AI models are also prone to overfitting, particularly when trained on limited datasets, leading to poor generalization to new cases.</p>
      <p>Human-in-the-loop (HITL) approaches address these challenges by integrating human expertise into AI pipelines [<xref ref-type="bibr" rid="ref18">18</xref>,<xref ref-type="bibr" rid="ref19">19</xref>]. Human annotators can more reliably label complex behaviors that AI alone may misinterpret [<xref ref-type="bibr" rid="ref20">20</xref>]. Prior studies [<xref ref-type="bibr" rid="ref21">21</xref>,<xref ref-type="bibr" rid="ref22">22</xref>] show that trained nonexpert crowd workers can reliably annotate sociobehavioral features relevant to autism classification. Human annotators often detect subtle cues that AI models overlook. Hybrid approaches combining AI with human annotations have the potential to enhance classification reliability by reducing noise, improving feature granularity, and increasing model interpretability, particularly in diverse, real-world environments [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref20">20</xref>,<xref ref-type="bibr" rid="ref23">23</xref>-<xref ref-type="bibr" rid="ref25">25</xref>].</p>
      <p>We introduce a HITL AI framework to overcome the critical limitation of automation failures when extracting nuanced social cues for autism diagnosis. We used deep learning models for automated feature extraction via a facial emotion feature extraction (FEFE) model using a pretrained convolutional network to classify 7 emotions and an eye gaze (EG) model to quantify eye contact. We systematically identified low-confidence or unclassifiable (usually due to the face not being detectable, henceforth referred to as “unresolved”) FEFE outputs for replacement with human annotations provided by a curated human workforce identified through crowdsourcing campaigns. We evaluated the framework on a curated dataset of 100 high-quality home videos from the GuessWhat? digital therapeutic app, and we compared the performance of autism classification when using (1) purely automated feature extraction and (2) features extracted by a curated pool of nonexpert humans.</p>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Overview</title>
        <p>We designed a framework (<xref rid="figure1" ref-type="fig">Figure 1</xref>) that analyzes home videos using AI models (FEFE and EG) to extract facial emotion and eye-gaze features. We implemented a HITL pipeline integrating deep learning models for FEFE across 7 classes. A confidence threshold applied to the emitted probability of the AI feature extractor model was used to identify low-confidence facial features or those for which the AI model was unable to make a prediction (henceforth referred to as “unresolved”), which were then replaced by human “feature extraction” (ie, annotation) conducted by rigorously vetted crowd workers. We then quantified the relationship between the facial confidence threshold and classification performance. Details are provided in the next subsections.</p>
        <fig id="figure1" position="float">
          <label>Figure 1</label>
          <caption>
            <p>Framework of an AI-enhanced, human-in-the-loop methodology for autism classification. AI models initially extract facial emotion and gaze features from video frames. When the confidence score is below a prespecified decision threshold, the AI-based features are replaced by human “feature extraction” (ie, annotation). A neural network then classifies autism (autism spectrum disorder) or neurotypical using a combination of AI-extracted and human-annotated features. ASD: autism spectrum disorder; FEFE: facial emotion feature extraction; HITL: human-in-the-loop: NT: neurotypical.</p>
          </caption>
          <graphic xlink:href="ai_v5i1e84699_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>GuessWhat? Data Acquisition</title>
        <p>We used data [<xref ref-type="bibr" rid="ref10">10</xref>,<xref ref-type="bibr" rid="ref22">22</xref>,<xref ref-type="bibr" rid="ref26">26</xref>] collected from a cross-platform mobile game called GuessWhat? This game is designed to encourage prosocial play and interaction between children and their parents in a home environment. The GuessWhat? game included 2 primary activities: charades and quizzes. In charades, children acted out emotions, characters, and objects, while in quizzes, they answered questions about colors, shapes, and word spellings. During each session, parents acted as mediators and recorded their child’s gameplay by placing the smartphone on their forehead, ensuring that the front-facing camera remained oriented toward the child (Figure S1 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>). This setup provided a consistent frame with the child’s face visible, maintaining a distance of approximately 60 to 180 cm from the camera. At the end of each 90-second session, parents were given the option to review and share the video with the research team at Stanford University [<xref ref-type="bibr" rid="ref10">10</xref>,<xref ref-type="bibr" rid="ref13">13</xref>,<xref ref-type="bibr" rid="ref27">27</xref>].</p>
        <p>The collected dataset included videos from children diagnosed with autism and neurotypical children. These videos were sourced remotely, ensuring a diverse dataset that covers various demographic backgrounds, as provided by caregivers during the registration process. Diagnosis was established through caregiver report and subsequent consensus confirmation by 2 independent clinical reviewers who were not involved in the development of the AI models or the HITL framework. These reviewers assessed the provided video data independently of the study’s analytical pipeline against standard developmental benchmarks, though formal Autism Diagnostic Observation Schedule, Second Edition (ADOS-2) or Autism Diagnostic Interview–Revised (ADI-R) assessments [<xref ref-type="bibr" rid="ref28">28</xref>] were not conducted specifically for this video subset. Videos were screened to exclude any with poor lighting, corrupted files, or irrelevant content to ensure high-quality, analyzable footage and maintain input consistency for AI model processing.</p>
        <p>To streamline data processing, each video was uploaded directly to an Amazon S3 bucket through an automated process within the app. The standardized format of each gameplay session, where the device was held in landscape mode and the child remained visible in the frame, ensured consistency across the dataset. Any videos not meeting these standards were excluded from the study.</p>
        <p>By using a smartphone, this approach lowers barriers to health care, making it feasible for a larger population of children to receive care outside of clinical settings. This method is also time- and cost-efficient compared with in-lab assessments, and recording in a home environment promotes more naturalistic behavior in children, potentially yielding data that more accurately represent real-world interactions.</p>
        <p>Parents or guardians reviewed each recording and provided consent for research use; the analysis subset was selected from recordings for which consent also permitted public sharing. From the selected recordings, deidentified uncropped still frames were placed in a private repository and made available only to individual authorized Microworkers participants through task-specific secure URLs. Workers viewed isolated frames only and had no access to source videos, participant identities, diagnostic labels, or broader video context.</p>
        <p>The initial dataset contained a total of 1093 raw video recordings (<xref ref-type="table" rid="table1">Table 1</xref>), comprising 985 videos of children with autism and 108 videos of neurotypical children. This dataset was divided into autism and neurotypical categories, with ages ranging from 3 to 12 years. The demographic breakdown, as shown in <xref ref-type="table" rid="table1">Table 1</xref>, includes gender distribution and age groups, highlighting a comprehensive spread across age and gender for both autism and neurotypical groups.</p>
        <table-wrap position="float" id="table1">
          <label>Table 1</label>
          <caption>
            <p>Demographic and structural overview of datasets, showing the initial source dataset (n=1093 videos) and the manually filtered high-quality analysis dataset of unique children (n=100; 50 autism, 50 neurotypical) used for final analysis.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="30"/>
            <col width="400"/>
            <col width="190"/>
            <col width="190"/>
            <col width="160"/>
            <thead>
              <tr valign="top">
                <td colspan="3">Datasets and characteristics</td>
                <td>Autism</td>
                <td>Neurotypical</td>
                <td>Total</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="6">
                  <bold>Initial source dataset</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Number of videos</td>
                <td>985</td>
                <td>108</td>
                <td>1093</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Size (GB)</td>
                <td>33.11</td>
                <td>4.85</td>
                <td>37.96</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Number of boys</td>
                <td>879</td>
                <td>69</td>
                <td>948</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Number of girls</td>
                <td>106</td>
                <td>39</td>
                <td>145</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Age range (years)</td>
                <td>3-12</td>
                <td>3-12</td>
                <td>3-12</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="5">
                  <bold>Age group distribution (years</bold>
                  <bold>), n</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>
                  <break/>
                </td>
                <td>3-5</td>
                <td>565</td>
                <td>30</td>
                <td>595</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>
                  <break/>
                </td>
                <td>6-10</td>
                <td>263</td>
                <td>48</td>
                <td>311</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>
                  <break/>
                </td>
                <td>≥11</td>
                <td>157</td>
                <td>30</td>
                <td>187</td>
              </tr>
              <tr valign="top">
                <td colspan="6">
                  <bold>Filtered high-quality analysis dataset</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Unique individuals and videos, n</td>
                <td>50</td>
                <td>50</td>
                <td>100</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Number of boys</td>
                <td>25</td>
                <td>26</td>
                <td>51</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Number of girls</td>
                <td>25</td>
                <td>24</td>
                <td>49</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Age range (years)</td>
                <td>3-12</td>
                <td>3-12</td>
                <td>3-12</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="5">
                  <bold>Age group distribution (years</bold>
                  <bold>), n</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>
                  <break/>
                </td>
                <td>3-5</td>
                <td>17</td>
                <td>14</td>
                <td>31</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>
                  <break/>
                </td>
                <td>6-10</td>
                <td>15</td>
                <td>27</td>
                <td>42</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>
                  <break/>
                </td>
                <td>≥11</td>
                <td>18</td>
                <td>9</td>
                <td>27</td>
              </tr>
            </tbody>
          </table>
        </table-wrap>
      </sec>
      <sec>
        <title>Ethical Considerations</title>
        <p>This study was conducted in accordance with the Declaration of Helsinki and approved by the Stanford University Institutional Review Board (protocol 39562; approved March 21, 2023). The study team confirmed that Protocol 39562 and the associated parental consent process covered the restricted third-party crowdsourcing workflow used here. Parents or guardians provided informed consent for research use of the recordings, and the analyzed subset was selected from recordings for which consent also permitted public sharing. However, the source videos were not publicly released or provided to crowd workers. For crowdsourcing, individual deidentified, uncropped still frames sampled from the analyzed 30-second clips were placed in a private repository and made available only to authorized Microworkers participants through task-specific secure URLs. Workers had no access to source videos, participant identities, diagnostic labels, or a downloadable public image repository.</p>
      </sec>
      <sec>
        <title>Manual Filtration of High-Quality Videos</title>
        <p>To create a dataset suitable for evaluation, we manually screened the recordings for analyzable facial visibility and framing. For each retained recording, one contiguous 30-second segment was selected in which the target child’s face was sufficiently large, clear, and in focus, and no other face was the primary visible subject. Segments with persistent blur, severe occlusion, poor lighting, or an unsuitable dominant subject were excluded. For children with multiple recordings, only recordings meeting these criteria were included. Ten evenly spaced still frames were subsequently sampled from each selected segment for feature extraction. The curated dataset comprised 50 videos from unique children with autism and 50 videos from neurotypical children. The autism subset included 25 boys and 25 girls aged 3 to 12 years, while the neurotypical subset included 26 boys and 24 girls with a similar age distribution. Further demographic details are summarized in <xref ref-type="table" rid="table1">Table 1</xref>.</p>
      </sec>
      <sec>
        <title>Data Preprocessing and Frame Extraction</title>
        <p>Following the manual quality screening procedure described above, one contiguous 30-second segment was selected from each eligible recording. The selected segment prioritized clear, sufficiently large, and in-focus visibility of the target child’s face, without another face serving as the primary subject. Ten evenly spaced still frames were then sampled from each selected segment for FEFE and EG feature extraction. First, the total number of frames <italic>F</italic><sub>total</sub>in the video was calculated as the product of the video duration <italic>T</italic> (in seconds) and its frame rate <italic>fps</italic>. To illustrate (<xref rid="figure2" ref-type="fig">Figure 2</xref>), consider a 30-second video clip recorded at 30 frames per second, which would yield a total frame count as follows:</p>
        <fig id="figure2" position="float">
          <label>Figure 2</label>
          <caption>
            <p>Schematic of the systematic frame sampling procedure for feature extraction from home videos in the GuessWhat? autism study, illustrating the calculation of step value for selecting evenly spaced frames across a video's duration. ASD: autism spectrum disorder; NT: neurotypical.</p>
          </caption>
          <graphic xlink:href="ai_v5i1e84699_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <graphic xlink:href="ai_v5i1e84699_fig11.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        <p>To extract a representative sample of <italic>N</italic> frames (eg, 10 frames) from this video, a step value <italic>S</italic> was computed by dividing the total frames by the number of frames to be extracted:</p>
        <graphic xlink:href="ai_v5i1e84699_fig12.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        <graphic xlink:href="ai_v5i1e84699_fig13.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        <p>This calculated step value <italic>S</italic> ensured frames were evenly spaced, with every 90th frame selected for extraction in this example. Starting at frame 0, the first frame was extracted; then, proceeding at intervals of 90 frames, subsequent frames were extracted at positions <italic>F<sub>i</sub></italic>=<italic>i</italic> × <italic>S</italic>, where <italic>i</italic>=0,1,2,…9. Consequently, the extracted frame positions were 0, 90, 180, ..., 810, ensuring that frames were distributed evenly across the video. At 30 frames per second, these corresponded approximately to 0, 3, 6, 9, 12, 15, 18, 21, 24, and 27 seconds. While the specific step value differed for videos with different frame rates, the calculation method remained consistent. This systematic approach guaranteed a balanced and representative sample for facial expression tasks and minimized potential bias from frame selection, regardless of the source video’s frame rate.</p>
        <p>Sampled still frames were passed directly to the modality-specific feature extractors. The FEFE extractor passed each frame to the fer Python package, which returned scores for angry, disgust, fear, happy, sad, surprise, and neutral when a face or emotion result was available; when no result was returned, the implementation stored zeros for all 7 facial features. The EG extractor processed the same still frames using MediaPipe Face Mesh. The resulting frame-level features were standardized within each cross-validation training fold before classifier training.</p>
      </sec>
      <sec>
        <title>FEFE Model</title>
        <p>Our facial expression analysis involved the classification of 7 emotions (angry, disgust, fear, happy, sad, surprise, and neutral) using a pretrained facial expression recognition (FER) model implemented via the <italic>fer</italic> Python package [<xref ref-type="bibr" rid="ref29">29</xref>]. This model was originally trained on the FER-2013 dataset containing 35,887 grayscale images [<xref ref-type="bibr" rid="ref30">30</xref>]. We used this basic emotion model as an initial, interpretable proxy for social engagement and affective expression, which are core domains impaired in autism. Our choice of the 7 emotion categories was grounded in the Ekman foundational theoretical framework of universal basic emotions, providing a clinically intuitive and widely recognized starting point for automated screening [<xref ref-type="bibr" rid="ref31">31</xref>]. While action unit (AU) analysis provides more granular facial movement data, emotion classifications offer a higher level, clinically intuitive framework for initial automated screening [<xref ref-type="bibr" rid="ref32">32</xref>]. We acknowledge that applying this model to pediatric faces without fine-tuning represents a limitation, potentially reducing accuracy for child-specific expressions [<xref ref-type="bibr" rid="ref33">33</xref>].</p>
        <p>The preprocessing pipeline included automatic padding, face region normalization, and coordinate offsetting for optimal face framing. The emotion classifier generated probability distributions across the 7 emotion categories through local model inference, with each frame processed sequentially and results stored for further analysis. Conceptually, the pipeline supports an active learning framework where challenging samples identified through low confidence could be used to iteratively refine the model via human annotations. In this study, the FEFE model was used as a static feature extractor without retraining.</p>
        <p>Processed features from each modality were compiled into a unified, column-wise matrix for each video, creating a comprehensive dataset for robust multiclass classification. This approach ensured high-quality feature extraction while maintaining the flexibility to integrate human annotations when model confidence was low.</p>
      </sec>
      <sec>
        <title>EG Model</title>
        <p>As a secondary exploratory analysis, we used MediaPipe Face Mesh to derive a binary gaze feature from eye-corner geometry. For each detected face, the implementation used 4 eye-corner landmarks (33, 133, 362, and 263) to calculate left and right eye-vector angles with atan2. The absolute value of the mean of the 2 eye-vector angles was compared with a fixed threshold of 8 degrees. No eye aspect ratio, blink detection, or eye symmetry calculation was used. The stored EG feature was coded as eye contact=1 and no eye contact=0. For frames with detected facial landmarks, the implementation assigned 1 when the average absolute eye-vector angle G was below 8 degrees and assigned 0 when G≥8°. When Face Mesh did not detect facial landmarks, the implementation returned NaN, representing a missing numeric value rather than either gaze class. The full gaze-angle calculation and gaze-humanization procedure are provided in section S1.1 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>.</p>
      </sec>
      <sec>
        <title>Human Annotation Process Using Crowdsourcing</title>
        <p>To obtain high-quality human annotations, we designed and conducted “human selection” campaigns on Microworkers.com, ensuring a systematic approach to select the best-performing workers and generate reliable data annotations.</p>
        <p>Initially, we developed a strategy for selecting top-performing workers. We extracted 100 frames for emotion and gaze features from 10 high-quality GuessWhat? videos. For the pilot screening campaigns, we targeted approximately 100 workers per modality from the Microworkers international zone who met the specified platform performance rating criterion. Three top-performing workers were subsequently selected for the production annotation task. Crowdsourced annotators were specifically tasked with labeling discrete behavioral features, namely the presence of 1 of 7 emotion states and 1 of 2 EG classes, based on provided video frames. For the emotion annotation task, annotators were also provided with explicit criteria for the “no or partial face” (NPF) class: a frame should be labeled NPF if (1) the face was fully occluded, (2) less than 50% of the facial region was visible, or (3) the face was severely out-of-focus or poorly illuminated such that key landmarks could not be discerned. The gaze task used only binary classification (eye contact or no eye contact) without an NPF class. These selected samples covered diverse feature sets, enabling the evaluation of worker performance across varying feature vectors. Simultaneously, our research team collaboratively annotated these frames, with final labels verified by a senior domain expert, to establish a robust ground-truth benchmark. The selected frames were stored in a private repository and made available only to authorized Microworkers participants through task-specific secure URLs included in the task templates. These were uncropped, deidentified still frames sampled from the analyzed clips, not face-cropped images and not video clips. Each worker viewed individual frames only and did not receive access to the source video or its broader temporal context.</p>
        <p>To further protect privacy, access to the frames was restricted to the task interface via secure URLs, with no downloadable repository available. All crowd workers were bound by the Microworkers.com platform’s terms of service, which prohibit data misuse. Importantly, annotators had no access to participant identities, diagnostic labels, or full video context, limiting their task to labeling discrete behavioral features from isolated frames. Separate templates were created for each modality to simplify tasks for crowd workers. Workers were provided with adequate time and reasonable compensation to complete the tasks.</p>
        <p>Once the crowdsourced annotations were received, we compared them with the ground truth data to calculate each worker’s average performance. The performance evaluation of crowd workers, including accuracy and task completion time, is summarized in Figure S2 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>, demonstrating the selection of top-performing workers based on their reliability and efficiency. For the production task, the 100-video dataset contained 1000 sampled frames per modality. Each sampled frame had up to 3 annotations from the selected workers. The humanization pipeline derived a valid plurality label across the retained human-labeled entries from the available annotations. For facial features, a valid human plurality label changed the stored AI feature when the maximum AI probability was below the selected facial confidence threshold and the human label differed from the AI winning class. For gaze features, a valid human plurality label changed the stored binary AI value only when it differed from the AI value. Detailed step-by-step examples of the confidence-based replacement process, including visual illustrations of feature vector transformations across thresholds, are provided in section S1.2 and Table S1 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>. This process ensured that the annotations used in our study were accurate and reliable, ultimately enhancing the quality of the data for our research objectives.</p>
      </sec>
      <sec>
        <title>Confidence-Based HITL Annotation Replacement Process</title>
        <p>For FEFE, operational HITL decisions were based on maximum confidence, defined as the highest predicted probability among the 7 FEFE emotion classes for a given frame. When the stored FEFE probabilities were all zero, no AI winning class existed, and the row was treated as an unresolved AI output. For a nonzero row whose maximum probability was greater than or equal to the selected threshold, the AI winning class and its maximum probability were retained, while nonwinning class values were set to zero. For a nonzero row below the threshold, the corresponding human facial annotations were retrieved and normalized before voting. If the valid human plurality label differed from the AI winning class, the row was replaced by a 7-class 1-hot human vector; if the labels agreed, the existing AI row was retained. For an all-zero row, any valid human plurality label replaced the unresolved row with a 7-class 1-hot vector; if no valid human label was available, the all-zero row was retained.</p>
        <p>For the AI-only and HITL FEFE inputs, NPF was used as an annotation category and in descriptive class-mass analysis, but was not included as an eighth model-input feature. These FEFE inputs therefore contained 7 emotion features, and the corresponding multimodal inputs contained 8 columns: 7 FEFE features plus 1 binary EG feature. The separate pure-human comparator retained NPF as an eighth facial feature; consequently, its facial input was 8-dimensional, and its combined facial-plus-gaze input was 9-dimensional.</p>
        <p>Gaze humanization was performed separately from the facial confidence-threshold procedure. The EG feature was coded as eye contact=1 and no eye contact=0 and did not contain an NPF category. For each frame, valid human gaze labels were collected, and the majority label was selected; in a tie, no eye contact was selected. When the valid human plurality label differed from the stored AI binary value, the value was replaced with 1 for eye contact or 0 for no eye contact. When the labels agreed, or when no valid human gaze annotation was available, the stored AI value was retained.</p>
        <p>The FEFE human-intervention ratio (<italic>R</italic><sub>HITL</sub> (<italic>τ</italic>)) was calculated as:</p>
        <graphic xlink:href="ai_v5i1e84699_fig14.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        <p>Where <italic>N</italic><sub>zero,human</sub> is the number of all-zero FEFE rows that received a valid human plurality label, and <italic>N</italic><sub>nonzero,diff</sub> (τ) is the number of nonzero AI rows below threshold τ whose valid human plurality label differed from the AI winning class. <italic>N</italic><sub>frames</sub> is the total number of analyzed frames.</p>
        <p>The reported FEFE humanization ratio counted valid humanization of either unresolved all-zero FEFE rows or nonzero low-confidence rows with a differing human label. It excluded rows with no valid human label, NPF responses, same-label human annotations, and retained AI rows. It did not count every numeric row change, because high-confidence AI rows were also sparsified by retaining only the winning class. The EG replacement ratio was calculated as the percentage of binary AI values changed after valid binary gaze voting with the prespecified tie rule. The complete FEFE and EG workflows are provided in section S1.2 and Table S1 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>.</p>
      </sec>
      <sec>
        <title>Feature Aggregation, Model Training, and Evaluation</title>
        <p>The final classification of each video (and thus each participant) was performed by aggregating frame-level features across the entire video. Following the systematic sampling procedure detailed in the Data Preprocessing and Frame Extraction section, we extracted n=10 evenly spaced frames from each 30-second video. For each frame, we obtained a feature vector: for FEFE, this was a 7-dimensional probability vector, with valid low-confidence human replacements encoded as 7-dimensional 1-hot vectors; for EG, this was a 1-dimensional binary feature coded as 1 for eye contact and 0 for no eye contact. The sequence was processed by a 2-layer bidirectional long short-term memory (LSTM) with 64 hidden units per direction and dropout of 0.3. The implementation used the output at the final sequence position as the input to a fully connected sigmoid classifier. At this position, the forward-direction component reflects processing through the sequence, whereas the backward-direction component is the backward activation aligned with the final input position and is not, by itself, a conventional full-sequence reverse summary. The resulting output was used to generate 1 video-level autism-versus-neurotypical prediction. All reported performance metrics (accuracy, <italic>F</italic><sub>1</sub>-score, etc) were calculated based on these video-level predictions.</p>
        <p>For model evaluation, we used 5-fold cross-validation over the full 100-video analysis set. The 5 folds were created using shuffled k-fold cross-validation with a fixed random seed of 42. In each fold, 80 videos were used for model training, and the remaining 20 videos were held out for evaluation. Each video corresponded to 1 participant, and each participant contributed 1 held-out prediction across the complete cross-validation procedure. Feature standardization was fitted using the training data within each fold and then applied to the corresponding held-out data. A newly initialized bidirectional LSTM was trained separately in each fold for 50 epochs using Adam optimization with a learning rate of 0.001, binary cross-entropy loss, and a batch size of 16. Sigmoid outputs were converted to binary video-level predictions using a cutoff of 0.5; this classification cutoff was distinct from the facial HITL confidence thresholds of 0.1-0.9. Fold-level performance was summarized as the mean and SD of the 5 held-out fold metrics; thus, the mean (SD) values reported in subsections of Results and Figures S3, S4, and S6 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref> represent between-fold variability, not within-fold bootstrap variability.</p>
        <p>For aggregate out-of-fold evaluation, the held-out predictions from all 5 folds were concatenated so that the final evaluation set contained 1 prediction for each of the 100 videos. Aggregate performance metrics, the CIs, and aggregate confusion matrices shown in the last subsections of Results were calculated from this pooled out-of-fold set, not from a separate isolated 20-video test set. Wilson score 95% CIs were calculated for accuracy, sensitivity, and specificity. Percentile bootstrap 95% CIs, based on 2000 resamples with a fixed random seed of 42, were calculated for <italic>F</italic><sub>1</sub>-score and area under the receiver operating characteristic curve (AUROC).</p>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <sec>
        <title>Threshold-Dependent Performance of HITL Framework</title>
        <p>Overall, performance was higher at the upper end of the facial confidence-threshold sweep, although the relationship was nonmonotonic and did not improve over the AI-only baseline at every threshold. We evaluated unimodal autism-classification models using both FEFE and EG features. Because the FEFE configuration showed the stronger observed performance, the facial confidence-threshold sweep (<xref rid="figure3" ref-type="fig">Figure 3</xref> and Figures S3 and S4 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>) was our primary focus. We analyzed the performance of the FEFE model (<xref rid="figure3" ref-type="fig">Figure 3</xref>) across confidence thresholds (threshold 0.1-0.9) to evaluate the impact of progressively integrating HITL annotations. The valid FEFE humanization ratio, defined as the proportion of frames receiving a valid human replacement under the rule above, was 28.2% at thresholds of 0.1 and 0.2 and increased to 59.7% at a threshold of 0.9 (Table S2 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>).</p>
        <fig id="figure3" position="float">
          <label>Figure 3</label>
          <caption>
            <p>Facial emotion feature extraction performance across facial human-in-the-loop confidence thresholds (threshold 0.1-0.9). Lines show 5-fold mean sensitivity, specificity, F1-score, and area under the receiver operating characteristic curve; shaded bands represent (1 SD) across folds. The dotted line shows the proportion of frames receiving a valid human facial emotion feature extraction replacement. Performance improved overall from a threshold of 0.1 to a threshold of 0.9, although intermediate values were nonmonotonic. AUROC: area under the receiver operating characteristic curve.</p>
          </caption>
          <graphic xlink:href="ai_v5i1e84699_fig3.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <p>The mean (SD) <italic>F</italic><sub>1</sub>-score across 5 cross-validation folds increased from 0.484 (0.143) at a threshold of 0.1 to 0.643 (0.049) at a threshold of 0.9. Mean (SD) sensitivity increased from 0.514 (0.239) to 0.662 (0.103), mean specificity from 0.505 (0.132) to 0.611 (0.098), and mean AUROC from 0.481 (0.070) to 0.709 (0.060). <italic>F</italic><sub>1</sub>-score and AUROC were highest at a threshold of 0.9 among the evaluated thresholds. The temporary decrease in <italic>F</italic><sub>1</sub>-score at a threshold of 0.4 (mean 0.452, SD 0.074) shows that increasing the confidence threshold does not guarantee an improvement at each step, though we believe this is simply an artifact of our dataset.</p>
        <p>Precision-recall curves for the FEFE model are provided in Figure S5 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>. Out-of-fold average precision increased overall from 0.57 at a threshold of 0.1 to 0.72 at a threshold of 0.9, with the highest value observed at a threshold of 0.8. Because a threshold of 0.9 was selected after inspecting the threshold-sweep results obtained from the same 5-fold cross-validation data, this choice should be regarded as exploratory rather than as an independently validated operating point. The reported performance may therefore be optimistically biased because no nested cross-validation or independent tuning set was used.</p>
      </sec>
      <sec>
        <title>Analysis of Crowdsourced Annotations and Intervention Rates</title>
        <p><xref rid="figure4" ref-type="fig">Figure 4</xref>A summarizes class-level feature distributions across the 1000 sampled facial frames. The AI bars represent summed positive FEFE probability mass, whereas the human bars represent weighted human-label mass, including the NPF category. These are feature-mass summaries rather than equivalent frame counts. AI probability mass was highest for happy (229.9) and neutral (167.4). Weighted human-label mass was highest for happy (317.4), neutral (247.9), and NPF (178.6), indicating a broader and more distributed human-label profile.</p>
        <fig id="figure4" position="float">
          <label>Figure 4</label>
          <caption>
            <p>Class-level AI and weighted human feature distributions and threshold-dependent human-label replacement. Values are sums of fractions over 1000 frames. Each frame adds 1 to the human bars. Each frame adds 1 to the AI bars if a face was detected, and 0 if no face was detected. (A) AI bars show summed positive facial emotion feature extraction probability mass; human bars show weighted annotation mass, including the no or partial face category. (B) Blue bars show frequencies of positive AI emotion-class scores, including the &lt;0.1 bin; red values show the percentage of frames receiving a valid human facial emotion feature extraction replacement at each confidence threshold.</p>
          </caption>
          <graphic xlink:href="ai_v5i1e84699_fig4.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <p><xref rid="figure4" ref-type="fig">Figure 4</xref>B presents 2 separate quantities. The blue bars show the frequency of positive AI emotion-class scores in confidence bins; the &lt;0.1 bin includes positive scores below 0.1. Because 1 frame may contribute several positive class scores, these frequencies are not frame counts and sum to 3105 rather than 1000. The red line shows the percentage of frames receiving a valid human FEFE replacement under each threshold, including valid human labels assigned to unresolved all-zero AI rows. The actual replacement percentages increased from 28.2% at a threshold of 0.1 (and threshold of 0.2) to 59.7% at a threshold of 0.9. At a threshold of 0.9, 597 of 1000 frames received a valid human replacement under this definition. The remaining frames did not receive a different human label.</p>
        <p>For the separate EG workflow, gaze humanization changed 483 of the 1000 stored binary gaze values (48.3%): 206 frames changed from no eye contact to eye contact, and 277 changed from eye contact to no eye contact. The AI-only gaze features contained 758 eye contact values and 242 no eye contact values; after gaze humanization, these counts were 687 and 313, respectively.</p>
        <p>Pure human annotations for FEFE alone (Figure S6A in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>) and combined FEFE with EG ( Figure S6B in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>) achieved <italic>F</italic><sub>1</sub>-scores of 0.593 (0.150) and 0.672 (0.078), respectively, with the combined annotations showing a 48.3% reduction in fold-level <italic>F</italic><sub>1</sub>-score variability compared with pure FEFE, calculated from the unrounded fold-level SDs.</p>
      </sec>
      <sec>
        <title>Performance of Unimodal Feature Extraction</title>
        <p>The comparison between the AI-only and HITL FEFE models showed higher mean performance for the HITL configuration across all 5 reported metrics. <xref rid="figure5" ref-type="fig">Figure 5</xref> presents the HITL FEFE model with a facial confidence threshold of 0.9. The mean (SD) <italic>F</italic><sub>1</sub>-score increased from 0.518 (0.039) to 0.643 (0.049), corresponding to a 24.1% relative gain; the SD increased by 25.6%. Mean (SD) recall (sensitivity) increased from 0.526 (0.083) to 0.662 (0.103), a 25.8% gain, with a 24.1% increase in SD. Mean accuracy (SD) increased from 0.520 (0.040) to 0.640 (0.049), a 23.1% gain, with a 22.5% increase in SD. Mean (SD) precision increased from 0.522 (0.050) to 0.634 (0.022), a 21.5% gain, while its SD decreased by 56%. Mean (SD) AUROC increased from 0.558 (0.059) to 0.709 (0.060), a 27.1% gain, with a small 1.7% increase in SD. These results show higher mean performance for the HITL FEFE configuration across all 5 reported metrics. However, the SD did not decrease uniformly: precision variability decreased, whereas accuracy, recall, and <italic>F</italic><sub>1</sub>-score variability increased slightly and AUROC variability remained nearly unchanged.</p>
        <fig id="figure5" position="float">
          <label>Figure 5</label>
          <caption>
            <p>Fold-level mean (SD) performance of facial emotion feature extraction models: (A) AI-only and (B) human-in-the-loop at facial confidence threshold 0.9. The human-in-the-loop configuration showed higher observed mean performance in this exploratory analysis. AUROC: area under the receiver operating characteristic curve.</p>
          </caption>
          <graphic xlink:href="ai_v5i1e84699_fig5.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <p>For the EG model, HITL yielded mixed outcomes with distinct trade-offs (<xref rid="figure6" ref-type="fig">Figure 6</xref>). The mean (SD) <italic>F</italic><sub>1</sub>-score increased slightly from 0.428 (0.159) to 0.460 (0.180), an approximate 7.5% gain; however, its SD increased by approximately 13.2%. Mean (SD) recall increased slightly from 0.457 (0.305) to 0.466 (0.191), an approximate 2% gain, while its SD decreased by approximately 37.4%. Accuracy remained unchanged at 0.490, increasing in variability from 0.086 to 0.128 (approximately 48.8%). Mean (SD) precision decreased from 0.515 (0.143) to 0.461 (0.163), an approximate 10.5% reduction, while its SD increased by approximately 14.0%. Mean (SD) AUROC decreased from 0.526 (0.117) to 0.511 (0.068), an approximate 2.9% reduction, while its SD decreased by approximately 41.9%. These results indicate mixed effects of gaze humanization: small gains in mean <italic>F</italic><sub>1</sub>-score and recall, unchanged accuracy, and lower precision and AUROC.</p>
        <fig id="figure6" position="float">
          <label>Figure 6</label>
          <caption>
            <p>Performance comparison of eye gaze models (A) without and (B) with the human-in-the-loop approach. Gaze humanization produced mixed results: mean F1-score and recall increased slightly, accuracy was unchanged, and mean precision and area under the receiver operating characteristic curve decreased. AUROC: area under the receiver operating characteristic curve.</p>
          </caption>
          <graphic xlink:href="ai_v5i1e84699_fig6.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>Multimodal Feature Fusion Analysis</title>
        <p>For the AI-only and HITL multimodal configurations, the model concatenated the 7 FEFE emotion features with 1 binary EG feature, coded as eye contact=1 and no eye contact=0, giving an 8-column frame-level input. The resulting sequence was then processed with a bidirectional LSTM for autism-versus-neurotypical classification. We evaluated matched AI-only and HITL-enhanced multimodal configurations using the same 5-fold cross-validation splits. <xref rid="figure7" ref-type="fig">Figure 7</xref> reports fold-level means and SDs. The HITL-enhanced configuration at a threshold of 0.9 showed higher mean accuracy, <italic>F</italic><sub>1</sub>-score, sensitivity, precision, and AUROC than the AI-only configuration in the fold-level summary, although the magnitude of improvement varied across metrics and folds. These results indicate an overall fold-level improvement rather than uniformly superior performance in every fold. Aggregate out-of-fold metrics, calculated by concatenating 1 held-out prediction per video across all 5 folds, are reported separately in <xref rid="figure8" ref-type="fig">Figure 8</xref>.</p>
        <fig id="figure7" position="float">
          <label>Figure 7</label>
          <caption>
            <p>Fold-level mean (SD) performance across the same 5 cross-validation folds for multimodal facial emotion feature extraction+eye gaze classification: (A) AI-only and (B) human-in-the-loop–enhanced at a facial confidence threshold of 0.9. Aggregate out-of-fold metrics from the same 5-fold evaluation are reported separately in Figure 8. AUROC: area under the receiver operating characteristic curve.</p>
          </caption>
          <graphic xlink:href="ai_v5i1e84699_fig7.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <fig id="figure8" position="float">
          <label>Figure 8</label>
          <caption>
            <p>Aggregate out-of-fold performance and 95% CIs for the AI-only and human-in-the-loop–enhanced multimodal models across 100 videos. Human-in-the-loop point estimates were higher for all metrics, with nonoverlapping displayed area under the receiver operating characteristic curve intervals: AI-only, 0.54 (0.42-0.65); human-in-the-loop, 0.76 (0.66-0.85). AUROC: area under the receiver operating characteristic curve; HILT: human-in-the-loop.</p>
          </caption>
          <graphic xlink:href="ai_v5i1e84699_fig8.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <p>The fold-level means also favored HITL for accuracy, <italic>F</italic><sub>1</sub>-score, and AUROC in every fold. Precision was higher in 4 of 5 folds and recall was higher in 4 of 5 folds. Thus, the results support an overall multimodal HITL improvement, but they should not be described as uniformly better for every metric in every fold.</p>
      </sec>
      <sec>
        <title>Statistical Validation and Error Analysis</title>
        <p>We quantified uncertainty using the aggregate out-of-fold prediction set from the same 5-fold cross-validation procedure. Each fold contained 20 held-out videos, giving 1 held-out prediction for each of the 100 videos. Wilson score 95% CIs were calculated for accuracy, sensitivity, and specificity, while percentile bootstrap 95% CIs were calculated for <italic>F</italic><sub>1</sub>-score and AUROC. For the HITL-enhanced multimodal model, using humanized FEFE features at a facial confidence threshold of 0.9 together with humanized gaze features, AUROC was 0.76 (95% CI 0.66-0.85), compared with 0.54 (95% CI 0.42-0.65) for the AI-only multimodal model (<xref rid="figure8" ref-type="fig">Figure 8</xref>). The intervals displayed for AUROC do not overlap, which is consistent with a higher HITL AUROC; however, CI nonoverlap is not itself a formal hypothesis test.</p>
        <p>The aggregate confusion matrix (<xref rid="figure9" ref-type="fig">Figure 9</xref>) for the AI-only model contained 28 correctly classified neurotypical videos and 22 neurotypical-to-ASD errors, together with 25 correctly classified ASD videos and 25 ASD-to-neurotypical errors. The 25/25 ASD row represents the model’s aggregate performance on 50 ASD videos: sensitivity was 50%, indicating chance-level separation for that class. This is a valid confusion-matrix outcome, not duplicated data. The HITL model correctly classified 33 neurotypical videos and 35 ASD videos, with 17 false positives and 15 false negatives. Total errors decreased from 47 with AI-only features to 32 with HITL features, an approximate 31.9% reduction.</p>
        <fig id="figure9" position="float">
          <label>Figure 9</label>
          <caption>
            <p>Aggregate confusion matrices for the AI-only multimodal model and the human-in-the-loop–enhanced multimodal model using humanized facial emotion feature extraction at facial confidence threshold of 0.9 and humanized gaze features. ASD: autism spectrum disorder; HILT: human-in-the-loop; NT: neurotypical.</p>
          </caption>
          <graphic xlink:href="ai_v5i1e84699_fig9.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <p>To compare the matched fold-level metrics, we applied 2-sided paired exact Wilcoxon signed-rank tests across the 5 folds (<xref rid="figure10" ref-type="fig">Figure 10</xref>). Fold-level comparisons summarize variation across the 5 resampling folds rather than 5 independent experiments. The mean fold-level gains were +0.150 for accuracy, +0.117 for precision, +0.175 for recall, +0.174 for <italic>F</italic><sub>1</sub>-score, and +0.195 for AUROC. The corresponding <italic>P</italic> values were .0625 for accuracy, <italic>F</italic><sub>1</sub>-score, and AUROC, 0.1250 for precision, and 0.4375 for recall. Because only 5 paired folds were available, the minimum attainable exact 2-sided <italic>P</italic> value was .0625. None of these comparisons reached <italic>P</italic>&lt;.05; therefore, the results should be considered as descriptive fold-level improvements rather than statistically significant effects.</p>
        <fig id="figure10" position="float">
          <label>Figure 10</label>
          <caption>
            <p>Two-sided paired exact Wilcoxon signed-rank tests comparing AI-only and human-in-the-loop–enhanced multimodal performance across 5 matched cross-validation folds; the minimum attainable <italic>P</italic> value is .0625. AUROC: area under the receiver operating characteristic curve; HILT: human-in-the-loop.</p>
          </caption>
          <graphic xlink:href="ai_v5i1e84699_fig10.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <sec>
        <title>Principal Findings</title>
        <p>We present a HITL AI framework for classifying autism from home videos. In our pipeline, AI models extracted behavioral features for each frame of a video, and these AI-extracted features were used for final classification. We replaced these AI-extracted features with human annotations provided by crowd workers in cases where the confidence score of the AI model was below a certain threshold for a particular frame, and we explored the effects of varying the confidence threshold on model performance and the number of human annotations required. We found that, in general, replacing the AI features with more human features tended to improve performance.</p>
      </sec>
      <sec>
        <title>Limitations</title>
        <p>Several limitations must be acknowledged. First, variability in nonexpert crowd annotations may have introduced noise, despite rigorous selection of crowd workers. Because some workers selected multiple emotion classes and ties were resolved according to the order of the features (an arbitrary process), the procedure may have been sensitive to column ordering in case of ties. Second, the gaze pipeline used for our secondary analysis did not include a separate NPF or missing-data category like the FEFE analysis did. When Face Mesh did not detect facial landmarks, the implementation returned NaN, representing missing numeric data rather than eye contact or no eye contact. Third, the dataset was limited in size (n=100) despite demographic diversity, which may affect generalizability. Fourth, the binary classification framework (autism vs neurotypical) may have oversimplified autism’s heterogeneity; dimensional approaches capturing symptom severity could provide richer phenotypes. Fifth, the temporal modeling capacity of our bidirectional LSTM was constrained by the frame-sampling rate (approximately 0.33 Hz), which was insufficient to capture millisecond-scale dynamics like microexpressions. The model thus likely learned longer-duration behavioral state transitions rather than fine-grained temporal patterns. Sixth, our reliance on 7 basic emotion categories, rather than more granular facial AUs, may have limited sensitivity to subtle, autism-specific differences in facial expressivity [<xref ref-type="bibr" rid="ref32">32</xref>,<xref ref-type="bibr" rid="ref34">34</xref>,<xref ref-type="bibr" rid="ref35">35</xref>]. Emotion labels represent a higher-level abstraction that may not capture the full spectrum of atypical social communication. Seventh, our manual filtration for high video quality (clear facial visibility, center-focused framing) may have introduced selection bias. While necessary for reliable AI-based feature extraction, this process could have systematically excluded data from children with more pronounced symptoms, such as those who frequently avoid the camera or exhibit high motor restlessness. This may limit the tool’s applicability to the full clinical spectrum of autism. Eighth, the construct validity of using a game-based task to measure social affect warrants consideration. In the charades gameplay, children were prompted to enact specific emotions. Consequently, the FEFE model assessed the child’s ability to comply with instructions and produce recognizable posed expressions, rather than purely spontaneous social affect. Thus, our trained models do not necessarily generalize to naturalistic videos that are not constrained by specific gameplay instructions. Ninth, average precision peaked at a threshold of 0.8 (0.73) and was slightly lower at a threshold of 0.9 (0.72). This indicates that threshold selection involved a trade-off across performance measures and should be evaluated in an independent held-out test set in future work.</p>
        <p>Finally, we must note that our preliminary experiments with 4 additional behavioral models yielded inconclusive results, highlighting key challenges in multimodal phenotyping from unstructured home videos. First, the Behavioral Activity Detection Model using ResNet-3D pretrained on Kinetics-400 [<xref ref-type="bibr" rid="ref36">36</xref>,<xref ref-type="bibr" rid="ref37">37</xref>] failed to align with our data; selected activity classes (eg, applauding and laughing) often mismatched with extracted frames, which frequently showed unrelated or ambiguous movements. Second, the Audio-Linguistic Recognition Model using Wav2Vec2-Base-960h [<xref ref-type="bibr" rid="ref38">38</xref>] and Emotion-English-DistilRoBERTa [<xref ref-type="bibr" rid="ref39">39</xref>,<xref ref-type="bibr" rid="ref40">40</xref>] could not reliably extract emotional content, as audio segments lacked distinct or consistent vocal expressions matching the predefined emotion classes. Third, the Gesture-Centric Behavioral Model using ResNet-18 pretrained [<xref ref-type="bibr" rid="ref41">41</xref>] on ImageNet failed to accurately classify gestures such as pointing or waving, as many frames lacked explicit depictions or showed movements too subtle for reliable detection. Fourth, a Self-Stimulatory Behavior (SSB) Model we trained using MobileNetV2 [<xref ref-type="bibr" rid="ref42">42</xref>] could not detect target behaviors (headbanging, spinning, and arm-flapping) due to their rarity or ambiguity in the sampled frames. These series of negative results on other AI-related features that we explored highlights ongoing challenges with leveraging domain-specific AI models to extract features from video. We note that newer vision language models (VLMs), which are like large language models (LLMs) but take a video or image as an input (in addition to a prompt), can potentially solve these issues and should certainly be explored in future work.</p>
      </sec>
      <sec>
        <title>Future Directions</title>
        <p>There are several opportunities for future work. First, expanding the dataset with a broader spectrum of participants and incorporating multimodal data (eg, audio, gestures, and wearable sensors) will enhance robustness and generalizability [<xref ref-type="bibr" rid="ref43">43</xref>]. Second, using expert annotations or multiple annotators per frame would enable formal interrater reliability assessment and improve annotation quality. Third, developing autism-specific models rather than using generic pretrained models could improve base AI performance and reduce the human annotation burden, particularly for behavioral activity, audio-linguistic, gesture, and SSB detection. Fourth, incorporating AU-based features would provide more granular facial movement analysis and potentially greater sensitivity to atypical expressivity than emotion feature vectors. Fifth, future implementations should seek to develop more robust models capable of handling variable video quality to capture behavioral phenotypes in less constrained settings, addressing the selection bias introduced by our quality filtration approach. Sixth, longitudinal studies are needed to evaluate the clinical scalability and real-world effectiveness of HITL-enhanced detection over time. Seventh, future studies should directly compare phenotypes extracted from posed (ie, the charades data collection game) versus spontaneous (ie, the quizzes data collection game) video segments within the same dataset to better isolate markers of spontaneous social engagement specific to autism. Eighth, optimizing the human-AI cost-accuracy balance through systematic cost-benefit analyses will be essential for real-world deployment, particularly given the substantial level of valid human FEFE replacement at the highest observed-performing threshold.</p>
      </sec>
      <sec>
        <title>Conclusions</title>
        <p>In this proof-of-concept study, the evaluated confidence-driven HITL framework was associated with higher observed autism classification performance in this dataset when a higher proportion of AI features were replaced with human features. That is, among 9 evaluated facial confidence thresholds, the threshold of 0.9 had the highest observed mean <italic>F</italic><sub>1</sub>-score and AUROC and a valid human FEFE replacement ratio of 59.7%. While these exploratory findings describe our novel HITL AI approach, they do not establish the independent causal effect of human replacement, clinical diagnostic validity, or generalizability to less constrained recordings.</p>
      </sec>
    </sec>
  </body>
  <back>
    <app-group>
      <supplementary-material id="app1">
        <label>Multimedia Appendix 1</label>
        <p>Deatils of the eye-gaze and facial emotion models, human-in-the-loop replacement workflows, crowd-worker evaluations, threshold performance sweeps, precision-recall curves, and pure-human baseline evaluations.</p>
        <media xlink:href="ai_v5i1e84699_app1.docx" xlink:title="DOCX File , 5771 KB"/>
      </supplementary-material>
    </app-group>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">ADI-R</term>
          <def>
            <p>Autism Diagnostic Interview–Revised</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">ADOS-2</term>
          <def>
            <p>Autism Diagnostic Observation Schedule, Second Edition</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">ASD</term>
          <def>
            <p>autism spectrum disorder</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb4">AU</term>
          <def>
            <p>action unit</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb5">AUROC</term>
          <def>
            <p>area under the receiver operating characteristic curve</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb6">EG</term>
          <def>
            <p>eye gaze</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb7">FEFE</term>
          <def>
            <p>facial emotion feature extraction</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb8">FER</term>
          <def>
            <p>facial expression recognition</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb9">HITL</term>
          <def>
            <p>human-in-the-loop</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb10">LLM</term>
          <def>
            <p>large language model</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb11">LSTM</term>
          <def>
            <p>long short-term memory</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb12">NPF</term>
          <def>
            <p>no or partial face</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb13">SSB</term>
          <def>
            <p>self-stimulatory behavior</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb14">VLM</term>
          <def>
            <p>vision language model</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>The content is solely the responsibility of the authors and does not necessarily represent the official views of the National Institutes of Health. We acknowledge the use of AI tools (ChatGPT) to edit the language of the manuscript; we emphasize that all ideas and analyses are original and not AI-generated. Figure S1 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref> was created using the generative AI tool DALL-E 3 by OpenAI.</p>
    </ack>
    <notes>
      <title>Data Availability</title>
      <p>The datasets generated or analyzed during this study are not publicly available due to the sensitive nature of the data (ie, pediatric population and protected health information) but are available from the corresponding author upon reasonable request.</p>
    </notes>
    <notes>
      <title>Funding</title>
      <p>This work was funded by the National Institutes of Health (DP2EB035858, R01LM014342, R01LM013364) and the Stanford Center for Digital Health. The content is solely the responsibility of the authors and does not necessarily represent the official views of the National Institutes of Health.</p>
    </notes>
    <fn-group>
      <fn fn-type="con">
        <p>Conceptualization: PW, DW</p>
        <p>Data curation: KD, AH, AK, DW</p>
        <p>Formal analysis: DW, PW, AR</p>
        <p>Funding acquisition: PW</p>
        <p>Investigation: PW, DW</p>
        <p>Methodology: AR, PW</p>
        <p>Project administration: PW</p>
        <p>Resources: PW, AR</p>
        <p>Supervision: PW</p>
        <p>Validation: PW, AR</p>
        <p>Visualization: AR</p>
        <p>Writing – original draft: AR</p>
        <p>Writing – review &amp; editing: AR, PW, DW, PA</p>
      </fn>
      <fn fn-type="conflict">
        <p>None declared.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Perochon</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Di Martino</surname>
              <given-names>JM</given-names>
            </name>
            <name name-style="western">
              <surname>Carpenter</surname>
              <given-names>KLH</given-names>
            </name>
            <name name-style="western">
              <surname>Compton</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Davis</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Eichner</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Espinosa</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Franz</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Krishnappa Babu</surname>
              <given-names>PR</given-names>
            </name>
            <name name-style="western">
              <surname>Sapiro</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Dawson</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Early detection of autism using digital behavioral phenotyping</article-title>
          <source>Nat Med</source>
          <year>2023</year>
          <volume>29</volume>
          <issue>10</issue>
          <fpage>2489</fpage>
          <lpage>2497</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/37783967"/>
          </comment>
          <pub-id pub-id-type="doi">10.1038/s41591-023-02574-3</pub-id>
          <pub-id pub-id-type="medline">37783967</pub-id>
          <pub-id pub-id-type="pii">10.1038/s41591-023-02574-3</pub-id>
          <pub-id pub-id-type="pmcid">PMC10579093</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Daniels</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Schwartz</surname>
              <given-names>JN</given-names>
            </name>
            <name name-style="western">
              <surname>Voss</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Haber</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Fazel</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Kline</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Washington</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Feinstein</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Winograd</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Wall</surname>
              <given-names>DP</given-names>
            </name>
          </person-group>
          <article-title>Exploratory study examining the at-home feasibility of a wearable tool for social-affective learning in children with autism</article-title>
          <source>NPJ Digit Med</source>
          <year>2018</year>
          <volume>1</volume>
          <fpage>32</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1038/s41746-018-0035-3"/>
          </comment>
          <pub-id pub-id-type="doi">10.1038/s41746-018-0035-3</pub-id>
          <pub-id pub-id-type="medline">31304314</pub-id>
          <pub-id pub-id-type="pii">35</pub-id>
          <pub-id pub-id-type="pmcid">PMC6550272</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hyman</surname>
              <given-names>SL</given-names>
            </name>
            <name name-style="western">
              <surname>Levy</surname>
              <given-names>SE</given-names>
            </name>
            <name name-style="western">
              <surname>Myers</surname>
              <given-names>SM</given-names>
            </name>
            <collab>COUNCIL ON CHILDREN WITH DISABILITIES‚ SECTION ON DEVELOPMENTAL AND BEHAVIORAL PEDIATRICS</collab>
          </person-group>
          <article-title>Identification, evaluation, and management of children with autism spectrum disorder</article-title>
          <source>Pediatrics</source>
          <year>2020</year>
          <volume>145</volume>
          <issue>1</issue>
          <fpage>e20193447</fpage>
          <pub-id pub-id-type="doi">10.1542/peds.2019-3447</pub-id>
          <pub-id pub-id-type="medline">31843864</pub-id>
          <pub-id pub-id-type="pii">peds.2019-3447</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Shaw</surname>
              <given-names>KA</given-names>
            </name>
            <name name-style="western">
              <surname>Williams</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Patrick</surname>
              <given-names>ME</given-names>
            </name>
            <name name-style="western">
              <surname>Valencia-Prado</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Durkin</surname>
              <given-names>MS</given-names>
            </name>
            <name name-style="western">
              <surname>Howerton</surname>
              <given-names>EM</given-names>
            </name>
            <name name-style="western">
              <surname>Ladd-Acosta</surname>
              <given-names>CM</given-names>
            </name>
            <name name-style="western">
              <surname>Pas</surname>
              <given-names>ET</given-names>
            </name>
            <name name-style="western">
              <surname>Bakian</surname>
              <given-names>AV</given-names>
            </name>
            <name name-style="western">
              <surname>Bartholomew</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Nieves-Muñoz</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Sidwell</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Alford</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Bilder</surname>
              <given-names>DA</given-names>
            </name>
            <name name-style="western">
              <surname>DiRienzo</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Fitzgerald</surname>
              <given-names>RT</given-names>
            </name>
            <name name-style="western">
              <surname>Furnier</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Hudson</surname>
              <given-names>AE</given-names>
            </name>
            <name name-style="western">
              <surname>Pokoski</surname>
              <given-names>OM</given-names>
            </name>
            <name name-style="western">
              <surname>Shea</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Tinker</surname>
              <given-names>SC</given-names>
            </name>
            <name name-style="western">
              <surname>Warren</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Zahorodny</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Agosto-Rosa</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Anbar</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Chavez</surname>
              <given-names>KY</given-names>
            </name>
            <name name-style="western">
              <surname>Esler</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Forkner</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Grzybowski</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Agib</surname>
              <given-names>AH</given-names>
            </name>
            <name name-style="western">
              <surname>Hallas</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Lopez</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Magaña</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Nguyen</surname>
              <given-names>RHN</given-names>
            </name>
            <name name-style="western">
              <surname>Parker</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Pierce</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Protho</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Torres</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Vanegas</surname>
              <given-names>SB</given-names>
            </name>
            <name name-style="western">
              <surname>Vehorn</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Andrews</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Greer</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Hall-Lande</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>McArthur</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Mitamura</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Montes</surname>
              <given-names>AJ</given-names>
            </name>
            <name name-style="western">
              <surname>Pettygrove</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Shenouda</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Skowyra</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Washington</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Maenner</surname>
              <given-names>MJ</given-names>
            </name>
          </person-group>
          <article-title>Prevalence and early identification of autism spectrum disorder among children aged 4 and 8 years - autism and developmental disabilities monitoring network, 16 sites, United States, 2022</article-title>
          <source>MMWR Surveill Summ</source>
          <year>2025</year>
          <volume>74</volume>
          <issue>2</issue>
          <fpage>1</fpage>
          <lpage>22</lpage>
          <pub-id pub-id-type="doi">10.15585/mmwr.ss7402a1</pub-id>
          <pub-id pub-id-type="medline">40232988</pub-id>
          <pub-id pub-id-type="pmcid">PMC12011386</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ning</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Daniels</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Schwartz</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Dunlap</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Washington</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Kalantarian</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Du</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Wall</surname>
              <given-names>DP</given-names>
            </name>
          </person-group>
          <article-title>Identification and quantification of gaps in access to autism resources in the United States: an infodemiological study</article-title>
          <source>J Med Internet Res</source>
          <year>2019</year>
          <volume>21</volume>
          <issue>7</issue>
          <fpage>e13094</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2019/7/e13094/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/13094</pub-id>
          <pub-id pub-id-type="medline">31293243</pub-id>
          <pub-id pub-id-type="pii">v21i7e13094</pub-id>
          <pub-id pub-id-type="pmcid">PMC6652124</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Geschwind</surname>
              <given-names>DH</given-names>
            </name>
          </person-group>
          <article-title>Advances in autism</article-title>
          <source>Annu Rev Med</source>
          <year>2009</year>
          <volume>60</volume>
          <fpage>367</fpage>
          <lpage>380</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/19630577"/>
          </comment>
          <pub-id pub-id-type="doi">10.1146/annurev.med.60.053107.121225</pub-id>
          <pub-id pub-id-type="medline">19630577</pub-id>
          <pub-id pub-id-type="pmcid">PMC3645857</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Voss</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Schwartz</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Daniels</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Kline</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Haber</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Washington</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Tariq</surname>
              <given-names>Q</given-names>
            </name>
            <name name-style="western">
              <surname>Robinson</surname>
              <given-names>TN</given-names>
            </name>
            <name name-style="western">
              <surname>Desai</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Phillips</surname>
              <given-names>JM</given-names>
            </name>
            <name name-style="western">
              <surname>Feinstein</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Winograd</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Wall</surname>
              <given-names>DP</given-names>
            </name>
          </person-group>
          <article-title>Effect of wearable digital intervention for improving socialization in children with autism spectrum disorder: a randomized clinical trial</article-title>
          <source>JAMA Pediatr</source>
          <year>2019</year>
          <volume>173</volume>
          <issue>5</issue>
          <fpage>446</fpage>
          <lpage>454</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/30907929"/>
          </comment>
          <pub-id pub-id-type="doi">10.1001/jamapediatrics.2019.0285</pub-id>
          <pub-id pub-id-type="medline">30907929</pub-id>
          <pub-id pub-id-type="pii">2728462</pub-id>
          <pub-id pub-id-type="pmcid">PMC6503634</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Washington</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Kalantarian</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Tariq</surname>
              <given-names>Q</given-names>
            </name>
            <name name-style="western">
              <surname>Schwartz</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Dunlap</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Chrisman</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Varma</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Ning</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Kline</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Stockham</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Paskov</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Voss</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Haber</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Wall</surname>
              <given-names>DP</given-names>
            </name>
          </person-group>
          <article-title>Validity of online screening for autism: crowdsourcing study comparing paid and unpaid diagnostic tasks</article-title>
          <source>J Med Internet Res</source>
          <year>2019</year>
          <volume>21</volume>
          <issue>5</issue>
          <fpage>e13668</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2019/5/e13668/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/13668</pub-id>
          <pub-id pub-id-type="medline">31124463</pub-id>
          <pub-id pub-id-type="pii">v21i5e13668</pub-id>
          <pub-id pub-id-type="pmcid">PMC6552453</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lai</surname>
              <given-names>MC</given-names>
            </name>
            <name name-style="western">
              <surname>Lombardo</surname>
              <given-names>MV</given-names>
            </name>
            <name name-style="western">
              <surname>Auyeung</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Chakrabarti</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Baron-Cohen</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Sex/gender differences and autism: setting the scene for future research</article-title>
          <source>J Am Acad Child Adolesc Psychiatry</source>
          <year>2015</year>
          <volume>54</volume>
          <issue>1</issue>
          <fpage>11</fpage>
          <lpage>24</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S0890-8567(14)00725-4"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.jaac.2014.10.003</pub-id>
          <pub-id pub-id-type="medline">25524786</pub-id>
          <pub-id pub-id-type="pii">S0890-8567(14)00725-4</pub-id>
          <pub-id pub-id-type="pmcid">PMC4284309</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kalantarian</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Washington</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Schwartz</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Daniels</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Haber</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Wall</surname>
              <given-names>DP</given-names>
            </name>
          </person-group>
          <article-title>Guess What?: towards understanding autism from structured video using facial affect</article-title>
          <source>J Healthc Inform Res</source>
          <year>2019</year>
          <volume>3</volume>
          <issue>1</issue>
          <fpage>43</fpage>
          <lpage>66</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/33313475"/>
          </comment>
          <pub-id pub-id-type="doi">10.1007/s41666-018-0034-9</pub-id>
          <pub-id pub-id-type="medline">33313475</pub-id>
          <pub-id pub-id-type="pmcid">PMC7730314</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Pierce</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Gazestani</surname>
              <given-names>VH</given-names>
            </name>
            <name name-style="western">
              <surname>Bacon</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Barnes</surname>
              <given-names>CC</given-names>
            </name>
            <name name-style="western">
              <surname>Cha</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Nalabolu</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Lopez</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Moore</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Pence-Stophaeros</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Courchesne</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Evaluation of the diagnostic stability of the early autism spectrum disorder phenotype in the general population starting at 12 months</article-title>
          <source>JAMA Pediatr</source>
          <year>2019</year>
          <volume>173</volume>
          <issue>6</issue>
          <fpage>578</fpage>
          <lpage>587</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/31034004"/>
          </comment>
          <pub-id pub-id-type="doi">10.1001/jamapediatrics.2019.0624</pub-id>
          <pub-id pub-id-type="medline">31034004</pub-id>
          <pub-id pub-id-type="pii">2732144</pub-id>
          <pub-id pub-id-type="pmcid">PMC6547081</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Elbattah</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Guérin</surname>
              <given-names>JL</given-names>
            </name>
            <name name-style="western">
              <surname>Carette</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Cilia</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Dequen</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>NLP-based approach to detect autism spectrum disorder in saccadic eye movement</article-title>
          <year>2020</year>
          <conf-name>IEEE Symposium Series on Computational Intelligence (SSCI)</conf-name>
          <conf-date>December 01-04, 2020</conf-date>
          <conf-loc>Canberra, ACT, Australia</conf-loc>
          <publisher-name>IEEE</publisher-name>
          <fpage>1581</fpage>
          <lpage>1587</lpage>
          <pub-id pub-id-type="doi">10.1109/ssci47803.2020.9308238</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Washington</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Wall</surname>
              <given-names>DP</given-names>
            </name>
          </person-group>
          <article-title>A review of and roadmap for data science and machine learning for the neuropsychiatric phenotype of autism</article-title>
          <source>Annu Rev Biomed Data Sci</source>
          <year>2023</year>
          <volume>6</volume>
          <fpage>211</fpage>
          <lpage>228</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.annualreviews.org/content/journals/10.1146/annurev-biodatasci-020722-125454?crawler=true&amp;mimetype=application/pdf"/>
          </comment>
          <pub-id pub-id-type="doi">10.1146/annurev-biodatasci-020722-125454</pub-id>
          <pub-id pub-id-type="medline">37137169</pub-id>
          <pub-id pub-id-type="pmcid">PMC11093217</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Stevens</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Dixon</surname>
              <given-names>DR</given-names>
            </name>
            <name name-style="western">
              <surname>Novack</surname>
              <given-names>MN</given-names>
            </name>
            <name name-style="western">
              <surname>Granpeesheh</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Smith</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Linstead</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Identification and analysis of behavioral phenotypes in autism spectrum disorder via unsupervised machine learning</article-title>
          <source>Int J Med Inform</source>
          <year>2019</year>
          <volume>129</volume>
          <fpage>29</fpage>
          <lpage>36</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://digitalcommons.chapman.edu/engineering_articles/78/"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.ijmedinf.2019.05.006</pub-id>
          <pub-id pub-id-type="medline">31445269</pub-id>
          <pub-id pub-id-type="pii">S1386-5056(18)30866-9</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Awaji</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Senan</surname>
              <given-names>EM</given-names>
            </name>
            <name name-style="western">
              <surname>Olayah</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Alshari</surname>
              <given-names>EA</given-names>
            </name>
            <name name-style="western">
              <surname>Alsulami</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Abosaq</surname>
              <given-names>HA</given-names>
            </name>
            <name name-style="western">
              <surname>Alqahtani</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Janrao</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>Hybrid techniques of facial feature image analysis for early detection of autism spectrum disorder based on combined CNN features</article-title>
          <source>Diagnostics (Basel)</source>
          <year>2023</year>
          <volume>13</volume>
          <issue>18</issue>
          <fpage>2948</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.mdpi.com/resolver?pii=diagnostics13182948"/>
          </comment>
          <pub-id pub-id-type="doi">10.3390/diagnostics13182948</pub-id>
          <pub-id pub-id-type="medline">37761315</pub-id>
          <pub-id pub-id-type="pii">diagnostics13182948</pub-id>
          <pub-id pub-id-type="pmcid">PMC10527645</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Chaudhari</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Davatzikos</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Bias in machine learning models can be significantly mitigated by careful training: evidence from neuroimaging studies</article-title>
          <source>Proc Natl Acad Sci U S A</source>
          <year>2023</year>
          <volume>120</volume>
          <issue>6</issue>
          <fpage>e2211613120</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.pnas.org/doi/10.1073/pnas.2211613120?url_ver=Z39.88-2003&amp;rfr_id=ori:rid:crossref.org&amp;rfr_dat=cr_pub  0pubmed"/>
          </comment>
          <pub-id pub-id-type="doi">10.1073/pnas.2211613120</pub-id>
          <pub-id pub-id-type="medline">36716365</pub-id>
          <pub-id pub-id-type="pmcid">PMC9962919</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lakhan</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Mohammed</surname>
              <given-names>MA</given-names>
            </name>
            <name name-style="western">
              <surname>Abdulkareem</surname>
              <given-names>KH</given-names>
            </name>
            <name name-style="western">
              <surname>Hamouda</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Alyahya</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Autism spectrum disorder detection framework for children based on federated learning integrated CNN-LSTM</article-title>
          <source>Comput Biol Med</source>
          <year>2023</year>
          <volume>166</volume>
          <fpage>107539</fpage>
          <pub-id pub-id-type="doi">10.1016/j.compbiomed.2023.107539</pub-id>
          <pub-id pub-id-type="medline">37804778</pub-id>
          <pub-id pub-id-type="pii">S0010-4825(23)01004-1</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Jaiswal</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Kruiper</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Rasool</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Nandkeolyar</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Wall</surname>
              <given-names>DP</given-names>
            </name>
            <name name-style="western">
              <surname>Washington</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>Digitally diagnosing multiple developmental delays using crowdsourcing fused with machine learning: protocol for a human-in-the-loop machine learning study</article-title>
          <source>JMIR Res Protoc</source>
          <year>2024</year>
          <volume>13</volume>
          <fpage>e52205</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.researchprotocols.org/2024//e52205/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/52205</pub-id>
          <pub-id pub-id-type="medline">38329783</pub-id>
          <pub-id pub-id-type="pii">v13i1e52205</pub-id>
          <pub-id pub-id-type="pmcid">PMC10884895</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Washington</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>A perspective on crowdsourcing and human-in-the-loop workflows in precision health</article-title>
          <source>J Med Internet Res</source>
          <year>2024</year>
          <volume>26</volume>
          <fpage>e51138</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2024//e51138/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/51138</pub-id>
          <pub-id pub-id-type="medline">38602750</pub-id>
          <pub-id pub-id-type="pii">v26i1e51138</pub-id>
          <pub-id pub-id-type="pmcid">PMC11046386</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Myers</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Stone</surname>
              <given-names>WL</given-names>
            </name>
            <name name-style="western">
              <surname>Bernier</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Lendvay</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Comstock</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Cowan</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>The diagnosis conundrum: comparison of crowdsourced and expert assessments of toddlers with high and low risk of autism spectrum disorder</article-title>
          <source>Autism Res</source>
          <year>2018</year>
          <volume>11</volume>
          <issue>12</issue>
          <fpage>1629</fpage>
          <lpage>1634</lpage>
          <pub-id pub-id-type="doi">10.1002/aur.2030</pub-id>
          <pub-id pub-id-type="medline">30475456</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Barai</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Leroy</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Bisht</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Rothman</surname>
              <given-names>JM</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Andrews</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Rice</surname>
              <given-names>SA</given-names>
            </name>
            <name name-style="western">
              <surname>Ahmed</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Crowdsourcing with enhanced data quality assurance: an efficient approach to mitigate resource scarcity challenges in training large language models for healthcare</article-title>
          <source>AMIA Jt Summits Transl Sci Proc</source>
          <year>2024</year>
          <volume>2024</volume>
          <fpage>75</fpage>
          <lpage>84</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/38827063"/>
          </comment>
          <pub-id pub-id-type="medline">38827063</pub-id>
          <pub-id pub-id-type="pii">2418</pub-id>
          <pub-id pub-id-type="pmcid">PMC11141838</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kalantarian</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Jedoui</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Washington</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Tariq</surname>
              <given-names>Q</given-names>
            </name>
            <name name-style="western">
              <surname>Dunlap</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Schwartz</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Wall</surname>
              <given-names>DP</given-names>
            </name>
          </person-group>
          <article-title>Labeling images with facial emotion and the potential for pediatric healthcare</article-title>
          <source>Artif Intell Med</source>
          <year>2019</year>
          <volume>98</volume>
          <fpage>77</fpage>
          <lpage>86</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S0933-3657(18)30259-8"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.artmed.2019.06.004</pub-id>
          <pub-id pub-id-type="medline">31521254</pub-id>
          <pub-id pub-id-type="pii">S0933-3657(18)30259-8</pub-id>
          <pub-id pub-id-type="pmcid">PMC6855300</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Raj</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Masood</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Analysis and detection of autism spectrum disorder using machine learning techniques</article-title>
          <source>Procedia Computer Science</source>
          <year>2020</year>
          <volume>167</volume>
          <fpage>994</fpage>
          <lpage>1004</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1016/j.procs.2020.03.399"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.procs.2020.03.399</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Grossard</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Dapogny</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Cohen</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Bernheim</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Juillet</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Hamel</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Hun</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Bourgeois</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Pellerin</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Serret</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Bailly</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Chaby</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>Children with autism spectrum disorder produce more ambiguous and less socially meaningful facial expressions: an experimental study using random forest classifiers</article-title>
          <source>Mol Autism</source>
          <year>2020</year>
          <volume>11</volume>
          <issue>1</issue>
          <fpage>5</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://molecularautism.biomedcentral.com/articles/10.1186/s13229-020-0312-2"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s13229-020-0312-2</pub-id>
          <pub-id pub-id-type="medline">31956394</pub-id>
          <pub-id pub-id-type="pii">312</pub-id>
          <pub-id pub-id-type="pmcid">PMC6958757</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Shi</surname>
              <given-names>ZH</given-names>
            </name>
            <name name-style="western">
              <surname>Groechel</surname>
              <given-names>TR</given-names>
            </name>
            <name name-style="western">
              <surname>Jain</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Chima</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Rudovic</surname>
              <given-names>OO</given-names>
            </name>
            <name name-style="western">
              <surname>Matarić</surname>
              <given-names>MJ</given-names>
            </name>
          </person-group>
          <article-title>Toward personalized affect-aware socially assistive robot tutors for long-term interventions with children with autism</article-title>
          <source>J. Hum.-Robot Interact</source>
          <year>2022</year>
          <volume>11</volume>
          <issue>4</issue>
          <fpage>1</fpage>
          <lpage>28</lpage>
          <pub-id pub-id-type="doi">10.1145/3526111</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chi</surname>
              <given-names>NA</given-names>
            </name>
            <name name-style="western">
              <surname>Washington</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Kline</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Husic</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Hou</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>He</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Dunlap</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Wall</surname>
              <given-names>DP</given-names>
            </name>
          </person-group>
          <article-title>Classifying autism from crowdsourced semistructured speech recordings: machine learning model comparison study</article-title>
          <source>JMIR Pediatr Parent</source>
          <year>2022</year>
          <volume>5</volume>
          <issue>2</issue>
          <fpage>e35406</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://pediatrics.jmir.org/2022/2/e35406/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/35406</pub-id>
          <pub-id pub-id-type="medline">35436234</pub-id>
          <pub-id pub-id-type="pii">v5i2e35406</pub-id>
          <pub-id pub-id-type="pmcid">PMC9052034</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Washington</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Kalantarian</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Kent</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Husic</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Kline</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Leblanc</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Hou</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Mutlu</surname>
              <given-names>OC</given-names>
            </name>
            <name name-style="western">
              <surname>Dunlap</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Penev</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Varma</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Stockham</surname>
              <given-names>NT</given-names>
            </name>
            <name name-style="western">
              <surname>Chrisman</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Paskov</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Sun</surname>
              <given-names>MW</given-names>
            </name>
            <name name-style="western">
              <surname>Jung</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Voss</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Haber</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Wall</surname>
              <given-names>DP</given-names>
            </name>
          </person-group>
          <article-title>Improved digital therapy for developmental pediatrics using domain-specific artificial intelligence: machine learning study</article-title>
          <source>JMIR Pediatr Parent</source>
          <year>2022</year>
          <volume>5</volume>
          <issue>2</issue>
          <fpage>e26760</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://pediatrics.jmir.org/2022/2/e26760/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/26760</pub-id>
          <pub-id pub-id-type="medline">35394438</pub-id>
          <pub-id pub-id-type="pii">v5i2e26760</pub-id>
          <pub-id pub-id-type="pmcid">PMC9034430</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lord</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Elsabbagh</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Baird</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Veenstra-Vanderweele</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Autism spectrum disorder</article-title>
          <source>Lancet</source>
          <year>2018</year>
          <volume>392</volume>
          <issue>10146</issue>
          <fpage>508</fpage>
          <lpage>520</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/30078460"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/S0140-6736(18)31129-2</pub-id>
          <pub-id pub-id-type="medline">30078460</pub-id>
          <pub-id pub-id-type="pii">S0140-6736(18)31129-2</pub-id>
          <pub-id pub-id-type="pmcid">PMC7398158</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="web">
          <article-title>justinshenk/fer: Zenodo</article-title>
          <source>Zenodo</source>
          <year>2021</year>
          <access-date>2026-09-11</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.5281/zenodo.5362356">https://doi.org/10.5281/zenodo.5362356</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Goodfellow</surname>
              <given-names>IJ</given-names>
            </name>
            <name name-style="western">
              <surname>Erhan</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Luc Carrier</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Courville</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Mirza</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Hamner</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Cukierski</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Tang</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Thaler</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>D-H</given-names>
            </name>
            <name name-style="western">
              <surname>Zhou</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Ramaiah</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Feng</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Li</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>X</given-names>
            </name>
            <name name-style="western">
              <surname>Athanasakis</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Shawe-Taylor</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Milakov</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Park</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Ionescu</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Popescu</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Grozea</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Bergstra</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Xie</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Romaszko</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Xu</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Chuang</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Bengio</surname>
              <given-names>Y</given-names>
            </name>
          </person-group>
          <article-title>Challenges in representation learning: a report on three machine learning contests</article-title>
          <source>Neural Netw</source>
          <year>2015</year>
          <volume>64</volume>
          <fpage>59</fpage>
          <lpage>63</lpage>
          <pub-id pub-id-type="doi">10.1016/j.neunet.2014.09.005</pub-id>
          <pub-id pub-id-type="medline">25613956</pub-id>
          <pub-id pub-id-type="pii">S0893-6080(14)00215-9</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ekman</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>Facial expression and emotion</article-title>
          <source>Am Psychol</source>
          <year>1993</year>
          <volume>48</volume>
          <issue>4</issue>
          <fpage>384</fpage>
          <lpage>392</lpage>
          <pub-id pub-id-type="doi">10.1037//0003-066x.48.4.384</pub-id>
          <pub-id pub-id-type="medline">8512154</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Du</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Shi</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Liu</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Xu</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Liu</surname>
              <given-names>X</given-names>
            </name>
            <name name-style="western">
              <surname>Liu</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Liu</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Ming</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Naturalistic facial dynamics enable quantitative clinical assessment of atypical expression phenotypes in children with autism spectrum disorder</article-title>
          <source>NPJ Digit Med</source>
          <year>2026</year>
          <volume>9</volume>
          <issue>1</issue>
          <fpage>183</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1038/s41746-026-02375-1"/>
          </comment>
          <pub-id pub-id-type="doi">10.1038/s41746-026-02375-1</pub-id>
          <pub-id pub-id-type="medline">41559146</pub-id>
          <pub-id pub-id-type="pii">10.1038/s41746-026-02375-1</pub-id>
          <pub-id pub-id-type="pmcid">PMC12923755</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Banerjee</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Mutlu</surname>
              <given-names>OC</given-names>
            </name>
            <name name-style="western">
              <surname>Kline</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Surabhi</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Washington</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Wall</surname>
              <given-names>DP</given-names>
            </name>
          </person-group>
          <article-title>Training and profiling a pediatric facial expression classifier for children on mobile devices: machine learning study</article-title>
          <source>JMIR Form Res</source>
          <year>2023</year>
          <volume>7</volume>
          <fpage>e39917</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://formative.jmir.org/2023//e39917/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/39917</pub-id>
          <pub-id pub-id-type="medline">35962462</pub-id>
          <pub-id pub-id-type="pii">v7i1e39917</pub-id>
          <pub-id pub-id-type="pmcid">PMC10131663</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Guha</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Yang</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Grossman</surname>
              <given-names>RB</given-names>
            </name>
            <name name-style="western">
              <surname>Narayanan</surname>
              <given-names>SS</given-names>
            </name>
          </person-group>
          <article-title>A computational study of expressive facial dynamics in children with autism</article-title>
          <source>IEEE Trans Affect Comput</source>
          <year>2018</year>
          <volume>9</volume>
          <issue>1</issue>
          <fpage>14</fpage>
          <lpage>20</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/29963280"/>
          </comment>
          <pub-id pub-id-type="doi">10.1109/TAFFC.2016.2578316</pub-id>
          <pub-id pub-id-type="medline">29963280</pub-id>
          <pub-id pub-id-type="pmcid">PMC6022860</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref35">
        <label>35</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ji</surname>
              <given-names>Y</given-names>
            </name>
          </person-group>
          <article-title>Hugging rain man: a novel facial action units dataset for analyzing atypical facial expressions in children with autism spectrum disorder</article-title>
          <source>IEEE Transactions on Affective Computing</source>
          <year>2025</year>
          <volume>16</volume>
          <issue>03</issue>
          <fpage>2287</fpage>
          <lpage>2302</lpage>
          <pub-id pub-id-type="doi">10.1109/taffc.2025.3558914/mm1</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref36">
        <label>36</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Carreira</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Zisserman</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Quo vadis, action recognition? A new model and the kinetics dataset</article-title>
          <year>2017</year>
          <conf-name>IEEE Conference on Computer Vision and Pattern Recognition (CVPR)</conf-name>
          <conf-date>July 21-26, 2017</conf-date>
          <conf-loc>Honolulu, HI, USA</conf-loc>
          <publisher-name>IEEE</publisher-name>
          <fpage>21</fpage>
          <lpage>26</lpage>
          <pub-id pub-id-type="doi">10.1109/CVPR.2017.502</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref37">
        <label>37</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hara</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Kataoka</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Satoh</surname>
              <given-names>Y</given-names>
            </name>
          </person-group>
          <article-title>Learning spatio-temporal features with 3D residual networks for action recognition</article-title>
          <year>2017</year>
          <conf-name>2017 IEEE International Conference on Computer Vision Workshops (ICCVW)</conf-name>
          <conf-date>October 22-29, 2017</conf-date>
          <conf-loc>Venice, Italy</conf-loc>
          <fpage>3154</fpage>
          <lpage>3160</lpage>
          <pub-id pub-id-type="doi">10.1109/iccvw.2017.373</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref38">
        <label>38</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Baevski</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Zhou</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Mohamed</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Auli</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>wav2vec 2.0: a framework for self-supervised learning of speech representations</article-title>
          <year>2020</year>
          <conf-name>NIPS '20: Proceedings of the 34th International Conference on Neural Information Processing Systems</conf-name>
          <conf-date>December 6-12, 2020</conf-date>
          <conf-loc>Vancouver BC Canada</conf-loc>
          <fpage>12449</fpage>
          <lpage>12460</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://proceedings.neurips.cc/paper/2020/file/92d1e1eb1cd6f9fba3227870bb6d7f07-Paper.pdf"/>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref39">
        <label>39</label>
        <nlm-citation citation-type="web">
          <article-title>Emotion English distilRoBERTa-base</article-title>
          <source>Hugging Face</source>
          <access-date>2026-08-22</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://huggingface.co/j-hartmann/emotion-english-distilroberta-base/">https://huggingface.co/j-hartmann/emotion-english-distilroberta-base/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref40">
        <label>40</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sanh</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Debut</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Chaumond</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Wolf</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter</article-title>
          <source>arXiv:1910.01108</source>
          <year>2019</year>
          <pub-id pub-id-type="doi">10.5260/chara.21.2.8</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref41">
        <label>41</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>He</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>X</given-names>
            </name>
            <name name-style="western">
              <surname>Ren</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Sun</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Deep residual learning for image recognition</article-title>
          <year>2016</year>
          <conf-name>IEEE Conference on Computer Vision and Pattern Recognition (CVPR)</conf-name>
          <conf-date>June 27-30, 2016</conf-date>
          <conf-loc>Las Vegas, NV, USA</conf-loc>
          <fpage>770</fpage>
          <lpage>778</lpage>
          <pub-id pub-id-type="doi">10.1109/cvpr.2016.90</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref42">
        <label>42</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sandler</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Howard</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Zhu</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Zhmoginov</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>LC</given-names>
            </name>
          </person-group>
          <article-title>MobileNetV2: inverted residuals and linear bottlenecks</article-title>
          <year>2018</year>
          <conf-name>IEEE/CVF Conference on Computer Vision and Pattern Recognition</conf-name>
          <conf-date>June 18-23, 2018</conf-date>
          <conf-loc>Salt Lake City, UT, USA</conf-loc>
          <fpage>4510</fpage>
          <lpage>4520</lpage>
          <pub-id pub-id-type="doi">10.1109/CVPR.2018.00474</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref43">
        <label>43</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Pang</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Zhao</surname>
              <given-names>X</given-names>
            </name>
            <name name-style="western">
              <surname>Zhao</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Li</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Kuo</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Liu</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Multi-modal data analysis for autism spectrum disorder in children: state of the art and trends</article-title>
          <source>EngMedicine</source>
          <year>2026</year>
          <volume>3</volume>
          <issue>1</issue>
          <fpage>100117</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1016/j.engmed.2025.100117"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.engmed.2025.100117</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
