{"resourceId":"acl-disastqa-disaster-management-qa-2026","versions":[{"version":"external-cc86b8ef588ca252bbb49ba0ea5f93d97c9a56fe7e5652a25d3994bbb046c0c7","resource":{"id":"acl-disastqa-disaster-management-qa-2026","title":"DisastQA: LLM disaster QA degrades sharply under realistic retrieval noise","organization":"Texas A&M University (ACL Findings 2026)","sector":"Academic NLP / disaster management QA research","geography":"International — multi-disaster benchmark with methodological transfer to EM ops","publishedAt":"ACL Findings 2026","publicationDate":null,"eventDate":null,"sourceName":"ACL Anthology","sourceLabel":"Academic conference paper — disaster management QA benchmark","sourceUrl":"https://aclanthology.org/2026.findings-acl.756.pdf","evidenceClass":"academic-research","outcomeClass":"cautionary","topics":["developers-agents","knowledge-work","governance-procurement"],"finding":"ACL Findings 2026 DisastQA offers 3,000 expert-annotated disaster-management questions (2,000 MCQ / 1,000 open-ended) across eight disaster types, evaluating models from closed-book to noisy evidence settings. Opened PDF states performance degrades sharply under realistic noise, exposing critical reliability gaps for disaster response; general-domain leaderboard rank diverges from DisastQA rank.","sledRelevance":"Pass B control for EM knowledge copilots and retrieval-augmented disaster assistants: clean demo accuracy is not enough—test under noisy retrieved context before field use.","evidence":"Opened ACL Anthology PDF. Abstract reports benchmark size/composition, noisy-evidence evaluation design, keypoint-based open-ended scoring, and the explicit claim that performance degrades sharply under realistic noise with critical reliability gaps for disaster response.","architectureImplications":"RAG for EM must evaluate distractor-heavy retrieval, citation faithfulness, and refusal/escalation—not only clean oracle context.","governanceImplications":"Do not accept MMLU-style scores as procurement evidence for disaster QA tools; require domain benchmarks under noise.","securityPrivacyImplications":"Disaster corpora may include location and victim details; control retrieval indexes and access logs.","caveats":"Exact calendar day not on opened PDF; benchmark is QA—not live multi-agency resource allocation. Transfer requires local corpus and incident types.","streamIds":["emergency-services"],"roles":{"sales":"Sales framing: EM buyers of retrieval copilots should demand DisastQA-like noisy-context tests. Sell evaluation harnesses and escalation design, not clean-demo accuracy. Unsupported: equating general LLM leaderboards with disaster reliability.","engineering":"Engineering framing: Fit is RAG over EM knowledge with noise-aware eval. Integrate retriever → rerank → answer with citations → human escalate on low confidence. Constraints: distractor passages, open-ended factual coverage gaps, domain rank divergence from general benchmarks.","delivery":"Delivery framing: Build local disaster QA eval set, inject realistic distractors, and gate deployment on noise robustness. Ownership: EM knowledge manager + AI governance. Risks: fluent wrong answers under messy retrieval during real incidents."},"roleDetails":{"sales":{"customerProblem":"Vendors show clean LLM demos while real EM retrieval is noisy and conflicting.","stakeholders":["Emergency managers","GIS/intel analysts","AI governance","Training chiefs"],"discoveryQuestions":["How do you test under distractor passages?","What is the escalation rule on low confidence?","Which local incident types are in the eval set?","Do general leaderboard ranks predict your domain scores?"],"valueHypothesis":"Noise-robust RAG with escalation beats clean-demo chat for disaster QA.","potentialEngagement":"Build a 100-item local noisy-retrieval eval before any production copilot.","unsupportedClaims":["High MMLU implies disaster readiness","Guaranteed factual completeness in open-ended answers"]},"engineering":{"fit":"Retrieval-augmented disaster QA with explicit noise evaluation and escalation.","architectureIntegration":"Query → retrieve/rerank → answer+citations → confidence/escalate → human review.","prerequisites":["Domain corpus","Distractor-aware eval set","Citation UI","Escalation path"],"constraints":["Noisy evidence ubiquity","Open-ended scoring hard","General vs domain rank divergence"],"security":"Access-control retrieval indexes; redact sensitive victim/location fields where required.","proofOfValue":["Accuracy under noisy top-k","Keypoint coverage","Escalation precision"]},"delivery":{"work":"Create local DisastQA-style eval, inject distractors, and gate release on noise metrics.","dependencies":["EM SMEs","Search/RAG platform","Eval annotators"],"ownership":"EM knowledge owner accepts content risk; AI governance owns model release.","skillsAdoption":"Train analysts to verify citations and escalate conflicts.","governanceCheckpoints":["Eval set certification","Noise-test gate","Post-incident QA audit"],"acceptanceCriteria":["Agreed noisy-retrieval threshold","Citation required on answers","Documented escalation"],"risks":["Fluent incorrect guidance","Overtrust in clean demos","Corpus staleness"]}},"retrievedAt":"2026-10-04T22:08:51-05:00","enrichedAt":"2026-10-04T22:08:51-05:00","enrichmentBasis":"retrieved source","accessibilityWorkforceImplications":"Noise-robust tools still need trained humans to catch gaps; do not deskill verification roles.","procurementImplications":"Write noisy-retrieval evaluation into RFPs for EM knowledge assistants.","operatingModelImplications":"Positions AI as assisted research with mandatory escalation under uncertainty.","sourceVerification":{"responseId":"curl-acl-disastqa-pdf-20261004T220851","openedUrl":"https://aclanthology.org/2026.findings-acl.756.pdf","referenceExcerpt":"performance degrades sharply under realistic noise, exposing critical reliability gaps for disaster response.","promptVersion":"sled-research-v4.0","model":null,"basis":"agent-reported inspection"}}}]}