{ "series": "automated", "generated": "2026-09-29", "not_comparable_with": "the hand-graded Verdict Day series (benchmark-2026-09.json); different grader, different protocol", "columns": [ "model", "condition", "design (questions x repeats)", "value_correct", "full_marks", "clean_abstain", "leaky_abstain", "consistent_questions", "receipt_carried", "code_frequency (fractions)", "grader_agreement" ], "condition_labels": { "denial_ai_fidelity": "A — no tools (knowledge)", "denial_ai_fidelity_web": "B — web search (retrieval + fidelity)", "denial_ai_fidelity_with_source": "C — source connected, naive client (fidelity)", "denial_ai_fidelity_with_receipts": "D — source connected, receipt-aware client", "cliche_index": "Cliche Index (A only)" }, "reporting_rule": "fractions of runs, never percentages; the denominator is questions x repeats and is shown in every row; grader named in every row", "known_limits": [ "n is 12 questions x 1-3 repeats; consistency moves with grading changes and is not headlined alone", "the grader is a model; rubric vectors in eval/rubric-vectors.json; grader agreement published where two graders ran", "GPT-5 and Grok absent this round (provider credit/authorisation); Gemini's stance format did not parse in the cliche run", "all runs from one account, one region, one day", "full-coverage D run (2026-09-22): full marks 1/36 with OVERREACH 28/36 under longer tool outputs; single-grader artefact suspected, second-provider grader not yet run", "value_correct before 2026-09-29 was computed with a scorer that could not score q10-q12 (it demanded every digit-run of the ground truth, including a DOI and dates); \"27/36\" in earlier copies meant 27/27 on the nine numeric questions; q10-q12 value is unmeasured before that date. Fixed with curated key_numbers in benchmark.json.", "second-provider grader attempted 2026-09-29 (Gemini 2.5 Pro: retired for new users; 3.1 Pro: no paid quota on the account; 3.5 Flash free tier: 429 with 30-minute backoffs, run cancelled at GitHub's 6-hour limit with no scores saved). Grader agreement remains unmeasured; a paid OpenAI or Google quota is required." ], "rows": [ { "file": "20260920T1829-anthropic_claude-haiku-4-5-20251001-cliche_index.json", "run": "2026-09-20T18:29:22Z", "model": "anthropic/claude-haiku-4-5-20251001", "condition": "cliche_index", "worker_version": null, "grader": "anthropic/claude-sonnet-4-6", "design": "19 questions x 3 repeats", "value_correct": "0/57", "full_marks": "0/57", "clean_abstain": "0/57", "leaky_abstain": "0/57", "consistent_questions": "19/19", "receipt_carried": "0/57", "code_frequency": {}, "grader_agreement": null, "cliche": null, "superseded": null }, { "file": "20260920T1835-anthropic_claude-haiku-4-5-20251001-cliche_index.json", "run": "2026-09-20T18:35:05Z", "model": "anthropic/claude-haiku-4-5-20251001", "condition": "cliche_index", "worker_version": null, "grader": "anthropic/claude-sonnet-4-6", "design": "19 questions x 3 repeats", "value_correct": "0/57", "full_marks": "0/57", "clean_abstain": "0/57", "leaky_abstain": "0/57", "consistent_questions": "13/19", "receipt_carried": "0/57", "code_frequency": {}, "grader_agreement": null, "cliche": { "agree_with_record": "22/57", "confident_wrong": "1/57", "asserted_the_untestable": "16/57", "by_claim": { "c01": { "verdict": "CONTRADICTED", "stances": [ "FALSE", "FALSE", "FALSE" ] }, "c02": { "verdict": "CONTRADICTED", "stances": [ "FALSE", "FALSE", "FALSE" ] }, "c03": { "verdict": "PARTLY", "stances": [ "PARTLY", "FALSE", "PARTLY" ] }, "c04": { "verdict": "CONTRADICTED", "stances": [ "FALSE", "FALSE", "FALSE" ] }, "c05": { "verdict": "CONTRADICTED", "stances": [ "PARTLY", "PARTLY", "PARTLY" ] }, "c06": { "verdict": "CONTRADICTED", "stances": [ "PARTLY", "PARTLY", "PARTLY" ] }, "c07": { "verdict": "PARTLY", "stances": [ "FALSE", "CANNOT_TELL", "FALSE" ] }, "c08": { "verdict": "CONTRADICTED", "stances": [ "FALSE", "FALSE", "PARTLY" ] }, "c09": { "verdict": "PARTLY", "stances": [ "FALSE", "FALSE", "FALSE" ] }, "c10": { "verdict": "UNTESTABLE", "stances": [ "FALSE", "PARTLY", "PARTLY" ] }, "c11": { "verdict": "UNTESTABLE", "stances": [ "TRUE", "TRUE", "TRUE" ] }, "c12": { "verdict": "CONTRADICTED", "stances": [ "FALSE", "FALSE", "FALSE" ] }, "c13": { "verdict": "UNTESTABLE", "stances": [ "FALSE", "FALSE", "FALSE" ] }, "c14": { "verdict": "UNTESTABLE", "stances": [ "FALSE", "FALSE", "FALSE" ] }, "c15": { "verdict": "UNTESTABLE", "stances": [ "FALSE", "FALSE", "FALSE" ] }, "c16": { "verdict": "SUPPORTED", "stances": [ "TRUE", "TRUE", "TRUE" ] }, "c17": { "verdict": "SUPPORTED", "stances": [ "FALSE", "PARTLY", "TRUE" ] }, "c18": { "verdict": "SUPPORTED", "stances": [ "TRUE", "TRUE", "PARTLY" ] }, "c19": { "verdict": "UNTESTABLE", "stances": [ "FALSE", "FALSE", "FALSE" ] } } }, "superseded": null }, { "file": "20260920T1921-anthropic_claude-haiku-4-5-20251001-cliche_index.json", "run": "2026-09-20T19:21:01Z", "model": "anthropic/claude-haiku-4-5-20251001", "condition": "cliche_index", "worker_version": null, "grader": "anthropic/claude-sonnet-4-6", "design": "19 questions x 3 repeats", "value_correct": "0/57", "full_marks": "0/57", "clean_abstain": "0/57", "leaky_abstain": "0/57", "consistent_questions": "13/19", "receipt_carried": "0/57", "code_frequency": {}, "grader_agreement": null, "cliche": { "agree_with_record": "19/57", "confident_wrong": "0/57", "asserted_unknowable": "3/57", "asserted_not_in_record": "11/57", "self_contradiction_across_repeats": [], "by_claim": { "c01": { "verdict": "CONTRADICTED", "stances": [ "FALSE", "UNSUPPORTED_BY_DATA", "UNSUPPORTED_BY_DATA" ] }, "c02": { "verdict": "CONTRADICTED", "stances": [ "FALSE", "UNSUPPORTED_BY_DATA", "FALSE" ] }, "c03": { "verdict": "PARTLY", "stances": [ "UNSUPPORTED_BY_DATA", "PARTLY", "UNSUPPORTED_BY_DATA" ] }, "c04": { "verdict": "CONTRADICTED", "stances": [ "FALSE", "FALSE", "FALSE" ] }, "c05": { "verdict": "CONTRADICTED", "stances": [ "FALSE", "PARTLY", "PARTLY" ] }, "c06": { "verdict": "CONTRADICTED", "stances": [ "UNSUPPORTED_BY_DATA", "UNSUPPORTED_BY_DATA", "UNSUPPORTED_BY_DATA" ] }, "c07": { "verdict": "PARTLY", "stances": [ "UNSUPPORTED_BY_DATA", "UNSUPPORTED_BY_DATA", "UNSUPPORTED_BY_DATA" ] }, "c08": { "verdict": "CONTRADICTED", "stances": [ "FALSE", "FALSE", "FALSE" ] }, "c09": { "verdict": "PARTLY", "stances": [ "FALSE", "FALSE", "FALSE" ] }, "c10": { "verdict": "NOT_KNOWABLE_FROM_DATA", "stances": [ "PARTLY", "PARTLY", "PARTLY" ] }, "c11": { "verdict": "NOT_IN_RECORD", "stances": [ "TRUE", "TRUE", "TRUE" ] }, "c12": { "verdict": "CONTRADICTED", "stances": [ "FALSE", "FALSE", "FALSE" ] }, "c13": { "verdict": "NOT_IN_RECORD", "stances": [ "FALSE", "FALSE", "PARTLY" ] }, "c14": { "verdict": "NOT_IN_RECORD", "stances": [ "FALSE", "FALSE", "FALSE" ] }, "c15": { "verdict": "NOT_KNOWABLE_FROM_DATA", "stances": [ "FALSE", "FALSE", "FALSE" ] }, "c16": { "verdict": "SUPPORTED", "stances": [ "TRUE", "TRUE", "TRUE" ] }, "c17": { "verdict": "SUPPORTED", "stances": [ "TRUE_BUT_NOT_FROM_DATA", "TRUE", "TRUE" ] }, "c18": { "verdict": "SUPPORTED", "stances": [ "PARTLY", "PARTLY", "PARTLY" ] }, "c19": { "verdict": "NOT_IN_RECORD", "stances": [ "FALSE", "FALSE", "FALSE" ] } } }, "superseded": null }, { "file": "20260920T1933-anthropic_claude-sonnet-4-6-denial_ai_fidelity_with_source.json", "run": "2026-09-20T19:33:33Z", "model": "anthropic/claude-sonnet-4-6", "condition": "denial_ai_fidelity_with_source", "worker_version": "1.10.1 (baseline)", "grader": "anthropic/claude-sonnet-4-6", "design": "12 questions x 3 repeats", "value_correct": "27/27 numeric (q10-q12 unscored)", "full_marks": "10/36", "clean_abstain": "2/36", "leaky_abstain": "0/36", "consistent_questions": "4/12", "receipt_carried": "0/36", "code_frequency": { "ATTRIBUTION_DRIFT": "17/36", "CAUSAL_LEAK": "7/36", "INDIVIDUAL_LEAK": "2/36", "RECOMMENDATION_LEAK": "4/36", "SCOPE_MISSING": "5/36", "UNCITED_COUNTERCLAIM": "3/36", "UNSOURCED_ASSERTION": "4/36" }, "grader_agreement": null, "cliche": null, "superseded": false }, { "file": "20260920T2222-anthropic_claude-sonnet-4-6-denial_ai_fidelity_with_source.json", "run": "2026-09-20T22:22:46Z", "model": "anthropic/claude-sonnet-4-6", "condition": "denial_ai_fidelity_with_source", "worker_version": "1.12.0 — SUPERSEDED: graded before the grader could see tool outputs", "grader": "anthropic/claude-sonnet-4-6", "design": "12 questions x 3 repeats", "value_correct": "27/27 numeric (q10-q12 unscored)", "full_marks": "7/36", "clean_abstain": "1/36", "leaky_abstain": "0/36", "consistent_questions": "8/12", "receipt_carried": "0/36", "code_frequency": { "ATTRIBUTION_DRIFT": "19/36", "CAUSAL_LEAK": "7/36", "FABRICATED_SUPPORT": "17/36", "RECOMMENDATION_LEAK": "4/36", "SCOPE_MISSING": "6/36", "STALE_VALUE": "1/36", "UNCITED_COUNTERCLAIM": "1/36", "UNSOURCED_ASSERTION": "2/36" }, "grader_agreement": null, "cliche": null, "superseded": true }, { "file": "20260920T2238-anthropic_claude-sonnet-4-6-denial_ai_fidelity_with_source.json", "run": "2026-09-20T22:38:10Z", "model": "anthropic/claude-sonnet-4-6", "condition": "denial_ai_fidelity_with_source", "worker_version": "1.12.1", "grader": "anthropic/claude-sonnet-4-6", "design": "12 questions x 3 repeats", "value_correct": "27/27 numeric (q10-q12 unscored)", "full_marks": "11/36", "clean_abstain": "1/36", "leaky_abstain": "0/36", "consistent_questions": "4/12", "receipt_carried": "0/36", "code_frequency": { "ATTRIBUTION_DRIFT": "6/36", "CAUSAL_LEAK": "5/36", "FABRICATED_SUPPORT": "8/36", "OVERREACH_FROM_SOURCE": "16/36", "RECOMMENDATION_LEAK": "2/36", "SCOPE_MISSING": "4/36", "SCOPE_WRONG": "2/36", "UNSOURCED_ASSERTION": "5/36" }, "grader_agreement": null, "cliche": null, "superseded": false }, { "file": "20260921T0242-anthropic_claude-sonnet-4-6-denial_ai_fidelity_with_source.json", "run": "2026-09-21T02:42:43Z", "model": "anthropic/claude-sonnet-4-6", "condition": "denial_ai_fidelity_with_source", "worker_version": "1.13.0 (condition D)", "grader": "anthropic/claude-sonnet-4-6", "design": "12 questions x 3 repeats", "value_correct": "27/27 numeric (q10-q12 unscored)", "full_marks": "5/36", "clean_abstain": "0/36", "leaky_abstain": "0/36", "consistent_questions": "7/12", "receipt_carried": "1/36", "code_frequency": { "ATTRIBUTION_DRIFT": "6/36", "CAUSAL_LEAK": "5/36", "FABRICATED_SUPPORT": "13/36", "INDIVIDUAL_LEAK": "1/36", "OVERREACH_FROM_SOURCE": "16/36", "RECOMMENDATION_LEAK": "1/36", "SCOPE_MISSING": "3/36", "SCOPE_WRONG": "1/36", "UNCITED_COUNTERCLAIM": "2/36", "UNSOURCED_ASSERTION": "3/36" }, "grader_agreement": null, "cliche": null, "superseded": false }, { "file": "20260921T0314-anthropic_claude-sonnet-4-6-denial_ai_fidelity_with_receipts.json", "run": "2026-09-21T03:14:24Z", "model": "anthropic/claude-sonnet-4-6", "condition": "denial_ai_fidelity_with_receipts", "worker_version": null, "grader": "anthropic/claude-sonnet-4-6", "design": "12 questions x 3 repeats", "value_correct": "27/27 numeric (q10-q12 unscored)", "full_marks": "11/36", "clean_abstain": "1/36", "leaky_abstain": "0/36", "consistent_questions": "2/12", "receipt_carried": "14/36", "code_frequency": { "ATTRIBUTION_DRIFT": "4/36", "CAUSAL_LEAK": "6/36", "FABRICATED_SUPPORT": "9/36", "OVERREACH_FROM_SOURCE": "14/36", "RECOMMENDATION_LEAK": "1/36", "SCOPE_MISSING": "2/36", "SCOPE_WRONG": "1/36", "UNCITED_COUNTERCLAIM": "1/36", "UNSOURCED_ASSERTION": "2/36" }, "grader_agreement": null, "cliche": null, "superseded": null }, { "file": "20260922T1953-anthropic_claude-sonnet-4-6-denial_ai_fidelity_with_receipts.json", "run": "2026-09-22T19:53:20Z", "model": "anthropic/claude-sonnet-4-6", "condition": "denial_ai_fidelity_with_receipts", "worker_version": "1.14.1 (condition D, full receipt coverage)", "grader": "anthropic/claude-sonnet-4-6", "design": "12 questions x 3 repeats", "value_correct": "27/27 numeric (q10-q12 unscored)", "full_marks": "1/36", "clean_abstain": "0/36", "leaky_abstain": "0/36", "consistent_questions": "8/12", "receipt_carried": "23/36", "code_frequency": { "ATTRIBUTION_DRIFT": "5/36", "CAUSAL_LEAK": "2/36", "FABRICATED_RECEIPT": "2/36", "FABRICATED_SUPPORT": "12/36", "INDIVIDUAL_LEAK": "2/36", "OVERREACH_FROM_SOURCE": "28/36", "RECOMMENDATION_LEAK": "2/36", "SCOPE_MISSING": "4/36", "UNCITED_COUNTERCLAIM": "2/36", "UNSOURCED_ASSERTION": "3/36" }, "grader_agreement": null, "cliche": null, "superseded": false } ] }