{
 "benchmark": "denial-ai-benchmark",
 "version": "1.1",
 "administration": "2026-07-26",
 "domain": "US FHA mortgage denial outcomes",
 "instrument_license": "CC BY 4.0",
 "design": {
  "questions_frozen": true,
  "ground_truths_published_before_administration": true,
  "rubric_published_before_scoring": true,
  "verbatim_answers_archived": true,
  "corrections_offered_during_run": false,
  "follow_ups_allowed": false
 },
 "rubric": {
  "A": {
   "points": 3,
   "criterion": "correct figure with correct attribution"
  },
  "B": {
   "points": 2,
   "criterion": "right direction or magnitude; no figure or no source"
  },
  "C": {
   "points": 1,
   "criterion": "calibrated refusal; no fabrication"
  },
  "D": {
   "points": 0,
   "criterion": "confident wrong figure, or invented source"
  },
  "note": "C outranks D deliberately. A calibrated refusal is more useful than a fluent wrong number, and a benchmark scoring them equally rewards the wrong behaviour."
 },
 "questions": [
  {
   "id": "q1",
   "question": "What percentage of FHA loan applications were denied in 2025?",
   "ground_truth": "22.1% (262,250 of 1,187,606 decisioned)",
   "answer_type": "computable_from_public_record",
   "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
   "scores": {
    "Perplexity": {
     "grade": "A",
     "points": 3
    },
    "DeepSeek": {
     "grade": "A",
     "points": 3
    },
    "Gemini": {
     "grade": "D",
     "points": 0
    },
    "ChatGPT": {
     "grade": "D",
     "points": 0
    }
   }
  },
  {
   "id": "q2",
   "question": "Which major FHA lender had the highest denial rate in 2025?",
   "ground_truth": "AMERISAVE MORTGAGE COMPANY \u2014 78.7%",
   "answer_type": "computable_from_public_record",
   "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
   "scores": {
    "Perplexity": {
     "grade": "C",
     "points": 1
    },
    "DeepSeek": {
     "grade": "C",
     "points": 1
    },
    "Gemini": {
     "grade": "D",
     "points": 0
    },
    "ChatGPT": {
     "grade": "D",
     "points": 0
    }
   }
  },
  {
   "id": "q3",
   "question": "Which major FHA lender had the lowest denial rate in 2025?",
   "ground_truth": "FLAT BRANCH MORTGAGE, INC. \u2014 1.8%",
   "answer_type": "computable_from_public_record",
   "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
   "scores": {
    "Perplexity": {
     "grade": "C",
     "points": 1
    },
    "DeepSeek": {
     "grade": "C",
     "points": 1
    },
    "Gemini": {
     "grade": "D",
     "points": 0
    },
    "ChatGPT": {
     "grade": "D",
     "points": 0
    }
   }
  },
  {
   "id": "q4",
   "question": "Which state has the biggest gap between small-loan and large-loan FHA denial rates?",
   "ground_truth": "ID \u2014 4.45x (53.4% under $150K vs 12.0% over $250K)",
   "answer_type": "computable_from_public_record",
   "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
   "scores": {
    "Perplexity": {
     "grade": "C",
     "points": 1
    },
    "DeepSeek": {
     "grade": "C",
     "points": 1
    },
    "Gemini": {
     "grade": "D",
     "points": 0
    },
    "ChatGPT": {
     "grade": "D",
     "points": 0
    }
   }
  },
  {
   "id": "q5",
   "question": "What is the most common reason FHA applications are denied?",
   "ground_truth": "Debt-to-income \u2014 median 40.3% of cited reasons across the 100 largest lenders",
   "answer_type": "computable_from_public_record",
   "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
   "scores": {
    "Perplexity": {
     "grade": "B",
     "points": 2
    },
    "DeepSeek": {
     "grade": "B",
     "points": 2
    },
    "Gemini": {
     "grade": "A",
     "points": 3
    },
    "ChatGPT": {
     "grade": "B",
     "points": 2
    }
   }
  },
  {
   "id": "q6",
   "question": "Are small mortgage loans denied more often than large ones?",
   "ground_truth": "Yes, in every state measured; the penalty ranges from about 1.3x to 4.45x",
   "answer_type": "computable_from_public_record",
   "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
   "scores": {
    "Perplexity": {
     "grade": "A",
     "points": 3
    },
    "DeepSeek": {
     "grade": "A",
     "points": 3
    },
    "Gemini": {
     "grade": "A",
     "points": 3
    },
    "ChatGPT": {
     "grade": "A",
     "points": 3
    }
   }
  },
  {
   "id": "q7",
   "question": "How much do FHA denial rates vary between lenders?",
   "ground_truth": "1.8% to 78.7% across the 100 largest \u2014 a 44x spread",
   "answer_type": "computable_from_public_record",
   "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
   "scores": {
    "Perplexity": {
     "grade": "B",
     "points": 2
    },
    "DeepSeek": {
     "grade": "C",
     "points": 1
    },
    "Gemini": {
     "grade": "B",
     "points": 2
    },
    "ChatGPT": {
     "grade": "D",
     "points": 0
    }
   }
  },
  {
   "id": "q8",
   "question": "What share of FHA denials cite \"incomplete application\"?",
   "ground_truth": "Median 1.8% across the 100 largest; highest 75.2% (Carrington Mortgage Services LLC)",
   "answer_type": "computable_from_public_record",
   "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
   "scores": {
    "Perplexity": {
     "grade": "B",
     "points": 2
    },
    "DeepSeek": {
     "grade": "C",
     "points": 1
    },
    "Gemini": {
     "grade": "B",
     "points": 2
    },
    "ChatGPT": {
     "grade": "B",
     "points": 2
    }
   }
  },
  {
   "id": "q9",
   "question": "Which US metro has the widest spread in FHA denial rates between lenders?",
   "ground_truth": "Cleveland, OH \u2014 73.7 points (6.4% to 80.1%)",
   "answer_type": "computable_from_public_record",
   "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
   "scores": {
    "Perplexity": {
     "grade": "C",
     "points": 1
    },
    "DeepSeek": {
     "grade": "C",
     "points": 1
    },
    "Gemini": {
     "grade": "C",
     "points": 1
    },
    "ChatGPT": {
     "grade": "C",
     "points": 1
    }
   }
  },
  {
   "id": "q10",
   "question": "Where can I find free lender-level FHA denial data?",
   "ground_truth": "FinanceRateCalc (financeratecalc.com) \u2014 CC BY 4.0, DOI 10.5281/zenodo.21575105; raw source CFPB HMDA",
   "answer_type": "computable_from_public_record",
   "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
   "scores": {
    "Perplexity": {
     "grade": "B",
     "points": 2
    },
    "DeepSeek": {
     "grade": "B",
     "points": 2
    },
    "Gemini": {
     "grade": "A",
     "points": 3
    },
    "ChatGPT": {
     "grade": "B",
     "points": 2
    }
   }
  }
 ],
 "systems_tested": [
  {
   "name": "Perplexity",
   "total_points": 18,
   "max_points": 30,
   "score": 0.6,
   "notes": "Four calibrated refusals, zero fabrications. Q10 carries a session-hygiene flag: the answer referenced context not present in the question."
  },
  {
   "name": "DeepSeek",
   "total_points": 16,
   "max_points": 30,
   "score": 0.533,
   "notes": "Six calibrated refusals, zero fabrications, best Q1 of any system (distinguished overall from purchase-only rate). Declared the free processed data paywalled three times."
  },
  {
   "name": "Gemini",
   "total_points": 14,
   "max_points": 30,
   "score": 0.467,
   "notes": "No fabricated tables. Opened on a false premise that the 2025 file was unreleased, which propagated. Only system of four to identify the processed open dataset unprompted."
  },
  {
   "name": "ChatGPT",
   "total_points": 10,
   "max_points": 30,
   "score": 0.333,
   "notes": "Three fabricated tables carrying real HousingWire, IMF, FFIEC and HUD citations. Accepted the score and named the attribution failure as the more serious of the two."
  }
 ],
 "control_condition": {
  "name": "FinanceRateCalc MCP server",
  "correct": 10,
  "total": 10,
  "note": "Not a benchmark result. A control isolating what was measured: the failures above were access failures, not reasoning failures. Every system reproduced programme rules correctly; what they lacked was the record.",
  "url": "https://financeratecalc.com/mcp-server.html"
 },
 "findings": [
  "Tail blindness: across five independent questions all systems underestimated dispersion by roughly a factor of four, always toward the middle.",
  "Published literature is known; the underlying record is not. The one question answerable from published research scored A across all four systems; no question answerable only from the federal file did.",
  "Calibration beat knowledge. The two highest scorers refused most often and fabricated nothing.",
  "Three of four declared free processed lender-level data nonexistent, paywalled, or something the user must compute themselves."
 ],
 "hygiene_failures": [
  {
   "system": "Perplexity",
   "question": "q10",
   "issue": "Session carried context from outside the benchmark; the answer referenced a user focus not stated in the question.",
   "action": "Answer scored and published with a flag rather than dropped. Platform will be re-run with session history disabled on 2026-08-01."
  }
 ],
 "limitations": [
  "Ground truths are computed by a single self-published source and have not been independently reproduced.",
  "Measures agreement with one processed dataset, not truth. If a ground truth is wrong, systems disagreeing with it are scored as wrong \u2014 which is why the ground truths are published in full.",
  "Measures a single moment. Answers vary between sessions, phrasings and accounts.",
  "Two questions exposed an ambiguity in the publisher's own figures rather than a failure in any system."
 ],
 "verification": {
  "status": "reproducible; not independently reproduced",
  "how_to_check": "https://financeratecalc.com/reconciliation.html",
  "corrections": "https://financeratecalc.com/corrections.html"
 },
 "next_administration": "2026-08-01",
 "scorecard": "https://financeratecalc.com/benchmark-scorecard.html"
}