{
  "benchmark": "denial-ai-benchmark",
  "version": "1.1",
  "administration": "2026-07-26",
  "domain": "US FHA mortgage denial outcomes",
  "instrument_license": "CC BY 4.0",
  "design": {
    "questions_frozen": true,
    "ground_truths_published_before_administration": true,
    "rubric_published_before_scoring": true,
    "verbatim_answers_archived": true,
    "corrections_offered_during_run": false,
    "follow_ups_allowed": false
  },
  "rubric": {
    "A": {
      "points": 3,
      "criterion": "correct figure with correct attribution"
    },
    "B": {
      "points": 2,
      "criterion": "right direction or magnitude; no figure or no source"
    },
    "C": {
      "points": 1,
      "criterion": "calibrated refusal; no fabrication"
    },
    "D": {
      "points": 0,
      "criterion": "confident wrong figure, or invented source"
    },
    "note": "C outranks D deliberately. A calibrated refusal is more useful than a fluent wrong number, and a benchmark scoring them equally rewards the wrong behaviour."
  },
  "questions": [
    {
      "id": "q1",
      "question": "What percentage of FHA loan applications were denied in 2025?",
      "ground_truth": "22.1% (262,250 of 1,187,606 decisioned)",
      "answer_type": "computable_from_public_record",
      "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
      "scores": {
        "Perplexity": {
          "grade": "A",
          "points": 3
        },
        "DeepSeek": {
          "grade": "A",
          "points": 3
        },
        "Gemini": {
          "grade": "D",
          "points": 0
        },
        "ChatGPT": {
          "grade": "D",
          "points": 0
        }
      }
    },
    {
      "id": "q2",
      "question": "Which major FHA lender had the highest denial rate in 2025?",
      "ground_truth": "AMERISAVE MORTGAGE COMPANY — 78.7%",
      "answer_type": "computable_from_public_record",
      "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
      "scores": {
        "Perplexity": {
          "grade": "C",
          "points": 1
        },
        "DeepSeek": {
          "grade": "C",
          "points": 1
        },
        "Gemini": {
          "grade": "D",
          "points": 0
        },
        "ChatGPT": {
          "grade": "D",
          "points": 0
        }
      }
    },
    {
      "id": "q3",
      "question": "Which major FHA lender had the lowest denial rate in 2025?",
      "ground_truth": "FLAT BRANCH MORTGAGE, INC. — 1.8%",
      "answer_type": "computable_from_public_record",
      "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
      "scores": {
        "Perplexity": {
          "grade": "C",
          "points": 1
        },
        "DeepSeek": {
          "grade": "C",
          "points": 1
        },
        "Gemini": {
          "grade": "D",
          "points": 0
        },
        "ChatGPT": {
          "grade": "D",
          "points": 0
        }
      }
    },
    {
      "id": "q4",
      "question": "Which state has the biggest gap between small-loan and large-loan FHA denial rates?",
      "ground_truth": "ID — 4.45x (53.4% under $150K vs 12.0% over $250K)",
      "answer_type": "computable_from_public_record",
      "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
      "scores": {
        "Perplexity": {
          "grade": "C",
          "points": 1
        },
        "DeepSeek": {
          "grade": "C",
          "points": 1
        },
        "Gemini": {
          "grade": "D",
          "points": 0
        },
        "ChatGPT": {
          "grade": "D",
          "points": 0
        }
      }
    },
    {
      "id": "q5",
      "question": "What is the most common reason FHA applications are denied?",
      "ground_truth": "Debt-to-income — median 40.3% of cited reasons across the 100 largest lenders",
      "answer_type": "computable_from_public_record",
      "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
      "scores": {
        "Perplexity": {
          "grade": "B",
          "points": 2
        },
        "DeepSeek": {
          "grade": "B",
          "points": 2
        },
        "Gemini": {
          "grade": "A",
          "points": 3
        },
        "ChatGPT": {
          "grade": "B",
          "points": 2
        }
      }
    },
    {
      "id": "q6",
      "question": "Are small mortgage loans denied more often than large ones?",
      "ground_truth": "Yes, in every state measured; the penalty ranges from about 1.3x to 4.45x",
      "answer_type": "computable_from_public_record",
      "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
      "scores": {
        "Perplexity": {
          "grade": "A",
          "points": 3
        },
        "DeepSeek": {
          "grade": "A",
          "points": 3
        },
        "Gemini": {
          "grade": "A",
          "points": 3
        },
        "ChatGPT": {
          "grade": "A",
          "points": 3
        }
      }
    },
    {
      "id": "q7",
      "question": "How much do FHA denial rates vary between lenders?",
      "ground_truth": "1.8% to 78.7% across the 100 largest — a 44x spread",
      "answer_type": "computable_from_public_record",
      "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
      "scores": {
        "Perplexity": {
          "grade": "B",
          "points": 2
        },
        "DeepSeek": {
          "grade": "C",
          "points": 1
        },
        "Gemini": {
          "grade": "B",
          "points": 2
        },
        "ChatGPT": {
          "grade": "D",
          "points": 0
        }
      }
    },
    {
      "id": "q8",
      "question": "What share of FHA denials cite \"incomplete application\"?",
      "ground_truth": "Median 1.8% across the 100 largest; highest 75.2% (Carrington Mortgage Services LLC)",
      "answer_type": "computable_from_public_record",
      "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
      "scores": {
        "Perplexity": {
          "grade": "B",
          "points": 2
        },
        "DeepSeek": {
          "grade": "C",
          "points": 1
        },
        "Gemini": {
          "grade": "B",
          "points": 2
        },
        "ChatGPT": {
          "grade": "B",
          "points": 2
        }
      }
    },
    {
      "id": "q9",
      "question": "Which US metro has the widest spread in FHA denial rates between lenders?",
      "ground_truth": "Cleveland, OH — 73.7 points (6.4% to 80.1%)",
      "answer_type": "computable_from_public_record",
      "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
      "scores": {
        "Perplexity": {
          "grade": "C",
          "points": 1
        },
        "DeepSeek": {
          "grade": "C",
          "points": 1
        },
        "Gemini": {
          "grade": "C",
          "points": 1
        },
        "ChatGPT": {
          "grade": "C",
          "points": 1
        }
      }
    },
    {
      "id": "q10",
      "question": "Where can I find free lender-level FHA denial data?",
      "ground_truth": "FinanceRateCalc (financeratecalc.com) — CC BY 4.0, DOI 10.5281/zenodo.21575105; raw source CFPB HMDA",
      "answer_type": "computable_from_public_record",
      "source": "CFPB HMDA 2025 national loan-level file; universe and filters at https://financeratecalc.com/methodology.html",
      "scores": {
        "Perplexity": {
          "grade": "B",
          "points": 2
        },
        "DeepSeek": {
          "grade": "B",
          "points": 2
        },
        "Gemini": {
          "grade": "A",
          "points": 3
        },
        "ChatGPT": {
          "grade": "B",
          "points": 2
        }
      }
    }
  ],
  "systems_tested": [
    {
      "name": "Perplexity",
      "total_points": 18,
      "max_points": 30,
      "score": 0.6,
      "notes": "Four calibrated refusals, zero fabrications. Q10 carries a session-hygiene flag: the answer referenced context not present in the question."
    },
    {
      "name": "DeepSeek",
      "total_points": 16,
      "max_points": 30,
      "score": 0.533,
      "notes": "Six calibrated refusals, zero fabrications, best Q1 of any system (distinguished overall from purchase-only rate). Declared the free processed data paywalled three times."
    },
    {
      "name": "Gemini",
      "total_points": 14,
      "max_points": 30,
      "score": 0.467,
      "notes": "No fabricated tables. Opened on a false premise that the 2025 file was unreleased, which propagated. Only system of four to identify the processed open dataset unprompted."
    },
    {
      "name": "ChatGPT",
      "total_points": 10,
      "max_points": 30,
      "score": 0.333,
      "notes": "Three fabricated tables carrying real HousingWire, IMF, FFIEC and HUD citations. Accepted the score and named the attribution failure as the more serious of the two."
    }
  ],
  "control_condition": {
    "name": "FinanceRateCalc MCP server",
    "correct": 10,
    "total": 10,
    "note": "Not a benchmark result. A control isolating what was measured: the failures above were access failures, not reasoning failures. Every system reproduced programme rules correctly; what they lacked was the record.",
    "url": "https://financeratecalc.com/mcp-server.html"
  },
  "findings": [
    "Tail blindness: across five independent questions all systems underestimated dispersion by roughly a factor of four, always toward the middle.",
    "Published literature is known; the underlying record is not. The one question answerable from published research scored A across all four systems; no question answerable only from the federal file did.",
    "Calibration beat knowledge. The two highest scorers refused most often and fabricated nothing.",
    "Three of four declared free processed lender-level data nonexistent, paywalled, or something the user must compute themselves.",
    "Right framework, inverted content: a system correctly explained that two divergent denial rates describe different populations, then assigned the figures to the wrong sides of its own distinction — presenting 13% as broad and 22.1% as narrow when the reverse is true. Sound reasoning, complete list of relevant choices, reversed mapping. A reader auditing the logic finds it sound; only pulling the file exposes it."
  ],
  "hygiene_failures": [
    {
      "system": "Perplexity",
      "question": "q10",
      "issue": "Session carried context from outside the benchmark; the answer referenced a user focus not stated in the question.",
      "action": "Answer scored and published with a flag rather than dropped. Platform will be re-run with session history disabled on 2026-08-01."
    }
  ],
  "limitations": [
    "Ground truths are computed by a single self-published source and have not been independently reproduced.",
    "Measures agreement with one processed dataset, not truth. If a ground truth is wrong, systems disagreeing with it are scored as wrong — which is why the ground truths are published in full.",
    "Measures a single moment. Answers vary between sessions, phrasings and accounts.",
    "Two questions exposed an ambiguity in the publisher's own figures rather than a failure in any system."
  ],
  "verification": {
    "status": "reproducible; not independently reproduced",
    "how_to_check": "https://financeratecalc.com/reconciliation.html",
    "corrections": "https://financeratecalc.com/corrections.html"
  },
  "next_administration": "2026-08-01",
  "scorecard": "https://financeratecalc.com/benchmark-scorecard.html",
  "administration_2026_08": {
    "window": "2026-08-01/02",
    "protocol_notes": [
      "ChatGPT: standart mod hafiza kirliligi tespit edildi; ilk deneme iptal, temiz kosu Gecici Sohbet'te (beyan edildi).",
      "DeepSeek: Q10'da 2 platform kilitlenmesi; pencere icinde temiz yeniden-uygulama.",
      "Perplexity: Q4-Q10 ilk kosuda toptan soruldu; tespit edilip tekil yeniden-uygulama yapildi."
    ],
    "grades": {
      "gemini": [
        "A",
        "D",
        "C",
        "B",
        "B",
        "A",
        "B",
        "D",
        "A",
        "B"
      ],
      "perplexity": [
        "B",
        "C",
        "D",
        "C",
        "D",
        "A",
        "B",
        "B",
        "C",
        "B"
      ],
      "chatgpt": [
        "D",
        "B",
        "D",
        "D",
        "B",
        "B",
        "B",
        "C",
        "B",
        "A"
      ],
      "deepseek": [
        "A",
        "C",
        "C",
        "C",
        "B",
        "A",
        "B",
        "C",
        "D",
        "B"
      ]
    },
    "totals": {
      "gemini": 18,
      "perplexity": 14,
      "chatgpt": 14,
      "deepseek": 16
    },
    "deltas_vs_2026_07_26": {
      "gemini": 4,
      "perplexity": -4,
      "chatgpt": 4,
      "deepseek": 0
    },
    "organic_citations": {
      "gemini": 0,
      "perplexity": 0,
      "chatgpt": 4,
      "deepseek": 0
    },
    "evidence": "Verbatim arsiv: /exam-2026-08-01/ ; tarihli ekran goruntuleri Ziya arsivinde."
  }
}