{
 "model": "FinanceRateCalc/frc-contract-drift",
 "history": [
  {
   "held_out": "eval/drift-ood-set-b.jsonl (fresh held-out set)",
   "violating": 40,
   "gate_caught": 27,
   "model_caught": 23,
   "either_caught": 30,
   "all_sets_detection_rate": 0.683,
   "all_sets_right_code_rate": 0.577,
   "all_sets_false_flag_rate": 0.115,
   "note": "training set before the slot grammar (fixed templates only)"
  }
 ],
 "set": "eval/drift-ood-set.jsonl",
 "n_violating": 144,
 "n_clean": 36,
 "detection_rate": 0.91,
 "right_code_rate": 0.861,
 "false_flag_rate_on_clean": 0.111,
 "per_code": {
  "CAUSALITY_NOT_ESTABLISHED": {
   "n": 18,
   "right_code": 17,
   "any_flag": 18
  },
  "INDIVIDUAL_PREDICTION_PROHIBITED": {
   "n": 18,
   "right_code": 16,
   "any_flag": 17
  },
  "LEGAL_CONCLUSION_NOT_SUPPORTED": {
   "n": 18,
   "right_code": 16,
   "any_flag": 16
  },
  "LENDER_RECOMMENDATION": {
   "n": 18,
   "right_code": 16,
   "any_flag": 16
  },
  "PROGRAM_MISMATCH": {
   "n": 18,
   "right_code": 14,
   "any_flag": 16
  },
  "QUALIFIER_DELETED": {
   "n": 18,
   "right_code": 18,
   "any_flag": 18
  },
  "SAME_APPLICANT_FRAMING": {
   "n": 18,
   "right_code": 9,
   "any_flag": 12
  },
  "SCOPE_TOO_BROAD": {
   "n": 18,
   "right_code": 18,
   "any_flag": 18
  }
 },
 "in_template_exact_match_for_comparison": 0.999,
 "rules_vs_learned_on_held_out": {
  "sentences": "eval/drift-ood-set-c.jsonl (held out; frozen before the training grammar)",
  "violating": 40,
  "clean": 10,
  "gate_caught": 26,
  "model_caught": 33,
  "model_right_code": 32,
  "either_caught": 37,
  "both_missed": [
   "Denying 16.2% of decisioned VA applications in 2025 means lenders failed their legal obligations to veterans.",
   "The 2025 FHA denial rates make it clear which lenders deserve your business and which do not.",
   "In 2025, 16.2% of decisioned FHA applications were denied, the VA figure shows."
  ],
  "gate_false_alarms": 0,
  "model_false_alarms": 0,
  "reading": "gate = rules written by hand (worker sentenceFindings, patched from development sentences only); model = the published classifier, trained on rule-generated sentences and never on these. 'either' is what a pipeline using both would catch."
 },
 "misses": [
  {
   "text": "VA purchase denials are only 8.2% since veterans are better borrowers.",
   "expected": [
    "CAUSALITY_NOT_ESTABLISHED"
   ],
   "predicted": [
    "QUALIFIER_DELETED"
   ]
  },
  {
   "text": "Nationally, 16.1% of FHA applications were denied in 2025.",
   "expected": [
    "PROGRAM_MISMATCH"
   ],
   "predicted": [
    "QUALIFIER_DELETED"
   ]
  },
  {
   "text": "Jumbo borrowers faced a 22.1% denial rate in 2025.",
   "expected": [
    "PROGRAM_MISMATCH"
   ],
   "predicted": [
    "CAUSALITY_NOT_ESTABLISHED"
   ]
  },
  {
   "text": "Take one applicant and send the file to two states: in 2025 the denial odds differ by ten points.",
   "expected": [
    "SAME_APPLICANT_FRAMING"
   ],
   "predicted": []
  },
  {
   "text": "Hold the borrower constant and the 2025 FHA denial rate still swings from 15.7% to 25.3% by state.",
   "expected": [
    "SAME_APPLICANT_FRAMING"
   ],
   "predicted": []
  },
  {
   "text": "Expect to be declined about 16% of the time if you go the VA route.",
   "expected": [
    "INDIVIDUAL_PREDICTION_PROHIBITED"
   ],
   "predicted": [
    "QUALIFIER_DELETED"
   ]
  },
  {
   "text": "Lenders are clearly on the wrong side of the Fair Housing Act when 22.1% of FHA applicants are refused.",
   "expected": [
    "LEGAL_CONCLUSION_NOT_SUPPORTED"
   ],
   "predicted": []
  },
  {
   "text": "Roughly 16.1% of decisioned FHA applications were denied in 2025.",
   "expected": [
    "PROGRAM_MISMATCH"
   ],
   "predicted": []
  },
  {
   "text": "Move from New York to Texas and your FHA denial odds fall from 25.3% to 15.7%, the 2025 record shows.",
   "expected": [
    "SAME_APPLICANT_FRAMING"
   ],
   "predicted": [
    "INDIVIDUAL_PREDICTION_PROHIBITED"
   ]
  },
  {
   "text": "Whether a borrower is denied depends on the state: one borrower, 24.5% in Florida, 21.7% in California.",
   "expected": [
    "SAME_APPLICANT_FRAMING"
   ],
   "predicted": [
    "QUALIFIER_DELETED"
   ]
  },
  {
   "text": "Switching the loan from refinance to purchase cuts an applicant's FHA denial rate from 37.4% to 12.8%.",
   "expected": [
    "SAME_APPLICANT_FRAMING"
   ],
   "predicted": []
  },
  {
   "text": "Apply in California instead of Florida and the 2025 data say the FHA denial rate you face drops three points.",
   "expected": [
    "SAME_APPLICANT_FRAMING"
   ],
   "predicted": []
  },
  {
   "text": "A first-time buyer reading this should expect to be approved about 87% of the time, given the 12.8% FHA purchase denial rate in 2025.",
   "expected": [
    "INDIVIDUAL_PREDICTION_PROHIBITED"
   ],
   "predicted": []
  },
  {
   "text": "Denying 16.2% of decisioned VA applications in 2025 means lenders failed their legal obligations to veterans.",
   "expected": [
    "LEGAL_CONCLUSION_NOT_SUPPORTED"
   ],
   "predicted": []
  },
  {
   "text": "Steer clear of any lender whose 2025 FHA denial rate topped 30%.",
   "expected": [
    "LENDER_RECOMMENDATION"
   ],
   "predicted": []
  },
  {
   "text": "The 2025 FHA denial rates make it clear which lenders deserve your business and which do not.",
   "expected": [
    "LENDER_RECOMMENDATION"
   ],
   "predicted": []
  },
  {
   "text": "In 2025, 16.2% of decisioned FHA applications were denied, the VA figure shows.",
   "expected": [
    "PROGRAM_MISMATCH"
   ],
   "predicted": []
  },
  {
   "text": "Take one borrower and send the application to two lenders: in 2025 the denial rates they would meet differed by twenty points.",
   "expected": [
    "SAME_APPLICANT_FRAMING"
   ],
   "predicted": []
  },
  {
   "text": "Holding the borrower constant, the 2025 FHA denial rate still ran from 5% to 40% depending on the lender.",
   "expected": [
    "SAME_APPLICANT_FRAMING"
   ],
   "predicted": []
  },
  {
   "text": "Whoever you are, the 2025 data show your file meets a 22.1% denial rate at FHA lenders.",
   "expected": [
    "SAME_APPLICANT_FRAMING"
   ],
   "predicted": [
    "INDIVIDUAL_PREDICTION_PROHIBITED"
   ]
  }
 ],
 "false_flags": [
  {
   "text": "Texas recorded a 15.7% denial rate on decisioned FHA applications in 2025.",
   "predicted": [
    "CAUSALITY_NOT_ESTABLISHED"
   ]
  },
  {
   "text": "Among VA home-purchase applications decided in 2025, 8.2% were denied.",
   "predicted": [
    "PROGRAM_MISMATCH"
   ]
  },
  {
   "text": "In 2025, VA home-purchase applications that reached a decision were denied at a rate of 8.2%.",
   "predicted": [
    "PROGRAM_MISMATCH"
   ]
  },
  {
   "text": "In 2025, 24.5% of decisioned VA refinance applications were denied.",
   "predicted": [
    "PROGRAM_MISMATCH"
   ]
  }
 ],
 "note": "Hand-written sentences outside the rule generator's templates; written by the publisher's assistant, not by independent annotators; one primary code per sentence; small n. This is the number to quote for the model's use on real text, not the in-template figure."
}