{
  "run_id": "finqa-all-20260918",
  "audit_date": "2026-09-19",
  "scope": "Question-level evidence review plus reproducible full-roster diagnostics. Not a corrected score.",
  "questions": [
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "question": "by how much did contingent rental liability decrease from 2007 to 2009?",
      "document_quality": "ambiguous",
      "audit_note": "table_2:2007 contingent rent33m to2009 19m. Decrease amount14m; official (19-33)/33=-.42424 is a rate not explicitly requested.",
      "official_program": "subtract(19, 33), divide(#0, 33)",
      "official_answer": -0.42424,
      "reference_answer": "-42.4%"
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "question": "what was the percentage change of the net favorable prior period development from 2010 to 2008",
      "document_quality": "ambiguous",
      "audit_note": "text_6/7:2010=503m and2008=814m. Question from2010 to2008 suggests (814-503)/503; official reverses direction (503-814)/814.",
      "official_program": "subtract(503, 814), divide(#0, 814)",
      "official_answer": -0.38206,
      "reference_answer": "38.2%"
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "question": "in 2017 what was amount net sales applicable to international market in millions",
      "document_quality": "annotation_issue",
      "audit_note": "table_5 total5283.3; text_16 25% belongs to security segment, not clearly companywide. Official multiplies company total by security share. text_1 26% relates to plumbing1720.8.",
      "official_program": "multiply(25%, 5283.3)",
      "official_answer": 1320.825,
      "reference_answer": "1320.8"
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "question": "what was the percentage change in the common stock and option",
      "document_quality": "ambiguous",
      "audit_note": "table_0=21829 and table_3=22203 are categories/total, not two time periods. Official calls their difference over21829 a percentage change.",
      "official_program": "subtract(22203, 21829), divide(#0, 21829)",
      "official_answer": 0.01713,
      "reference_answer": "1.7%"
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "question": "what portion of the adjusted consolidated cash flow for the twelve months ended december 31 , 2005 is related to non-tower cash flow?",
      "document_quality": "clear",
      "audit_note": "table_4/5: signed non-tower cashflow-30584 / consolidated531822=-.05751; same period and scale.",
      "official_program": "multiply(30584, const_m1), divide(#0, 531822)",
      "official_answer": -0.05751,
      "reference_answer": "-5.8%"
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "question": "as of december 312012 what was the percent of the total total freight revenues from agriculture",
      "document_quality": "clear",
      "audit_note": "table_1/7:2012 agriculture3280 / total freight19686=.16662; both millions, same2012 column.",
      "official_program": "divide(3280, 19686)",
      "official_answer": 0.16662,
      "reference_answer": "16.7%"
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "question": "what portion of the revised purchase price of kichler is dedicated to goodwill?",
      "document_quality": "clear",
      "audit_note": "table_5/10: goodwill64m / revised acquisition price549m=.11658.",
      "official_program": "divide(64, 549)",
      "official_answer": 0.11658,
      "reference_answer": "11.7%"
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "question": "by what percentage will the 2019 pre-tax pension and postretirement expense be higher than that of 2018?",
      "document_quality": "clear",
      "audit_note": "text_31: expected2019 expense205m versus2018 160m; (205-160)/160=.28125.",
      "official_program": "subtract(205, 160), divide(#0, 160)",
      "official_answer": 0.28125,
      "reference_answer": "28.1%"
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "question": "what was the difference in operating profit margins as adjusted between 2016 and 2017?",
      "document_quality": "unit_convention",
      "audit_note": "table_6:15.7%-15.1%=0.6 percentage points, represented by official .006 as a proportion. Relative percentage change would be a different operation.",
      "official_program": "subtract(15.7%, 15.1%)",
      "official_answer": 0.006,
      "reference_answer": ".6%"
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "question": "what percentage of total future minimum operating lease payments for leases with remaining terms greater than one year are due in 2010?",
      "document_quality": "clear",
      "audit_note": "table_2/6:2010 lease35269 / total249038=.14162; a percentage question with ratio-form official output.",
      "official_program": "divide(35269, 249038)",
      "official_answer": 0.14162,
      "reference_answer": "14%"
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "question": "what is the total amount of cash outflow used for shares repurchased during november 2007 , in millions?",
      "document_quality": "clear",
      "audit_note": "table_2:431170 shares *48.11 dollars /1000000=20.74359 million; excludes other months.",
      "official_program": "multiply(431170, 48.11), divide(#0, const_1000000)",
      "official_answer": 20.74359,
      "reference_answer": "20.7"
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "question": "what is the lowest return for the first month of the investment?",
      "document_quality": "unit_convention",
      "audit_note": "table_3:100 initial to89.27 yields-.1073 proportional return=-10.73%. Official subtracts only, retaining percentage-number scale-10.73.",
      "official_program": "subtract(89.27, const_100)",
      "official_answer": -10.73,
      "reference_answer": "-10.73%"
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "question": "in 2017 what was the ratio of the htm investment securities ( period-end ) compared to investment securities portfolio ( period 2013end )",
      "document_quality": "clear",
      "audit_note": "table_6/7:2017 HTM47733 / investment securities247980=.19249. Ratio explicitly requested; other percent form is a unit-compliance issue.",
      "official_program": "divide(47733, 247980)",
      "official_answer": 0.19249,
      "reference_answer": "0.19"
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "question": "what percentage of total freight revenues was automotive in 2013?",
      "document_quality": "annotation_issue",
      "audit_note": "table_2/7: automotive2077 / total freight20684=.10042. Official denominator21963 is total operating revenue including other revenue, contradicting question.",
      "official_program": "divide(2077, 21963)",
      "official_answer": 0.09457,
      "reference_answer": "9%"
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "question": "what percentage of total accounts payable and other current liabilities was accrued casualty costs at december 31 , 2010?",
      "document_quality": "clear",
      "audit_note": "table_5/9:2010 casualty325 / total current liabilities2713=.11979.",
      "official_program": "divide(325, 2713)",
      "official_answer": 0.11979,
      "reference_answer": "12%"
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "question": "what was the percent of the change in the net revenues from dec . 282012 dec . 29 2013",
      "document_quality": "clear",
      "audit_note": "table_1:2012 net revenue53341 to2013 52708; (52708-53341)/53341=-.01187. Year/day OCR strings are awkward but column years establish direction.",
      "official_program": "subtract(52708, 53341), divide(#0, 53341)",
      "official_answer": -0.01187,
      "reference_answer": "-1.2%"
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "question": "considering the year 2015 , what was the lowest return for the investment?",
      "document_quality": "ambiguous",
      "audit_note": "table_3:2015 value99.90 vs inception100 implies cumulative-.1%; year2015-only return vs2014 105.72 differs. Question does not explicitly specify cumulative. Returning99.90 is asset value, not gain.",
      "official_program": "subtract(99.90, 100)",
      "official_answer": -0.1,
      "reference_answer": "-0.1%"
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "question": "what was the percentage gain recognized on the commercial mortgages held for sale in 2009",
      "document_quality": "annotation_issue",
      "audit_note": "text_24/25: gain107m and sales5.4bn. Official107/5.4=19.81481 omits1000 unit conversion; same-units result107/5400=.0198148. Another.3bn sale category also exists. A candidate using table_3 balance1301 chooses a different denominator too.",
      "official_program": "divide(107, 5.4)",
      "official_answer": 19.81481,
      "reference_answer": "2%"
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "question": "what was the operating margin for 2014?",
      "document_quality": "clear",
      "audit_note": "table_1/2:2014 operating income88.2 / sales450.4=.19583.",
      "official_program": "divide(88.2, 450.4)",
      "official_answer": 0.19583,
      "reference_answer": "19.58%"
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "question": "for the five years ended 12/31/2006 what is the performance difference of the class b common stock of united parcel service , inc . and the s&p 500 index?",
      "document_quality": "unit_convention",
      "audit_note": "table_1/2:five-year terminal total-return indices148.92 and135.02 (base100); difference13.9 index/percentage points. Official13.9; equivalent ratio difference.139 requires unit context.",
      "official_program": "subtract(148.92, const_100), subtract(135.02, const_100), subtract(#0, #1)",
      "official_answer": 13.9,
      "reference_answer": "13.9%"
    }
  ],
  "models": [
    {
      "model": "claude-haiku-4.5",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 6,
        "official_match": 7,
        "scale_difference_candidate": 4,
        "invalid_program": 3
      }
    },
    {
      "model": "claude-sonnet-4.5",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 7,
        "official_match": 6,
        "scale_difference_candidate": 6,
        "invalid_program": 1
      }
    },
    {
      "model": "claude-sonnet-5",
      "counts": {
        "official_match": 13,
        "other_numeric_or_interpretation_mismatch": 4,
        "invalid_program": 1,
        "scale_difference_candidate": 2
      }
    },
    {
      "model": "nova-pro",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 7,
        "scale_difference_candidate": 7,
        "official_match": 3,
        "invalid_program": 3
      }
    },
    {
      "model": "nova-lite",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 5,
        "invalid_program": 14,
        "scale_difference_candidate": 1
      }
    },
    {
      "model": "gpt-5.4",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 6,
        "official_match": 10,
        "scale_difference_candidate": 2,
        "invalid_program": 2
      }
    },
    {
      "model": "gpt-5.5",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 5,
        "official_match": 13,
        "scale_difference_candidate": 2
      }
    },
    {
      "model": "gpt-5.6-terra",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 7,
        "scale_difference_candidate": 9,
        "official_match": 4
      }
    },
    {
      "model": "gpt-5.6-luna",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 7,
        "scale_difference_candidate": 7,
        "official_match": 6
      }
    },
    {
      "model": "nova-micro",
      "counts": {
        "invalid_program": 19,
        "other_numeric_or_interpretation_mismatch": 1
      }
    },
    {
      "model": "nemotron-nano-3-30b",
      "counts": {
        "invalid_program": 18,
        "other_numeric_or_interpretation_mismatch": 2
      }
    },
    {
      "model": "glm-4.7-flash",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 6,
        "invalid_program": 9,
        "official_match": 5
      }
    },
    {
      "model": "gpt-oss-20b",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 3,
        "scale_difference_candidate": 5,
        "official_match": 9,
        "invalid_program": 3
      }
    },
    {
      "model": "gpt-oss-120b",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 7,
        "official_match": 11,
        "scale_difference_candidate": 2
      }
    },
    {
      "model": "qwen3-32b",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 7,
        "invalid_program": 4,
        "official_match": 5,
        "scale_difference_candidate": 4
      }
    },
    {
      "model": "nemotron-super-3-120b",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 9,
        "official_match": 10,
        "scale_difference_candidate": 1
      }
    },
    {
      "model": "llama4-scout",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 10,
        "invalid_program": 4,
        "scale_difference_candidate": 3,
        "official_match": 3
      }
    },
    {
      "model": "ministral-14b",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 7,
        "scale_difference_candidate": 6,
        "official_match": 6,
        "invalid_program": 1
      }
    },
    {
      "model": "qwen3-235b-a22b",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 10,
        "official_match": 8,
        "scale_difference_candidate": 1,
        "invalid_program": 1
      }
    },
    {
      "model": "llama4-maverick",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 12,
        "invalid_program": 2,
        "official_match": 5,
        "scale_difference_candidate": 1
      }
    },
    {
      "model": "llama-3.3-70b-bedrock",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 7,
        "official_match": 11,
        "scale_difference_candidate": 1,
        "invalid_program": 1
      }
    },
    {
      "model": "gemma-4-31b-bedrock",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 5,
        "official_match": 12,
        "invalid_program": 3
      }
    },
    {
      "model": "llama-3.1-8b",
      "counts": {
        "invalid_program": 20
      }
    },
    {
      "model": "llama-3.3-70b",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 8,
        "official_match": 10,
        "invalid_program": 1,
        "scale_difference_candidate": 1
      }
    },
    {
      "model": "qwen3.6-27b",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 4,
        "official_match": 14,
        "scale_difference_candidate": 1,
        "invalid_program": 1
      }
    },
    {
      "model": "exaone-3.5-32b",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 6,
        "invalid_program": 10,
        "scale_difference_candidate": 3,
        "official_match": 1
      }
    },
    {
      "model": "grok-4.3",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 6,
        "scale_difference_candidate": 9,
        "official_match": 5
      }
    },
    {
      "model": "grok-4.6",
      "counts": {
        "other_numeric_or_interpretation_mismatch": 7,
        "official_match": 9,
        "scale_difference_candidate": 3,
        "invalid_program": 1
      }
    }
  ],
  "evaluations": [
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "276eee90c58898aaef7b4a0fa1bcb2c122870578c56612f77f324979823ba84c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(33, 19)\", \"evidence\": [\"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -0.38206,
      "prediction_sha256": "0f0f3997baf90234f433505b939e80f23ad29114664e52d0be327d17b40c39a1",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "official_match",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"subtract(503, 814), divide(#0, 814)\",\n  \"evidence\": [\"text_6\", \"text_7\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 1373.658,
      "prediction_sha256": "ca92e5d63d38d9337e5de8770c0666ed3354a834db18b2112aa8ff800826e018",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"multiply(5283.3, 0.26)\", \"evidence\": [\"table_5\", \"text_1\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.01713,
      "prediction_sha256": "dbae2488f3599b2d66c581b9805d9cdbead4079d36e9c0084feb89e7c99bd4ec",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "official_match",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(22203, 21829), divide(#0, 21829)\", \"evidence\": [\"table_3\", \"table_0\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": -0.05751,
      "prediction_sha256": "add505fae109b53aca139aa58f244dc7bb872d3dd9ce79f0de1060b923c920fc",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"divide(-30584, 531822)\", \"evidence\": [\"table_4\", \"table_5\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 16.66159,
      "prediction_sha256": "03c78752eec5490ca07905ce2995e3388ea13fc4f8eff0f58b3ce9dc78c2be1a",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\"program\": \"divide(3280, 19686), multiply(#0, const_100)\", \"evidence\": [\"table_1\", \"table_7\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "deefb86f742692dc912307f4cbdf7648877bdb4e6a837676ba145bfa8395f272",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"divide(64, 549)\", \"evidence\": [\"table_5\", \"table_10\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 28.125,
      "prediction_sha256": "abf59471a9611e511bce13f608cdf6d45e53225a6f3fb14295c0d5a7e125746c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\"program\": \"subtract(205, 160), divide(#0, 160), multiply(#1, 100)\", \"evidence\": [\"text_31\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.006,
      "prediction_sha256": "e066cc392db242cda8926d12d12e5adafb7489e160939bd06c4b77bb8562cb44",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(0.157, 0.151)\", \"evidence\": [\"table_6\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.1621,
      "prediction_sha256": "d130ff66dc1145b7aa592e4702dd03a0f76d32ca65046bf2533a2c7ea5ac348c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\"program\": \"divide(35269, 249038), multiply(#0, const_100)\", \"evidence\": [\"table_2\", \"table_6\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 20743588.7,
      "prediction_sha256": "0c4fb00e566d45abd80c954097809fd3497469f6220ef47979c7bcd5a80c2498",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"multiply(431170, 48.11)\", \"evidence\": [\"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "2e0caac09600c82bf2f36fd5937930013891a959b33fd00e68ac5ab716d8a23f",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"table_min(delphi automotive plc ( 1 ), none)\",\n  \"evidence\": [\"table_1\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "43c6452f31e7f17f1c9ba2abba0e2dd34befe2766beb45b27e682ffa8665bff8",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"divide(47733, 247980)\", \"evidence\": [\"table_6\", \"table_7\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 10.04158,
      "prediction_sha256": "877f9028ca5a5e496e7e6b87791f1b7efe0ac8796bca79a4098d7f455d3f6aa3",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"divide(2077, 20684), multiply(#0, const_100)\", \"evidence\": [\"table_2\", \"table_7\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 11.97936,
      "prediction_sha256": "a6d5674bfe2188c0774ab3ce694f772f517446a777ec82c9c6b1f58e412084cd",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\"program\": \"divide(325, 2713), multiply(#0, const_100)\", \"evidence\": [\"table_5\", \"table_9\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "1d595df9658a9c69850dafaafeef3d429e8ec5b43138444eec0a0c1739415131",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"divide(subtract(52708, 53341), 53341)\", \"evidence\": [\"table_1\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 99.9,
      "prediction_sha256": "c145d26b2be8b239e283126c0ed72e28b3973686a56f38eecfe8615ef8fa75df",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"table_min(s&p 400 capital goods, none)\", \"evidence\": [\"table_3\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 1.07,
      "prediction_sha256": "881e23ebd177f3b1bb296a73e0d8986f4e1a557fb8c5b7a1966677a5ac706dcd",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"divide(107, const_100)\",\n  \"evidence\": [\"text_25\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "15ce29217e9cf35dd34213962c28baeea10468ac08bc1b9ac2042ce4d3464e71",
      "status": "invalid_program",
      "evaluation_error": "Expecting value: line 1 column 1 (char 0)",
      "evaluation_error_code": "invalid_json",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "claude-haiku-4.5",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"divide(425.3, 2443.28)\", \"evidence\": [\"text_6\", \"text_7\"]}\n```\n\nWait, let me reconsider. The document states in text_7 that \"operating margin of 17.4%\" for 2014. This is a directly stated answer.\n\n```json\n{\"program\": \"add(17.4, const_0)\", \"evidence\": [\"text_7\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "f94e62c22b6158b9c70746949ffbc5fb9a876b232a320ba8739286734f880b5c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-haiku-4.5",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(148.92, 135.02)\", \"evidence\": [\"table_1\", \"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "764cf1106c1af74a3ee40c4bf20fd25fec837c861775270462de3106b16a69e1",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"subtract(33, 19)\",\n  \"evidence\": [\"table_2\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 32.35294,
      "prediction_sha256": "98e1200c2883f2a4671ae24e62edf9d1a545844c5fbd8685ceec7f10e4b734f1",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"subtract(6.8, 4.6), divide(#0, 6.8), multiply(#1, const_100)\",\n  \"evidence\": [\"table_3\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 1373.658,
      "prediction_sha256": "252d14b7564206580cd4489b21ac200d6e5334aa30dd9018369fa8ff179d1c76",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"multiply(5283.3, 0.26), add(#0, const_0)\",\n  \"evidence\": [\"text_1\", \"table_5\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -1.68446,
      "prediction_sha256": "806afd27e27dc061337bb462f436f41d1a98d3d8ff10f5e7513a2d0bfe7499d7",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"subtract(21829, 22203), divide(#0, 22203), multiply(#1, const_100)\",\n  \"evidence\": [\"table_0\", \"table_3\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": -0.05751,
      "prediction_sha256": "09b4669efb01cc99417ed6b9caefe1589bbf5d2b9f71f84634e48fad6eaffff9",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"divide(-30584, 531822)\",\n  \"evidence\": [\"table_4\", \"table_5\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 16.66159,
      "prediction_sha256": "3b35d4b49ce030fbee983fc8d793abe225816075883a4ca9f8f3e5f29b6187dc",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\n  \"program\": \"divide(3280, 19686), multiply(#0, const_100)\",\n  \"evidence\": [\"table_1\", \"table_7\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "de6b0ec9ad8ee774fc36bb1e6026786f715b67bbeacf6609fb0d55b774b6ccb9",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"divide(64, 549)\",\n  \"evidence\": [\"table_5\", \"table_10\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 28.125,
      "prediction_sha256": "a9f9b8021d20b91058262cc3785ec8fd66698eadc296f62caae98dbcdbe4ca1d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\n  \"program\": \"subtract(205, 160), divide(#0, 160), multiply(#1, const_100)\",\n  \"evidence\": [\"text_31\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.6,
      "prediction_sha256": "a69554c5ea9a1b77d69b3d3dc989cd0d2955e3630cea18d5aff35892c9e39559",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "scale_difference_candidate",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\n  \"program\": \"subtract(15.7, 15.1)\",\n  \"evidence\": [\"table_6\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.1621,
      "prediction_sha256": "032a2155f12b8429362c9356ba0a5d50e2a70582bd0ae03368f1af3bfef1729a",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\n  \"program\": \"divide(35269, 249038), multiply(#0, const_100)\",\n  \"evidence\": [\"table_2\", \"table_6\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 20.74359,
      "prediction_sha256": "4e3ec425a48e8d10df96d3f73901f7ea15b9d8299eb15fdd722c90463dd5a8bf",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"multiply(431170, 48.11), divide(#0, 1000000)\",\n  \"evidence\": [\"table_2\", \"text_12\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "b6fa38d78a2ec0ce1b1bcdf13e56ceb06c1a0c4b413e11e02675168a25ceb75c",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"table_min(automotive supplier peer group ( 3 ), none)\",\n  \"evidence\": [\"table_3\", \"text_0\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "4fbbeb4161fcd587b8a38fa614a39968f10cd4d7808f7545b5c28cd6b1406d74",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"divide(47733, 247980)\",\n  \"evidence\": [\"table_6\", \"table_7\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 10.04158,
      "prediction_sha256": "b60b535a835f35348c3b2347bb5224164e9476f3eb66f6d6eebef35aff52924c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"divide(2077, 20684), multiply(#0, const_100)\",\n  \"evidence\": [\"table_2\", \"table_7\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 11.97936,
      "prediction_sha256": "53e1982bbd92ec10f7d04fac522d9ace92ff71f054178e721560aad098c228c1",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\n  \"program\": \"divide(325, 2713), multiply(#0, const_100)\",\n  \"evidence\": [\"table_5\", \"table_9\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -1.1867,
      "prediction_sha256": "64cc9e7890d4afd1862ca6b98a87c93b1f98d40531a03312ab5db48c1f8f8724",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\n  \"program\": \"subtract(52708, 53341), divide(#0, 53341), multiply(#1, const_100)\",\n  \"evidence\": [\"table_1\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 99.9,
      "prediction_sha256": "d3ec8483055457c86d2d406a28379b00a9327253ae24fa420bf1b2832c894e6b",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"table_min(s&p 400 capital goods, none)\",\n  \"evidence\": [\"table_3\", \"text_0\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.1019,
      "prediction_sha256": "b9a0fb02a84e41f20ae1ad23d185ee41bc7d1ef4020588136b0b8b2f1a8ee30f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"divide(107, 1050)\",\n  \"evidence\": [\"table_1\", \"text_25\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19583,
      "prediction_sha256": "a4aad76d133fc7919ac40ab77a5fc87659575e324ea468b16a23ddede0039135",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"divide(88.2, 450.4)\",\n  \"evidence\": [\"table_1\", \"table_2\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "0455f37d6e03167513c914614d0803b437ac75c98b91e20644c0361ab2fdf2db",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "claude-sonnet-4.5",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"subtract(148.92, 135.02)\",\n  \"evidence\": [\"table_1\", \"table_2\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -0.42424,
      "prediction_sha256": "247d9cd6db7983e72335539c6178b9cca001859fca2c389bffc346b251882387",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "official_match",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"subtract(19, 33)\", \"divide(#0, 33)\"], \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -0.38206,
      "prediction_sha256": "2dcfff02e54991806a37a67e99663d5c337fa03e4367a44ce7621c330ab38efa",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "official_match",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"subtract(503, 814)\", \"divide(#0, 814)\"], \"evidence\": [\"text_6\", \"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 1373.658,
      "prediction_sha256": "9bec041d11b3813b8a74ccf44a3bc7af2afbd98119e707f3d304446f1496c963",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"multiply(5283.3, 0.26)\"], \"evidence\": [\"table_5\", \"text_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -0.01684,
      "prediction_sha256": "99bbc64d1a56735acb0ca8d3466891c234b9f86bbb6886aa020b015a4b73bb86",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"subtract(21829, 22203)\", \"divide(#0, 22203)\"], \"evidence\": [\"table_0\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": -0.05751,
      "prediction_sha256": "ba8a44c6b1b268099f308db3a7cd99c270741e7d8fc9f3c5182905df7b67c8f8",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"divide(-30584, 531822)\"], \"evidence\": [\"table_4\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.16662,
      "prediction_sha256": "900acc41187cc0b0888a1257adbf17c1b003c6fe96b854560fa4cf3232a79f87",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"divide(3280, 19686)\"], \"evidence\": [\"table_1\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "86de3d4d023ecb24b8408fd7ebae3a1fb87cb3836b00abe9c7e812130651e9d2",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"divide(64, 549)\"], \"evidence\": [\"table_5\", \"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.28125,
      "prediction_sha256": "5e17dfc06f551dd210626283eb73e4b8c13fc2318c142045753207fd81dc9c76",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"subtract(205, 160)\", \"divide(#0, 160)\"], \"evidence\": [\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "866e5fcd070e015fa0a4a4011fdea071483d8de52dd1185a0fd741f565444a53",
      "status": "invalid_program",
      "evaluation_error": "missing or oversized program",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "claude-sonnet-5",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"subtract(15.7, 15.1), divide(#0, const_1)\"], \"evidence\": [\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.14162,
      "prediction_sha256": "0c5fadaa25332d4525106376be788833f21392568052bb591e435546efd8c490",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"divide(35269, 249038)\"], \"evidence\": [\"table_2\", \"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 20.74359,
      "prediction_sha256": "8875d8290c35af9c9a1b286eafdb54226c9db43547c09be53397d2c4eeaae0fe",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"multiply(431170, 48.11)\", \"divide(#0, const_1000000)\"], \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -0.1073,
      "prediction_sha256": "8e25a3459c3bae2d0da528bd3ed96d54b2ca8da3d1ff22ba1190f6a972b183ad",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "scale_difference_candidate",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [
        "multiply_100"
      ],
      "output_text": "{\"program\": [\"subtract(89.27, 100.00)\", \"divide(#0, 100.00)\"], \"evidence\": [\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "fce624839ff37a4c4fde4fa8dbbd397842d120754d25351bb5b27f44ae6e6df8",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"divide(47733, 247980)\"], \"evidence\": [\"table_6\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.10042,
      "prediction_sha256": "fc0391cb5047b77a4ad31df28b357ba46bb1bfdd4e947fec381037dcb9e833ab",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"divide(2077, 20684)\"], \"evidence\": [\"table_2\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.11979,
      "prediction_sha256": "42cb1422d962f527a9400afb1945a3045ce7154debfed3bb301c434c1efb20c5",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"divide(325, 2713)\"], \"evidence\": [\"table_5\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -0.01187,
      "prediction_sha256": "6b8f6f0fff6f49ea737f1b9cf1345ff058cb19448644ba3d504c5b663a5f0706",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"subtract(52708, 53341)\", \"divide(#0, 53341)\"], \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -0.001,
      "prediction_sha256": "e6b9d7fcc39ad74cbb9a20c2805fd7cc5a22b922d8d77323c48cbb771a957b52",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "scale_difference_candidate",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [
        "multiply_100"
      ],
      "output_text": "{\"program\": [\"subtract(99.90, 100.00)\", \"divide(#0, 100.00)\"], \"evidence\": [\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.08224,
      "prediction_sha256": "06ac05fda6186ffad875829fafa97d4b5feea7cc34d4d4bad84826ac3e72232a",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"divide(107, 1301)\"], \"evidence\": [\"text_25\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19583,
      "prediction_sha256": "b4bd4f8e0b7d3f15318f8e7b2211c812ab09b579f2fb02e68cffcc22b2869d4c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"divide(88.2, 450.4)\"], \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "8bd829fcaf305b99a7212180bde40f5e78d4487d25a784b08bd1c36dcd61e5e3",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "program_step_array"
      ],
      "model": "claude-sonnet-5",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": [\"subtract(148.92, 100.00)\", \"subtract(135.02, 100.00)\", \"subtract(#0, #1)\"], \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.14,
      "prediction_sha256": "e9bc20143b031c06bbfcbaf68e30763ce684b817b2ad25b1f61002e5ccafe601",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-pro",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(33, 19), divide(#0, const_100)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -38.20639,
      "prediction_sha256": "65b8b750696b7fe1c79efcc146c8ca4722d97cdee3783d5d1128c4c6ca2be902",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-pro",
      "category": "scale_difference_candidate",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"subtract(503, 814), divide(#0, 814), multiply(#1, const_100)\", \"evidence\": [\"text_6\", \"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 1.37366,
      "prediction_sha256": "e85c18320863968e36b3253fbab5666da1a61787b4dc276c85d680aee6d3329c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-pro",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(5283.3, 0.26), divide(#0, const_1000)\", \"evidence\": [\"text_1\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 1.71332,
      "prediction_sha256": "e27c36665232998db49832c0ea8a7e81d06dd885d8740a8c005d4f945c41c80f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-pro",
      "category": "scale_difference_candidate",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"subtract(22203, 21829), divide(#0, 21829), multiply(#1, const_100)\", \"evidence\": [\"table_0\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.05751,
      "prediction_sha256": "10e70fcd1f3895405d951c631e618c3078454e2451e6e2d11b1de3deb2c1df98",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-pro",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(531822, 501238), divide(#0, 531822)\", \"evidence\": [\"table_4\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 16.66159,
      "prediction_sha256": "9236062fa6158cda1f5710cfd24dbdfbec1dd122da8b54798c1db8d9e06ea867",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-pro",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"divide(3280, 19686), multiply(#0, const_100)\", \"evidence\": [\"table_1\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "3ef3697bfeb8a3d21eaabbac8fecc3a37f6d57ba2cefd4138d64a976e42c0958",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-pro",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(64, 549)\", \"evidence\": [\"table_5\", \"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 28.125,
      "prediction_sha256": "ce936789cd229fea733ca6cde3b018027b70d603c20f7ff9839917da59dc08bf",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-pro",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\n  \"program\": \"subtract(205, 160), divide(#0, 160), multiply(#1, const_100)\",\n  \"evidence\": [\"text_31\"]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 60.0,
      "prediction_sha256": "35b85255286819c3dd606027468f9ec4891463281210e610e165d307c9867348",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-pro",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(15.7, 15.1), multiply(#0, const_100)\", \"evidence\": [\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "08b4f40148e3a120d3e3c895b0e2104279d4ee4cb382a6405128aa4d2a5aec0d",
      "status": "invalid_program",
      "evaluation_error": "unsupported operation",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-pro",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"add(83382, 63060), add(#0, 35269), add(#1, 21598), add(#2, 14860), add(#3, 30869), divide(35269, 249038), multiply(#5, const_100), round(#6, const_1)\", \"evidence\": [\"table_0\", \"table_1\", \"table_2\", \"table_3\", \"table_4\", \"table_5\", \"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 20.74359,
      "prediction_sha256": "f79be276463abcb90759112d29cc507e2dc8497972f97ea32f4be4aa124b9e38",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-pro",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(431170, 48.11), divide(#0, const_1000000)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "cfccc397609e1faddb14278cdfc11617dd5818b775b120da5294dd4c5f650f32",
      "status": "invalid_program",
      "evaluation_error": "unknown or empty table row",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-pro",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_min(november 17 2011, none), table_min(december 31 2011, none), table_min(december 31 2012, none), table_min(december 31 2013, none), table_min(december 31 2014, none), table_min(#0, #1), table_min(#2, #3), table_min(#4, #5), table_min(#6, #7), table_min(#8, #9), subtract(100, #10), divide(#11, const_100)\", \"evidence\": [\"table_0\", \"table_1\", \"table_2\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "fe441f6b918d4cb9b1069dd0721efa6161701a68b4ec5172eac1c186f3478f02",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-pro",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(47733, 247980)\", \"evidence\": [\"table_6\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 10.04158,
      "prediction_sha256": "8a936cfd7f48a8d9a52e9315f081506779e9125e9b266bb82424da9ab79cebe6",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-pro",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(2077, 20684), multiply(#0, const_100)\", \"evidence\": [\"table_2\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 11.97936,
      "prediction_sha256": "ae6b35e0fe7d2bfa520702fb3c71b9e915aad2491179f1f24a0de3e2b3802f8a",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-pro",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"divide(325, 2713), multiply(#0, const_100)\", \"evidence\": [\"table_5\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 1.1867,
      "prediction_sha256": "b520395a9a5d331ae184223bf04c4465b2c35f8e5160506c88ff9b579bfc6162",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-pro",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(53341, 52708), divide(#0, 53341), multiply(#1, const_100)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "b9a85dd9d91c82da75b19856bf6a9f0addaa7729902af6594214ccd86306e8b3",
      "status": "invalid_program",
      "evaluation_error": "unknown or empty table row",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-pro",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"table_min(december 31 2015, none)\",\n  \"evidence\": [\"table_0\", \"table_1\", \"table_2\", \"table_3\"]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -25.05353,
      "prediction_sha256": "1272bcdf34aee1bd6de192cd4b9b54b0580fec1a310e04263cf103739819be7d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-pro",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(1050, 1401), divide(#0, 1401), multiply(#1, const_100)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 19.58259,
      "prediction_sha256": "8f2f9cda80cc9197f9b7849e0ce03ee1e173fe1105a9e690380aa45bcb384b97",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-pro",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"divide(88.2, 450.4), multiply(#0, const_100)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.139,
      "prediction_sha256": "67caeeaa9012b2ada22d5649bdf3dccf71463f30f6292c3647ddf208bea9baf9",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-pro",
      "category": "scale_difference_candidate",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [
        "multiply_100"
      ],
      "output_text": "{\"program\": \"subtract(148.92, 135.02), divide(#0, const_100)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -0.66667,
      "prediction_sha256": "07f9daae9b5ff32f7e4cc03c3c8e68dd3bc61667af5a4be511784a225fccfb6e",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-lite",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(33, 22), subtract(#0, 33), divide(#1, 33)\", \"evidence\": [\"table_2\", \"table_2\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 41.31944,
      "prediction_sha256": "40741ead2bf333b4ab01a503cbc128568d218fff1c46e42d977f3a3796b8ea74",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-lite",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(const_814, const_576), divide(#0, const_576), multiply(#1, const_100)\", \"evidence\": [\"text_7\", \"text_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 13.73658,
      "prediction_sha256": "c25384c0d45315018243987e21c67fe27a88c83b40643c50c5312c20a697bcfb",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-lite",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(5283.3, 0.26), divide(#0, 100)\", \"evidence\": [\"table_5\", \"text_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "d199019ee68c48fcb3ca11ffb68d5467d80207afa3d13b616911a017800e3e43",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_1'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-lite",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_1, table_2), divide(#0, table_0), multiply(#1, const_100)\", \"evidence\": [\"table_1\", \"table_2\", \"table_0\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "d3bb5da10d607f154964e04342e669c5b348fa840aa319ad4438ae10a9f44e24",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_5'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-lite",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(table_5, table_4)\", \"evidence\": [\"table_4\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "ca885c4e89a897e1080b5bfbbb298989e339042f0cc5909b6cf90f955152f231",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_1[2012]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-lite",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(table_1[2012], table_7[2012])\", \"evidence\": [\"table_1\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "85b435a164cc975df70f2cc574c919e4c1c0bbba87a55846c1cc38e699fe86be",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_5[goodwill]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-lite",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(table_5[goodwill], table_10[total])\", \"evidence\": [\"table_5\", \"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -0.28125,
      "prediction_sha256": "f809b9acc79533b5c4aedc14602337adc9098a46b550a888313b872826026c6d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-lite",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(160, 205), divide(#0, 160)\", \"evidence\":[\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "bcf1b7cd4e90dfdb119124992592b0837241bd211b3b9fb73fc2887ba0ddb5af",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_6[2017]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-lite",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_6[2017], table_6[2016])\", \"evidence\": [\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "9f6edb999b40baebf5e08557913b67610d39e85723bab6280b1d0b84eeec8492",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_2'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-lite",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(table_2, table_6), multiply(#0, const_100)\", \"evidence\": [\"table_2\", \"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "50f3b956e672a7f737e81f02019e10248714672fa0d788f6bdfccd460a9dcca0",
      "status": "invalid_program",
      "evaluation_error": "arithmetic requires exactly two operands without thousands separators",
      "evaluation_error_code": "wrong_arity",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-lite",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(431170, 48.11), add(#0, 8310, 44.99)\", \"evidence\": [\"table_2\", \"table_3\", \"text_18\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "a2f9bcc855bf7949509c5b1773b3dd1d9024a27379faf4e05aeb7d9aa8633a6d",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_1[november 17 2011]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-lite",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_1[november 17 2011], table_2[november 17 2011]), subtract(table_1[december 31 2011], table_2[december 31 2011]), greater(#0, #1)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "4def37f5ae0aeb25b9341146242c37b3f6cd945e16ec87a50b750f18f65daf23",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_6[2017]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-lite",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(table_6[2017], table_7[2013])\", \"evidence\": [\"table_6\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "3c49d05d160e98de3520c61692f7a01316bea27812a22df9a8f1425045e67857",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_2[2013]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-lite",
      "category": "invalid_program",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(table_2[2013], table_7[2013]), multiply(#0, const_100)\", \"evidence\": [\"table_2\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "4768a0ca6c393c38efc115ed7fb28b455f435367d9506fcee2f7b908855d52b4",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_5[325]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-lite",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(table_5[325], table_9[2713]), multiply(#0, const_100)\", \"evidence\": [\"table_5\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.01187,
      "prediction_sha256": "5f87e57bdc8b7cdd7c6e45f868e1ddc3021c974396921aaa63cbd05db6d67d17",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-lite",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(53341, 52708), divide(#0, 53341)\", \"evidence\": [\"table_1\", \"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "8d3bad0fd5ffb23406b9e3ade6c0d954284f7b643f89ebbd296efa8502603726",
      "status": "invalid_program",
      "evaluation_error": "table operation requires one row label and none",
      "evaluation_error_code": "wrong_arity",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-lite",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_min(table_1, december 31 2013, december 31 2014, december 31 2015, december 31 2016)\", \"evidence\": [\"table_1\", \"table_2\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "3085254c046ae5f6de287c6a948094cb3a0a4e730ba0ffe1b04c5d8043c04f2a",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_1[dec.31 2009]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-lite",
      "category": "invalid_program",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_1[dec.31 2009], table_1[dec . 312008]), divide(#0, table_1[dec . 312008])\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "df97cee19795d09d10e1fc276722151b19034fe3b5a561e55c159ebed80d1f52",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_2[88.2]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-lite",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(table_2[88.2], table_1[450.4])\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.139,
      "prediction_sha256": "c729d35e0f6802cd8dc3aa55f48b8322e180fc09778a5785ff0950edd06736ea",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-lite",
      "category": "scale_difference_candidate",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [
        "multiply_100"
      ],
      "output_text": "{\"program\": \"subtract(148.92, 135.02), divide(#0, 100.00)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "8c2d8739af3ac841d25246153269a09608601b178f41e00439d9bbb079b9dee9",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(33, 19)\",\"evidence\":[\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -0.38206,
      "prediction_sha256": "2bbb74495b23bcce8e91418418780d63ace5d7782a49d6925a45e49b51fa7944",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "official_match",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(503, 814), divide(#0, 814)\",\"evidence\":[\"text_6\",\"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 447.408,
      "prediction_sha256": "2d5f6517a8071f378d3c274d89b178e6cae39bbf34fe57cd7f2ca702b64b27ed",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"multiply(1720.8, 0.26)\",\"evidence\":[\"text_1\",\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 21829.0,
      "prediction_sha256": "c8fe7ce5d3a6e3cdde0bb9f48636f85dcd48740d69c70bd4f3aa37489e9bdd26",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"add(21829, const_0)\",\"evidence\":[\"table_0\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": -0.05751,
      "prediction_sha256": "c87248eec71e0fbdb040fe9472eaab04c24ed2cc316fbfa1c09cf80f259c5dac",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(-30584, 531822)\",\"evidence\":[\"table_4\",\"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.16662,
      "prediction_sha256": "be2ea7aeb046d6ff281958ba98f46407ec7eafe2eb8166ef773044413396e40f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(3280, 19686)\",\"evidence\":[\"table_1\",\"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "fa0e9320898b8d88b875631e2c5dd182f5b87638224deca0f19955d75094eb03",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(64, 549)\", \"evidence\": [\"table_5\", \"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.28125,
      "prediction_sha256": "5c93b46aa88ed632242628db2256063b4c324d8d0a1ef4a92e23eb3f4b03d675",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(205, 160), divide(#0, 160)\",\"evidence\":[\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.6,
      "prediction_sha256": "1a67dc2460f31b3cb15e912638ee1a89a02c6ae37fce580f16c3f4062b95f368",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "scale_difference_candidate",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"subtract(15.7, 15.1)\",\"evidence\":[\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.14162,
      "prediction_sha256": "e652bf0930ba741422015bbb374e26137bfe82e4b0ddad624d67faeeb95566f5",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(35269, 249038)\", \"evidence\": [\"table_2\", \"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 20743588.7,
      "prediction_sha256": "f7cc59eb2758de2bd71a81de4e48ac7a98fc6d540ede7ef5db2de780e61431d6",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(431170, 48.11)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "173092ac07a050a1daa17b4ea7e451c19a59b08bca97b7a582910999476c65c7",
      "status": "invalid_program",
      "evaluation_error": "invalid numeric argument",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"table_min(company index, none)\",\"evidence\":[\"table_0\",\"table_1\",\"table_2\",\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "e83a2990424a03bbc79697c6cd4768501630b7ee33fc70e220869926464c98ea",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(47733, 247980)\",\"evidence\":[\"table_6\",\"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.10042,
      "prediction_sha256": "b0ca38f50b52962672469580983a56221311df493626bcd66ece0ea3b635c518",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(2077, 20684)\",\"evidence\":[\"table_2\",\"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 11.97936,
      "prediction_sha256": "2cdeee18811fb8df99231891e2b086cea3941f1773aecc6303d2f73256dd820c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"divide(325, 2713), multiply(#0, const_100)\", \"evidence\": [\"table_5\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -0.01187,
      "prediction_sha256": "02834772a5143590e40abc7705241dec53e4ccf3e882fd0f3e5715d0652b7c32",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(52708, 53341), divide(#0, 53341)\",\"evidence\":[\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "abdaac53d4b1f0471c8c7e7d6e7996a22a21763434e88cc05cec8b67512cbe74",
      "status": "invalid_program",
      "evaluation_error": "unknown or empty table row",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"table_min(december 31 2015, none)\",\"evidence\":[\"table_1\",\"table_2\",\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.08224,
      "prediction_sha256": "328bbad17f11d2c89fa4285b386efb399a062426633cd70b775b6d5912fba0ac",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(107, 1301)\", \"evidence\": [\"text_25\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19583,
      "prediction_sha256": "51f9708dfb2a60fa4c84cd1a141798d78e7c6dac7462d5e303525daed28b105e",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(88.2, 450.4)\",\"evidence\":[\"table_1\",\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "2c6f21fcd91b2905b3fce6c5bdfb0d480066d2b70f1829cd03880e028ebc2f9d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.4",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(148.92, 135.02)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "30880c89ac160f9c670e6ccde84b06b48a4404a859b19568ef85b73ed2f00c11",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(33, 19)\",\"evidence\":[\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -0.38206,
      "prediction_sha256": "aabd345039ce2b1663e0589f2ef3020c5d3869b0569b5c38fe41d717388a3445",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "official_match",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(503, 814), divide(#0, 814)\",\"evidence\":[\"text_6\",\"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 447.408,
      "prediction_sha256": "600ad22966239594bc7a287dcebd992fffce9b1fb379339806e05a90890f1c6a",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"multiply(1720.8, 0.26)\",\"evidence\":[\"text_1\",\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.01713,
      "prediction_sha256": "5cd2aec90a126db0038f465f18b47e365000b9bf6c66e2fb7527d7a2057094ce",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "official_match",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(22203, 21829), divide(#0, 21829)\",\"evidence\":[\"table_0\",\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": -0.05751,
      "prediction_sha256": "07676ed8200db64ffcbbbefbc786d80cc52ee591fca263e1c51c31f564ef7c06",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(-30584, 531822)\",\"evidence\":[\"table_4\",\"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.16662,
      "prediction_sha256": "a469750bfc7bc4c39e0f79a9e1e91c2b0ab6d96a9263fe5926cf2ffd81ce528d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(3280, 19686)\",\"evidence\":[\"table_1\",\"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "a17e7e9db8a6b5aebe706d738c3a44349dee93072891c70ddd118a03a2e9db63",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(64, 549)\",\"evidence\":[\"table_5\",\"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.28125,
      "prediction_sha256": "acae0ec9bdb910d530b211e6a3a81b5554d856bfc5f3d3d7604e30787ee0ad3f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(205, 160), divide(#0, 160)\",\"evidence\":[\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.006,
      "prediction_sha256": "2f5e36c3c3231360fde8b92e9744fa969dc67d1e4807b93762bd4895c14a32ca",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(15.7%, 15.1%)\",\"evidence\":[\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.1621,
      "prediction_sha256": "7b0bcf3c6adf2fa6535ea9298f5b4733a2386abbf13a575d0a6208a5cd3687b0",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"divide(35269, 249038), multiply(#0, const_100)\",\"evidence\":[\"table_2\",\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 20.74359,
      "prediction_sha256": "f4af77b0e4b08b0c5698419b93201ae431dddaf11087c0772e518998b29e20ec",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"multiply(431170, 48.11), divide(#0, const_1000000)\",\"evidence\":[\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -0.1073,
      "prediction_sha256": "3d37ac2d7717135a592e2563c52365b1ec2042190094dc5ad10dfb9036d690ba",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "scale_difference_candidate",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [
        "multiply_100"
      ],
      "output_text": "{\"program\":\"subtract(89.27, 100.00), divide(#0, 100.00)\",\"evidence\":[\"table_1\",\"table_2\",\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "09a55b71db81bb4bef8e9d33e6a7a82b0d2f6c8cd7fa9628da56c5a4be1ff4df",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(47733, 247980)\",\"evidence\":[\"table_6\",\"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 10.04158,
      "prediction_sha256": "7b3406163e984ef3de089111acef980c8091de396cf77059d60f0fb2c6b4d197",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(2077, 20684), multiply(#0, const_100)\",\"evidence\":[\"table_2\",\"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.11979,
      "prediction_sha256": "4759a63bbcaab0c01a674c4d20ffc0848aaa2b5bcd869b5426c2f71d0afa6cdb",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(325, 2713)\",\"evidence\":[\"table_5\",\"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -0.01187,
      "prediction_sha256": "d6c6a97c4c356b156c01f22c8c7561c532f71dfdd4af11c23ac910e80a6390e0",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(52708, 53341), divide(#0, 53341)\",\"evidence\":[\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 99.9,
      "prediction_sha256": "32e2ba5d939ad4b59c6a06b1b818c3dffac302053de709a1afaa9ff5636d224d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"add(99.90, const_0)\",\"evidence\":[\"table_1\",\"table_2\",\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.08224,
      "prediction_sha256": "b35671cf52128341c68a9ed19e98e25d3708cd902d82d07e2c51d85a159c3a57",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(107, 1301)\",\"evidence\":[\"text_25\",\"table_0\",\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19583,
      "prediction_sha256": "a3ae7e7b7a933f7308fa1005ef9175d276e2666ad5043685cda69b8a9263f210",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(88.2, 450.4)\",\"evidence\":[\"table_1\",\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "bedb08d30647ae84ef199347b97b8825c566acf06b66ac91a25f5fa1d0e866ef",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.5",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(148.92, 135.02)\",\"evidence\":[\"table_1\",\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "50cf3ef339e48e5b2fc21a9dd97cd45908e26c1f60182ada4854346e936c22f1",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(33, 19)\",\"evidence\":[\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 61.82903,
      "prediction_sha256": "bd9aba28a292d37c126e30aaf7d724e4ae9c3c556a8372046c286152e93d4e95",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(814, 503), divide(#0, 503), multiply(#1, const_100)\",\"evidence\":[\"text_6\",\"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 447.408,
      "prediction_sha256": "ea394f1849253d0bf73354486ccc44bd6ab1e8e4261ab7c411741c173f1a11b3",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"multiply(1720.8, 0.26)\",\"evidence\":[\"text_1\",\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 98.31554,
      "prediction_sha256": "b7dc5fc6a6f2cb406bef271b12dbeb4d511e853def44687cbda98693cc69538b",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(21829, 22203), multiply(#0, const_100)\",\"evidence\":[\"table_0\",\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -5.7508,
      "prediction_sha256": "37c5b4ed4cd4bcbcb655ed3da87b6e633c78e583b578f75e85d633e02b2df398",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"divide(-30584,531822), multiply(#0,const_100)\",\"evidence\":[\"table_4\",\"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 16.66159,
      "prediction_sha256": "a6b3d49bac7b06c7b3d65e38c93fa6686a72c45d6e5b4f76cd91f6d717584580",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"divide(3280,19686), multiply(#0,const_100)\",\"evidence\":[\"table_1\",\"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 11.65756,
      "prediction_sha256": "7267e98503be50958be81eaf4c29e4094b600b815534676c6ce388bb0537c839",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"divide(64,549), multiply(#0,const_100)\",\"evidence\":[\"table_5\",\"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 28.125,
      "prediction_sha256": "887f712097aaadd97e8820118a337171c8a1b1bb7bef32154de1f314ccbd019d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"subtract(205, 160), divide(#0, 160), multiply(#1, const_100)\",\"evidence\":[\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.006,
      "prediction_sha256": "8f904fcfb82bb19542c3f84ec4b9e78cea82b5d11e2da4512228a0d1205d493f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(0.157, 0.151)\",\"evidence\":[\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.1621,
      "prediction_sha256": "cb0d5a3a6e294d0f3d37c0b903b5429a922915481b3ae41c022a4df46ff737e0",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"divide(35269, 249038), multiply(#0, const_100)\",\"evidence\":[\"table_2\",\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 20.74359,
      "prediction_sha256": "6e8715a93737fb49abfd54f1eba23c3fd6f69bd63b1a1b0c2a5b571483737dfd",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"multiply(431170,48.11), divide(#0,const_1000000)\",\"evidence\":[\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -0.1073,
      "prediction_sha256": "0f14b2dba1b29b44aa6b0b0ac4e8be7a0517376d5b25ff54ad1c460bcca08d1a",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "scale_difference_candidate",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [
        "multiply_100"
      ],
      "output_text": "{\"program\":\"subtract(89.27, 100.00), divide(#0, 100.00)\",\"evidence\":[\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 19.24873,
      "prediction_sha256": "18a9f9d7b06606e2b7e9e4eb8c045316ba100210e59fd599983494ec4645b887",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"divide(47733, 247980), multiply(#0, const_100)\",\"evidence\":[\"table_6\",\"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 10.04158,
      "prediction_sha256": "ebf31a1b416d35ff1a6f7eeae6bf6fc0f8aef7c5a6234e166b6bd8514ca657d1",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(2077,20684), multiply(#0,const_100)\",\"evidence\":[\"table_2\",\"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 11.97936,
      "prediction_sha256": "fbef3e3d07afcb87eb51d99fcb785b0a82db6c50fa92fe21fab81fefd05dc3b5",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"divide(325,2713), multiply(#0,const_100)\",\"evidence\":[\"table_5\",\"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -1.1867,
      "prediction_sha256": "9f4c2cc1a8ca73437943927e9f47f744ffc8a41cfbb88c458f62fbafbcae30a7",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"subtract(52708, 53341), divide(#0, 53341), multiply(#1, const_100)\",\"evidence\":[\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 99.9,
      "prediction_sha256": "15819a234724f488aff96c8e3f54c1c8a941e9484dbed9d4f2f96b1d4e3afeed",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"add(99.90,const_0)\",\"evidence\":[\"table_1\",\"table_2\",\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 8.22444,
      "prediction_sha256": "0c74658ccdb6e4b63f5b17ae4bb6e8c5d6fbff7f54957901a1787e833bd43bfe",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(107,1301), multiply(#0,const_100)\",\"evidence\":[\"text_25\",\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19583,
      "prediction_sha256": "39d0a40eae8bd0e4e3dba28e96ff8f0004df4622c84914dc917cbc602cbe39fd",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(88.2, 450.4)\",\"evidence\":[\"table_1\",\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "5b3065887358258090826c1d74b6d36aa24bd3a32832f82f179e819f3e6c78cd",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-terra",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(148.92, 135.02)\",\"evidence\":[\"table_1\",\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "38b3616cff6ff62a3569c04ee5fe62c490b3ea108a43da2d754ef1904727cb3b",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(33, 19)\",\"evidence\":[\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -38.20639,
      "prediction_sha256": "72a3a26f07bbecc8483a4f92cfaa3c6619c913712207741698d450cc4f054780",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "scale_difference_candidate",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"subtract(503, 814), divide(#0, 814), multiply(#1, const_100)\",\"evidence\":[\"text_6\",\"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 148.125,
      "prediction_sha256": "eeec867ea726823c9ba0522b5fbba67f41f3e4e04a5536e63276f40189f758fb",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"multiply(592.5, 0.25)\",\"evidence\":[\"table_4\",\"text_16\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.01713,
      "prediction_sha256": "21bca90acee8649732d1779a37bb6252ed608d7589b115a079730edb6a334bdc",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "official_match",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(22203, 21829), divide(#0, 21829)\",\"evidence\":[\"table_0\",\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -5.7508,
      "prediction_sha256": "ebdd1773e9d3fb61e55f3bf432eff210e2c7585d9043614442208aa3ac14aa47",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"divide(-30584, 531822), multiply(#0, const_100)\",\"evidence\":[\"table_4\",\"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 16.66159,
      "prediction_sha256": "1e949a72cb2f0f95aa34e24e59d0249e1f09666a1d16de9266bea10f21c77d52",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"divide(3280, 19686), multiply(#0, const_100)\",\"evidence\":[\"table_1\",\"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "5aabfced11762c8056fcd752037e616ba11401c3f9094b935ca312594604c15d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(64, 549)\",\"evidence\":[\"table_5\",\"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 28.125,
      "prediction_sha256": "d30120bb5ed66bf4a5dded3495fa10b6a3e7c00b4ddd1ab29f9fcefbdf8804b0",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"subtract(205, 160), divide(#0, 160), multiply(#1, const_100)\",\"evidence\":[\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.006,
      "prediction_sha256": "50b2064d6d1cc10ec4ef5ad8aa29ba9ca7532423da167aebcd001cd7e38b6fd7",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(15.7%, 15.1%)\",\"evidence\":[\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.1621,
      "prediction_sha256": "cda7a8d4b8bc4f4f25c5dfac2842bb42e385f2ddc0b0ee2d939a4849c85d8c96",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"divide(35269,249038), multiply(#0,const_100)\",\"evidence\":[\"table_2\",\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 21.11746,
      "prediction_sha256": "c0643ac85b3d364e78bf11d7d231c31ae02ad22d43f9dd1c985af0760a96cf5c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"multiply(431170, 48.11), multiply(8310, 44.99), add(#0, #1), divide(#2, const_1000000)\",\"evidence\":[\"table_2\",\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -10.73,
      "prediction_sha256": "80a4a02df09e518f2a6af0c25d9673ee4dad4731003449c2c387842a897d65a9",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(89.27, 100), divide(#0, 100), multiply(#1, const_100)\",\"evidence\":[\"table_1\",\"table_2\",\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "772ed3a31cca269e9297eda77c65e72dea5b1e044b86d6783d0227c55cd5782b",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(47733, 247980)\",\"evidence\":[\"table_6\",\"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 10.04158,
      "prediction_sha256": "b5a149f7a7d2cca1002ac66f8186684eb2bc1cae022010b850ea59c76e84dd41",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(2077, 20684), multiply(#0, const_100)\",\"evidence\":[\"table_2\",\"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 11.97936,
      "prediction_sha256": "d66b7cf7118597aec2ea1dea6c77d1393a593afb10ec6e53541109c66b8c7b52",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"divide(325,2713), multiply(#0,const_100)\",\"evidence\":[\"table_5\",\"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -1.1867,
      "prediction_sha256": "588333e0ab6762026d8415e008421455f4dfd51af1eb25c75a24cc8f7c0f879b",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"subtract(52708, 53341), divide(#0, 53341), multiply(#1, const_100)\",\"evidence\":[\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 99.9,
      "prediction_sha256": "a4eada191e1bf06a37ba0acb2b5e1f74519fb19a45210d76a4bc15c4effa18b7",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"add(99.90, const_0)\",\"evidence\":[\"table_0\",\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 8.22444,
      "prediction_sha256": "ad96a45e89c855cc2333368f3552f40f9736a4faf69138477b880be959bbf203",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(107, 1301), multiply(#0, const_100)\",\"evidence\":[\"text_25\",\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.174,
      "prediction_sha256": "97472483802b4caf5afbbf043d967b474a7d2ddd3a62e9a5531066f4672dbc37",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"add(0.174, const_0)\",\"evidence\":[\"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "4321beb1ae52bbcdcddcd567cec84bbc6234425d44cb110c45fc7cbf4880dac0",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-5.6-luna",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(148.92, 135.02)\",\"evidence\":[\"table_1\",\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "17a80c661ac6d505b32bfc2a8d54634a75b9dacd28fe8991ad0bf0f2ed588dec",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_2[2009]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"subtract(table_2[2009], table_2[2007])\",\n  \"evidence\": [\"table_2\"]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 41.31944,
      "prediction_sha256": "2e6a6566ad7ec8fd240dc80c3311e7a55098c0c56bd42eb29b53d6aaf3a13be4",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nova-micro",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"subtract(814, 576), divide(#0, 576), multiply(#1, const_100)\",\n  \"evidence\": [\"table_3\"]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "4b3fb6aaa439df8d6a0726e09065091269d2a4bdd73b82b9d4cb475e0141ed79",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"multiply(const_25, table_sum(total, none))\",\n  \"evidence\": [\"table_5\"]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "07c4bfb1109361fcfefce2b62057b8462a32082580254f4a6c5d55f66c718b5f",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_3[2]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_3[2], table_0[1]), divide(#0, const_100)\", \"evidence\": [\"table_0\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "cff25c65de0b9eb05d2f5b64d8635c6a6d7a149f9f6b192f054ea6cc0115b1e1",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_5'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"subtract(table_5, table_2), divide(#0, table_1)\",\n  \"evidence\": [\"table_1\", \"table_2\", \"table_5\"]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "96fb435d3a2dffbc286f5ab12f756970bb7c560d3cb5d9bbeb64612577805822",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_1[2012]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"divide(table_1[2012], table_7[2012])\",\n  \"evidence\": [\"table_1\", \"table_7\"]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "9ab7772c21ce6926cd824736a780040fa689390431a2390a19bd929b20d990c7",
      "status": "invalid_program",
      "evaluation_error": "Expecting ',' delimiter: line 1 column 67 (char 66)",
      "evaluation_error_code": "invalid_json",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(table_5[goodwill], table_10[total]), evidence\": [\"table_5\", \"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "ee02bb0bb490895ebbc1bf1fad9d52cf07579b414c3587dac07ac4b6168dd626",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"subtract(#0, #1), divide(#2, #1), multiply(#3, const_100)\",\n  \"evidence\": [\"table_1\", \"table_2\"]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "194e55285c51f4841610f961ccc5dffcab3251ca24caa2e89951895d2dfc3426",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_6[2017]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"subtract(table_6[2017], table_6[2016]), divide(#0, const_100)\",\n  \"evidence\": [\"table_6\"]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "04cd93e52c240f25b7ddf220e00690f3638cac69d3c5477b5c8fee48f61c0e5a",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"divide(table_sum(table_2, none), table_sum(table_6, none))\",\n  \"evidence\": [\"table_2\", \"table_6\"]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "bf11e0a824101bc2a0df4c19efd8eb56c8c0d152830fa866f2dc6430f7bff1cb",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_2[average pricepaid per share]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"subtract(table_2[average pricepaid per share], table_1[average pricepaid per share]), multiply(#0, table_2[total number of shares purchased ( a )]), divide(#1, const_1000000)\",\n  \"evidence\": [\"table_1\", \"table_2\"]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "bfd459a0ceaf7fdbcf8fee998297bdb0665471ae4ed7af2fe85b2acd12a0e28e",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_1[1]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_1[1], table_1[0]), divide(#0, const_100)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "1bdbbec9e4c47fcf32de1535953ce3ba4db0d07e3a4cea0fef57c599338e0613",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_6[2017]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"divide(table_6[2017], table_7[2013])\",\n  \"evidence\": [\"table_6\", \"table_7\"]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "a69c4cbb8637edbb0beeeefe9ec0f097135257320c72028abf500b2ab672d1f1",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_2[automotive]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"divide(table_2[automotive], table_7[total freight revenues]), multiply(#0, const_100)\",\n  \"evidence\": [\"table_2\", \"table_7\"]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "ea08513fe1479e9afe6b2cbaaceb64ade82a2076d94cab66fca33b6cf7ababc2",
      "status": "invalid_program",
      "evaluation_error": "arithmetic requires exactly two operands without thousands separators",
      "evaluation_error_code": "wrong_arity",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(table_5[dec. 31, 2010], table_9[dec. 31, 2010]), multiply(#0, const_100)\", \"evidence\": [\"table_5\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "3484400492707beba585e8ccfab5ca32b3a71021118c388baba0638220f07c31",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"subtract(#0, #1), divide(#2, const_100)\",\n  \"evidence\": [\"table_1\", \"table_2\"]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "2a22de2dde2fb28645755891feea5a1d8934b926c3a3fc9121a3ce23ca6bc92f",
      "status": "invalid_program",
      "evaluation_error": "Extra data: line 1 column 180 (char 179)",
      "evaluation_error_code": "invalid_json",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_1[4], table_1[3]), subtract(table_2[4], table_2[3]), subtract(table_3[4], table_3[3]), min(#0, #1, #2)\", \"evidence\": [\"table_1\", \"table_2\", \"table_3\"]}}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "62bfd0bc1a0f2d656d68792944c469d7ab4b155db35ea43909e4411dfb0be17e",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_1[dec.31 2009]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"subtract(table_1[dec.31 2009], table_1[dec.31 2008]), divide(#0, table_1[dec.31 2008]), multiply(#1, const_100)\",\n  \"evidence\": [\"table_1\"]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "de6b310545aded15cebd342515fe87ea63f9747cce971d9c91f208772d6d94f6",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_2[2014]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"divide(table_2[2014], table_1[2014])\",\n  \"evidence\": [\"table_1\", \"table_2\"]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "71265307ca793af30ff6dd7b8f1de86a8008c1d337b21a45958c4701dc1090d5",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_1[6]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nova-micro",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_1[6], table_2[6]), divide(#0, const_100)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "9e4a57da7a98a82be4d7670a18ba5cc8de1aeb3a78e55c1d565949549975a055",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(#0, #1)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "480fa2a63a62b09344d2eb473b5083248d2065e253ccfa67a4f05aa62728b021",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(#0, 100)\", \"evidence\": [\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 186.4,
      "prediction_sha256": "c3741238446da08e897cd59a23ad55aec651c77750a130e392783e86b23822ae",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(1720.8, 1534.4)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "f587211740081160414fc589a1857a845a5308c818dab4c314d6b73ff84740d6",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(#0, 100)\", \"evidence\": [\"table_0\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "1ebe8d594915ea010db37efe332d03dcf1b5f408203202511627c42b4bbd9189",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(#1, #0), divide(#2, #1)\", \"evidence\": [\"table_1\", \"table_0\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "3081246dd9de69fcda1c511ad9fd2f5b6a24aae67c557674682450ca5a861430",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(subtract(#0, #1), #0)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "15d770838d6cd0a2a8b92c90255e55096ac5f1e70e21d6df88457a466f36253a",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(#1, const_100)\", \"evidence\": [\"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "daa71ca396a4a707dc7d7ddafe91daf9209e5ffc7220cd95bfab42b1001b22e7",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(#0, 160), divide(#1, 160), multiply(#2, 100)\", \"evidence\": [\"table_1\", \"table_2\", \"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "5f6f8aacb021b16c395d2c6091d071f11858380c42b21b88bc5f4249d037df4e",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(#1, #0)\", \"evidence\": [\"table_5\", \"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "1c2dd6525619572cc0a1ab0927a36c868cb4f45db779120d27ad99ec4c70b9d6",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(subtract(#2, #1), #0)\", \"evidence\": [\"table_2\", \"table_1\", \"table_0\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "894890e667137dba33ef7ddf68ade6d8522a9efef69db8fd82061ed17ab2cb17",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(#0, #1)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "533d8a473a997ebb2d8262f3b4c0f21ac900dd08c9c51a0f07eaf628ff5b3697",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(#0, const_0)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "5781ecad5d415eb91a82f0082af7d30b7ab6ab119f784e093160b2be8e4bbf26",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(#0, #1)\", \"evidence\": [\"table_6\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "6c5c96c023a64e55f393cc342cdde8967fb7d190c1418c1a0264eddd7e13fdc0",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(subtract(#0, 2077), 100)\", \"evidence\": [\"table_2\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "06c73098b925c9cc0e2ffcc3ba7dc38c678927826d09ad7b974622a64f11b17f",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(subtract(#1, #0), #0)\", \"evidence\": [\"table_5\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "b06b7aaa3f4ceaab02b74218e67549fd6100e304fb6dcb4f36ff241224ef1b9a",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(#0, #1), divide(#2, const_100)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "7714f55a4bcec66c8f36799f36b5217c3740b19ac46be9f2fb7a9d38889db29e",
      "status": "invalid_program",
      "evaluation_error": "table operation requires none",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_min(#0, #1, #2)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.5,
      "prediction_sha256": "ac1f8d2aa5a9689c67dfa70f383bc29b0531e342fd676612fc4eaa16b5ddcb00",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(1050, 1000), divide(#0, const_100)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "fd8e114522ae358c1950addf33c3993d873b7cd35d2de4b87fd0caa189a8d0fb",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(#0, const_100)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "6d45822dc7d673f8fda9c619b20fc3be6a29bb8fe3d5098b26ba64ab7d84a362",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "nemotron-nano-3-30b",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(#0, #1)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.14,
      "prediction_sha256": "209fd2f854ddf006c510a1b066c33905427954ba117dc94963348c9da1b021a9",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(33, 19), divide(#0, const_100)\", \"evidence\": [\"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -0.32353,
      "prediction_sha256": "5c5c164f63fdfb494b485b99199112fe79fc298e06ce14c448cd79a97e629e17",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(4.6, 6.8), divide(#0, 6.8)\", \"evidence\": [\"table_3\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "400b48d1d69c2978e7a759a22911deb45113dd6611bc34b75ca96b3ba6b9ab59",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_1'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "invalid_program",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"multiply(table_1, const_26), divide(#0, const_100)\", \"evidence\": [\"table_1\", \"text_1\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 22103.0,
      "prediction_sha256": "412fabcbe05f01600f1bf06353de06b6b9217e2b79d5deac09c9a294a57dc914",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"add(21829, 249), add(#0, 125), divide(#1, const_100), subtract(#2, const_1), divide(#3, const_1), multiply(#4, const_100)\", \"evidence\": [\"table_0\", \"table_1\", \"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "372c5198dd24a974bb01c1122fcc7940634e19f2d38764dabf09a7a9c6c29e82",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_4'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"add(table_4, const_0), add(table_5, const_0), divide(#1, #0)\", \"evidence\": [\"table_1\", \"table_2\", \"table_4\", \"table_5\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "897f85f43dae5c39f0b59d411fd6c851572625659fee867d0ef299097dc2694b",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"divide(#0, #1), multiply(#2, const_100)\", \"evidence\": [\"table_1\", \"table_7\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "6485334b1c8b80b0f38f63134f1169c71a62b24ecdfb2e3569c19315da02c175",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_10'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(table_10, table_5), divide(#0, table_10)\", \"evidence\": [\"table_0\", \"table_1\", \"table_2\", \"table_3\", \"table_4\", \"table_5\", \"table_6\", \"table_7\", \"table_8\", \"table_9\", \"table_10\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.28125,
      "prediction_sha256": "c3fcd8219277b7f821921445038bec6a85e8c97c8fdf43e6eb113a35706800d6",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(205, 160), divide(#0, 160)\", \"evidence\": [\"text_31\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "de22c2c9dcc79d76ef18e57bc79e809fbe59a478b1593433c1ae442d970a8867",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_6'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(table_6, table_5)\", \"evidence\": [\"table_5\", \"table_6\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.14162,
      "prediction_sha256": "36a8d37c0a225f8f5eaa2784b13163cc2ee0b3183bd128696492142b104d6a69",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"divide(35269, 249038)\", \"evidence\": [\"table_2\", \"table_6\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "ed95b7fa8505fcbb074f4e39c7ba138ae81831c0dc61011e05f632dc2716ec0a",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"add(table_sum(november 26 to december 31 2007, none), const_0)\", \"evidence\": [\"table_3\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "bd296614481aedaa6b4262594bbbc1a9708282e92fbeceda6821ce62a2e7ab13",
      "status": "invalid_program",
      "evaluation_error": "unknown or empty table row",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"table_min(november 17 2011, none)\", \"evidence\": [\"table_0\", \"table_1\", \"table_2\", \"table_3\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "1380bd8d2bbec9c18194d6b566a05d1c2b687adb4300390c33d93f200a2ce90f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "glm-4.7-flash",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(47733, 247980)\", \"evidence\": [\"table_6\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 10.04158,
      "prediction_sha256": "4fddbdc87f9bd1a68cb939f737063711a20e47a6f9ab4fadcd59100e72df2040",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"divide(2077, 20684), multiply(#0, const_100)\", \"evidence\": [\"table_1\", \"table_7\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "0a53744eba268d24fa51d8de00d3c40acec692b3c903e08a5610716030e87085",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"divide(#0, #1), multiply(#2, const_100)\", \"evidence\": [\"table_5\", \"table_9\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -395.07,
      "prediction_sha256": "0529161d8508a4ccf4bfbe3248cf4b53d7e4d2d7eeb0abebdc805919ca40e2cf",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(13834, 53341), divide(#0, const_100)\", \"evidence\": [\"table_1\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "bc540e2074a9d558e06f7c3ee29f4d46b727817f9634a61242ee5a56a462d327",
      "status": "invalid_program",
      "evaluation_error": "unknown or empty table row",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "glm-4.7-flash",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_min(november 18 2013, none)\", \"evidence\": [\"table_0\", \"table_1\", \"table_2\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.2,
      "prediction_sha256": "ace8dd2f4475ed49b671b26fc98b99677bd7e934b1de4d91bb9c4a9b1c437474",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(120, 100), divide(#0, const_100)\", \"evidence\": [\"table_1\", \"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19583,
      "prediction_sha256": "c73100d9ac430ddfe1feeef79d4956dd23e18501db2a30c5f66fb1db1d774907",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"divide(88.2, 450.4)\", \"evidence\": [\"table_1\", \"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "cff2634686dbea043b2e3ed4470cd9b03752aef9d9c69513447ffecaee1c06ce",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "glm-4.7-flash",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(148.92, 135.02)\", \"evidence\": [\"table_1\", \"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "fe6439adc029625527565b8beb797eaf92341bdfa8531a955bd68e83bb479150",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(33, 19)\",\"evidence\":[\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -38.20639,
      "prediction_sha256": "88226ce96f1f10cea16bb7b4cbfc201ff800fb28555e5b4d1f88ff9ef0aca1e5",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "scale_difference_candidate",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"subtract(503,814), divide(#0,814), multiply(#1, const_100)\",\"evidence\":[\"text_6\",\"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 1373.658,
      "prediction_sha256": "a90e240d0ad3bc346b537524ec52c1cc51656f3b469ab2d93b9bec53a7690726",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"multiply(5283.3, 0.26)\",\"evidence\":[\"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.01713,
      "prediction_sha256": "c09d0e4a117d107799976e21daabebb0091ea1166193351c725850d90f262613",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "official_match",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(22203,21829), divide(#0,21829)\",\"evidence\":[\"table_0\",\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": -0.05751,
      "prediction_sha256": "335706a504874cc7ab4a6ca2a308d36cf33151b687659fba95d91d233b083ac4",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(-30584, 531822)\",\"evidence\":[\"table_4\",\"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.16662,
      "prediction_sha256": "4e06f3b76f705ce5f6fd4fc44435f8ccf6da7339f99b4dad6f38b1a790dbee63",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(3280,19686)\",\"evidence\":[\"table_1\",\"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "0d6036361d9e8238bbed642ea4a99372aa7870e34e208857548258364341b754",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(64, const_549)\",\"evidence\":[\"table_5\",\"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.28125,
      "prediction_sha256": "a74a09df7147e96eaf3fff08690d178cae9155c81d09d5e22c840984628eca0c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(205,160), divide(#0,160)\",\"evidence\":[\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.6,
      "prediction_sha256": "7faee902a53cc6697e91174be8b85aeedeee60fe1f88d288ae8376876f162635",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "scale_difference_candidate",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"subtract(15.7, 15.1)\",\"evidence\":[\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.1621,
      "prediction_sha256": "0c8d90f94da31dfea80b14466951527e490b041b02168a96cb9af19ae8b4ee5f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"divide(35269, 249038), multiply(#0, const_100)\",\"evidence\":[\"table_2\",\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "03e6064407f7b0e1d4b9f6a830f2082030d64d63bedf2ff1a21b9d96056db8f9",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"add(multiply(431170,48.11), multiply(8310,44.99))\",\"evidence\":[\"table_2\",\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -0.1073,
      "prediction_sha256": "5e1712d6a93a87366ec373777c5590c59c2d97890d9dff659dfe0c17f3fa68b3",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "scale_difference_candidate",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [
        "multiply_100"
      ],
      "output_text": "{\"program\":\"subtract(89.27, 100), divide(#0, 100)\",\"evidence\":[\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "903a57e4e683bba472532a18e625f226d4c303ee53ecc17acda8cba85b56d209",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(47733, 247980)\",\"evidence\":[\"table_6\",\"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "2ca310263d04006f6645ab37c7a9a15dd1df259f5e5333bd983d3032c4bb4b29",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "invalid_program",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"multiply(divide(2077,20684), const_100)\",\"evidence\":[\"table_2\",\"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 11.97936,
      "prediction_sha256": "56273a26a1eb94ddb265771b6ba8ec2e5e699eed3bb67634a2bc9a52398ef045",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"divide(325,2713), multiply(#0, const_100)\",\"evidence\":[\"table_5\",\"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -0.01187,
      "prediction_sha256": "c7eb1646a19357c1ac22de3fba025a616c07ac8d47d4a0131a650687a578e290",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(52708, 53341), divide(#0, 53341)\",\"evidence\":[\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 99.9,
      "prediction_sha256": "943feb04938cb8ef7f4d6718218717e2725650d7853b85412729e1eeab080941",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"add(99.90, const_0)\",\"evidence\":[\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "dd30b1f0f2db3cbb32a884bebafafe17e68e4d2d70bf21844f96e3cb10908f36",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "invalid_program",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"multiply(divide(107,1050),100)\",\"evidence\":[\"table_1\",\"post_text\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19583,
      "prediction_sha256": "7595b458a8eccd84d99bbc96b602346b576bf61392d3d6a2cbed65b209f9d65d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(88.2, 450.4)\",\"evidence\":[\"table_1\",\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "c871b585f63c52e5cf997c8e01f379ebe363dce55dd56bb6fe966937d75f7664",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-20b",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(148.92,135.02)\",\"evidence\":[\"table_1\",\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "ff906f8b681cf010760f98d9473754388d5611a80adc9449999ca4625b37cdae",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(33, 19)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -0.38206,
      "prediction_sha256": "4e78152da55344f63efbbf2ad0a999c0ce4b0a85e180e04faf82651095a63a27",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "official_match",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(503,814),divide(#0,814)\",\"evidence\":[\"text_6\",\"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 1320.825,
      "prediction_sha256": "f9901ec2d43a9842a8d0b525e2aa374dcd64eef83d8cf013f67b44f744ec1ed6",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "official_match",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(5283.3, const_4)\", \"evidence\": [\"table_5\", \"text_16\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 98.31554,
      "prediction_sha256": "a2de89d048b1de802d07990bb746d067a82d9574d1ff720f0c6f5cb4fdd103c9",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(21829, 22203), multiply(#0, const_100)\", \"evidence\": [\"table_0\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": -0.05751,
      "prediction_sha256": "3d2cb7918aa3c2aed4248acccf4950121cd7f99b4d5bdc18a6d461cca0a1e868",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(-30584, 531822)\", \"evidence\": [\"table_4\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 16.66159,
      "prediction_sha256": "7aca1c116fe3b159c643eb2ce0a0feb9cb67f728e33caefcd2ad00c4340c728b",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\":\"divide(3280,19686), multiply(#0,const_100)\",\"evidence\":[\"table_1\",\"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "bace7436de7e6dda71cd3880c680f932d8ee8b48c3c02eb3d27b7b696144fb5d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(64, 549)\", \"evidence\": [\"table_5\", \"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.28125,
      "prediction_sha256": "c54e4ddc147415364d13794ebb7a0b82256b85c985b6d461cb2a0546802079fb",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(205,160),divide(#0,160)\",\"evidence\":[\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.006,
      "prediction_sha256": "a7af1f529339348f7dd58e8d989ce9b8cd2367c273baa1ef30a44fedf1b36e34",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(0.157,0.151)\", \"evidence\": [\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.14162,
      "prediction_sha256": "c2144ac981812eff3a5795fc0f4e79312a189c852e2775a324b32fba9889f260",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"divide(35269, 249038)\",\"evidence\":[\"table_2\",\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 21.11746,
      "prediction_sha256": "1ae4589c9eaff6e47f5b6d0dfafbc48cc88f354bc08f06c992984a92596b8575",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"multiply(431170, 48.11), multiply(8310, 44.99), add(#0, #1), divide(#2, const_1000000)\",\"evidence\":[\"table_2\",\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -0.1073,
      "prediction_sha256": "7ad515366585d9ea3f9f80db62376e399f298c96d89ed1e3dcf28e1dca71073f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "scale_difference_candidate",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [
        "multiply_100"
      ],
      "output_text": "{\"program\": \"subtract(89.27, 100), divide(#0, const_100)\", \"evidence\": [\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "293b79399613d4f31b30a28bc5575ac21d297b3eba5f8f6f312db33a26c3ccdb",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(47733, 247980)\", \"evidence\": [\"table_6\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.10042,
      "prediction_sha256": "705b9c5fad75ebd1f17a69d12b628fb542b8dbeb81556848c11659b40ec695a8",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(2077, 20684)\", \"evidence\": [\"table_2\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.11979,
      "prediction_sha256": "73f72c3e438698bd3169c010f530d5847967c82d26c85379af90bf10b436f7f1",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(325, 2713)\", \"evidence\": [\"table_5\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -0.01187,
      "prediction_sha256": "7a06f7941a6c8e48a8442cd2f3fcb9671898065efe1f7cfbcc0b4c4e2bb9a806",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\":\"subtract(52708,53341), divide(#0,53341)\",\"evidence\":[\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 99.9,
      "prediction_sha256": "47615aa9e683914002a30c53c8846745feb58b4a759d4047adea9f452d2b367b",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"add(99.90, const_0)\", \"evidence\": [\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.08224,
      "prediction_sha256": "03654bd38604bf75fb9c6d1dbf1e06ed81026f35a7ce087aff59ce5bbdcf20ba",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(107,1301)\", \"evidence\": [\"text_25\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 17.4,
      "prediction_sha256": "29facb4fb42132a0e3218e4c84cd8577f96816522d5a845ff760eb39e0778c1e",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"add(17.4, const_0)\", \"evidence\": [\"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "e8446f60c61df9e18d285bf7bd4bbdde2333f3bb15a2dae08988385a1f5ac1bd",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gpt-oss-120b",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(148.92, 135.02)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.14,
      "prediction_sha256": "c2c96a658a094f2d252fd2eccf9e79e209a43e5efa93d963bb78e0bea5478405",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "qwen3-32b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(33, 19), divide(#0, const_100)\", \"evidence\": [\"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -3.11,
      "prediction_sha256": "143665c0ab2f0b84226d524d0b808bf964e27053511a7c476792312da28e6104",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "qwen3-32b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"subtract(503, 814), divide(#0, const_100)\",\n  \"evidence\": [\"text_6\", \"text_7\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 26.0,
      "prediction_sha256": "a8b5359b4764be4dddc7a1674c1e06fb024ae7b76a250d1df51a318c4ada5c8b",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "qwen3-32b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"multiply(const_26, const_100), divide(#0, const_100)\", \"evidence\": [\"text_1\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 22203.0,
      "prediction_sha256": "8eac80e02e67d33b393f26baed0c2ee67453bee908c7a045d82d41912f5c0988",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "qwen3-32b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"add(21829, 249), add(#0, 125), add(#1, const_0)\", \"evidence\": [\"table_0\", \"table_1\", \"table_2\", \"table_3\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "87ce4a6f09faa727f0b36937cb46f1183938d235a9d12ef367a6416fe1994f9f",
      "status": "invalid_program",
      "evaluation_error": "Expecting value: line 1 column 1 (char 0)",
      "evaluation_error_code": "invalid_json",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "qwen3-32b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": [\n    \"subtract(531822, 524804), divide(#0, 531822)\",\n    \"multiply(#1, const_100)\"\n  ],\n  \"evidence\": [\"table_4\", \"table_2\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "0a408e6ee9f00f0d28b1b06570e571a3066523ecfe33d292989e8f712e0707c2",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "qwen3-32b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(table_sum(agricultural, none), table_sum(total freight revenues, none)), multiply(#0, const_100)\", \"evidence\": [\"table_1\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "1c23d8d82947ce07883273efbb4a4eb0d0cb249cbe6a0c83503c78ae02ee70f7",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "qwen3-32b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"divide(64, 549)\", \"evidence\": [\"table_5\", \"table_10\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.28125,
      "prediction_sha256": "f22bc6e181967782d794045431f1dfdc27c8eb94bd18a2bbf2e93318f20ddb06",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-32b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(205, 160), divide(#0, const_160)\", \"evidence\": [\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.6,
      "prediction_sha256": "52ca5557d5c61b4528026adb4812108182776662d902b6bd51845be127802bb6",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-32b",
      "category": "scale_difference_candidate",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"subtract(15.7, 15.1), subtract(#0, const_0)\", \"evidence\": [\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.1621,
      "prediction_sha256": "1d85cf729bb3048bb03d5bc7d13c981b5f2485e0ec2be8c7e52e257c37781d78",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "qwen3-32b",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\n  \"program\": \"table_sum(2010, none), table_sum(total, none), divide(#0, #1), multiply(#2, const_100)\",\n  \"evidence\": [\"table_2\", \"table_6\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 20743588.7,
      "prediction_sha256": "b667095191f901af8e0758174e59bde58e079a388652e15759154bd806c60a45",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-32b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(431170, 48.11)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "c13a5d327586e3b767f793b06e4ae011e33800db44d0fe2d587460b1bf2d343c",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "qwen3-32b",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_min(delphi automotive plc ( 1 ), none)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "2de6c92381d1eb2ec6407815f87e3fb6c5536f1cd6c4f5332ba1743af5864f73",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "qwen3-32b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"divide(47733, 247980)\",\n  \"evidence\": [\"table_6\", \"table_7\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "918f79ada34bc9e803d9fa9bae7e859eb88d47e2fe584f51bf257f8c3b8121e5",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "qwen3-32b",
      "category": "invalid_program",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(table_sum(automotive, none), table_sum(total freight revenues, none), const_100)\", \"evidence\": [\"table_1\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 11.97936,
      "prediction_sha256": "85dea7290cbcdfd9bbfdad31eb45c8709dd308c37ca6876e4945b3f22a691eaa",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-32b",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"divide(325, 2713), multiply(#0, const_100)\", \"evidence\": [\"table_5\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -1.1867,
      "prediction_sha256": "884816ed4270a5622f4cdcca5264ed1f27e8f8369278ff28a36c0e0e029d9c3e",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "qwen3-32b",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\n  \"program\": \"subtract(52708, 53341), divide(#0, 53341), multiply(#1, const_100)\",\n  \"evidence\": [\"table_1\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 91.16,
      "prediction_sha256": "33918c8903c8c044a69de26badc6b4ae4e9b90342511d4674b0ef7b71832b5af",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-32b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_min(allegion plc, none)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 42.62948,
      "prediction_sha256": "a1673dd81e737e17f46e9dd46fb37f006f80690ab459ffcab50634a5c50feae8",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "qwen3-32b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(107, const_0), divide(#0, 251), multiply(#1, const_100)\", \"evidence\": [\"text_25\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.19583,
      "prediction_sha256": "c8fda7660ef4908a057d192da3e685a38f46929d9c3a53d75690ae3e7b5fdda2",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "qwen3-32b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"divide(88.2, 450.4)\", \"evidence\": [\"table_0\", \"table_1\", \"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "da541f5f520405427cf0d8b2413cd8fdc91b1f059156bfaa1cc8e0abb83adbb5",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence",
        "program_step_array"
      ],
      "model": "qwen3-32b",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": [\n    \"subtract(148.92, 100.00)\",\n    \"subtract(135.02, 100.00)\",\n    \"subtract(#0, #1)\"\n  ],\n  \"evidence\": [\"table_1\", \"table_2\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "a0c7d07f9793e826bd733639757262715ef645aeded04cb8d895a98959ec2246",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(33, 19)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.47826,
      "prediction_sha256": "e7457e5cee1ac4be368ada5e0cd010b1d7f70dd55cccd51ba3a6a4878c74cb68",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(6.8, 4.6), divide(#0, 4.6)\", \"evidence\": [\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 70.406,
      "prediction_sha256": "904fdf414c5a4c104767e966260a1ffb59d82b7a4bf2369ad67563194cc8830c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(502.9, const_0.14)\", \"evidence\": [\"table_3\", \"text_11\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.01713,
      "prediction_sha256": "8f00091412367c7d55e7d6b4f9a1366507def543c180d5d236dbbf355f4934a2",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "official_match",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(22203, 21829), divide(#0, 21829)\", \"evidence\": [\"table_0\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.05751,
      "prediction_sha256": "5bbba4d07ec9a91a08111d108f0424b720600dc8846544cf0fa5ffdcc8f8c23d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(30584, 531822)\", \"evidence\": [\"table_4\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.16662,
      "prediction_sha256": "51b2354afa6290c168b95b6fbbc6d35a3d5df4f83382341c36e3c9c50b35ce85",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(3280, 19686)\", \"evidence\": [\"table_1\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "ad33ecd0d94678905269abac3ac1fd4515f60ce2ad5f238f385efeb8797c8011",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(64, 549)\", \"evidence\": [\"table_0\", \"table_5\", \"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.28125,
      "prediction_sha256": "debdbda452f77170775ac1d342152ddd89dc8fbe16362d62f5ba13e827241837",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(205, 160), divide(#0, 160)\", \"evidence\": [\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.6,
      "prediction_sha256": "e35dddc002f20059d315e3271f469db7e6cc2367c38c190438d692e67fbe77f0",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "scale_difference_candidate",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"subtract(15.7, 15.1)\", \"evidence\": [\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.14162,
      "prediction_sha256": "cefa636b1f1b20e1efef7d4e58f7294b6afab59be8eaada1eaed4addf461de84",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(35269, 249038)\", \"evidence\": [\"table_2\", \"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 20743588.7,
      "prediction_sha256": "bb5c3716df45dccbf6194e7e401344abd21e5798b5cc1ea0199ef4b3efd2c57c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(431170, 48.11)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.98,
      "prediction_sha256": "cd8c28489614e50747441c8255492d9f2ed1210a9b8dbd56b70acab3f90f278b",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(100.98, 100.00)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "187aeb45147845f862adb23df775ee18aec5fbd47cf113655f9b9171084e1c60",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(47733, 247980)\", \"evidence\": [\"table_6\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.10042,
      "prediction_sha256": "9226e85eb35e2c78a5b4273a08bbbfe78245d463044dd0140aef60f54acf0982",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(2077, 20684)\", \"evidence\": [\"table_2\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.11979,
      "prediction_sha256": "0985f1dd1bca79e5672416a08d00c03e638a5d0fa00c9cb8bdcaeea1c8b55ea9",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(325, 2713)\", \"evidence\": [\"table_5\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -0.01187,
      "prediction_sha256": "2cb9e163129295e755e05855b9a5972dede6c522021e762acd32b0f23f16d4a0",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(52708, 53341), divide(#0, 53341)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 37.71,
      "prediction_sha256": "37b3bae737b5748c778bdf61fe81fcdfe876c57f98ee312a5e7be4d3a8117471",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(137.71, 100.00)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 3.18327,
      "prediction_sha256": "e380eb497eaea221d443ebb915e48f4c7e7ebb573209f1f20239930c2faad09e",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(1050, 251), divide(#0, 251)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19583,
      "prediction_sha256": "404fd44d351ed175818dae20ed10200aebdf2aa7634814b5ff44edacf2dfffab",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(88.2, 450.4)\", \"evidence\": [\"table_2\", \"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "28ab5be851b3dfdcae88ef4aaef03ccb86e2b62c90135cc3adced121f4a45958",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "nemotron-super-3-120b",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(148.92, 135.02)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "723b84a7fe57851c6d474187cc7782e7249ebed07ad724fdb4a78f57e9628474",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(33, 19)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.38206,
      "prediction_sha256": "38bb39e9b93baa30c4c6cd58b7c7988ae8c1807ddc89cca90e7bfb24a2797a54",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(814, 503), divide(#0, 814)\", \"evidence\": [\"text_6\", \"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 1373.658,
      "prediction_sha256": "a96f5c539dbbdf10cbbf4a940a48cff1324be3b00687da429fb446f078eb2d27",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(0.26, 5283.3)\", \"evidence\": [\"text_1\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.992,
      "prediction_sha256": "8056ab8d8c896d90ecb1b29d215ce25cc6cc792486c00b08995c734b54f8b7d3",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(249, 125), divide(#0, 125)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.94249,
      "prediction_sha256": "aa037a3ad6045f8468a6321dad86f4875d164246675116b6a7a2d52b1f15762b",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(531822, 30584), divide(#0, 531822)\", \"evidence\": [\"table_4\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "9bed0320ccbb81940ded087b48f1105b4c79e520563afbebead8713d731f4790",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_1[2012]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(table_1[2012], table_7[2012]), multiply(#0, const_100)\", \"evidence\": [\"table_1\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -0.88342,
      "prediction_sha256": "35fd0d6b69a1b1d6577be1d893bd294cc91ac2a493ad0a6663c7ae17fbc76b4a",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(64, 549), divide(#0, 549)\", \"evidence\": [\"table_5\", \"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 28.125,
      "prediction_sha256": "7f77776b03a6436711da666e5d13aea2bc91254addcf8a309afd819bcffa06dd",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"subtract(205, 160), divide(#0, 160), multiply(#1, const_100)\", \"evidence\": [\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "14991a4f69c10d9ffa1fdd53509cfe6f7123c2e9bd605a23601ff7cd24f7ffd7",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_6[2017]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_6[2017], table_6[2016])\", \"evidence\": [\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.1621,
      "prediction_sha256": "0dc0ac8e194826d3dadd59bc00e9aca725bdfdab791a2612c5c0ef12c61d3e05",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"table_sum(2010, none), table_sum(total, none), divide(#0, #1), multiply(#2, const_100)\", \"evidence\": [\"table_2\", \"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 20743588.7,
      "prediction_sha256": "071fa06d264aa883ca1939bcf9a892e3d048ed9aba45831838fc29844521a064",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(431170, 48.11)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "0805a5d8dac25a6fede37b8cb55210ee3fa58a71d16dcf4c43d00442596d0f40",
      "status": "invalid_program",
      "evaluation_error": "table operation requires none",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(100.98, 100), subtract(100.80, 100), subtract(89.27, 100), table_min(#0, #1, #2)\", \"evidence\": [\"table_1\", \"table_2\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "f10dbc3c3fbd6e44a8413137469d6d121d446593261b8166b468790c080676d5",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(47733, 247980)\", \"evidence\": [\"table_6\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 10.04158,
      "prediction_sha256": "7dc60432ab92fe681bc05ff8747026923fcf0c9c83cbfa2824ff61ef5e8c4152",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(2077, 20684), multiply(#0, const_100)\", \"evidence\": [\"table_2\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 11.97936,
      "prediction_sha256": "3f1bf7e0b8d2c207050a8629c70a2b0bbc8ab816c519e7e8b4a645f9d9dced2f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"divide(325, 2713), multiply(#0, const_100)\", \"evidence\": [\"table_5\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.01201,
      "prediction_sha256": "46c113769c0c65d4bdd949289fe921a55d653cf6bc5976059ab86aaec84dc377",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(53341, 52708), divide(#0, 52708)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "e2a16d7ce92a3973436592465e42e7a232da391797eb27096b4b48a9331afcb1",
      "status": "invalid_program",
      "evaluation_error": "table operation requires none",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(137.71, 100), subtract(134.67, 100), subtract(119.22, 100), subtract(99.90, 100), table_min(#0, #1, #2, #3)\", \"evidence\": [\"table_1\", \"table_2\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.33429,
      "prediction_sha256": "cee0cb8b2918222d8919d8c25b7de4263a2bdbe3bd65347f0b49e399729e07c7",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(1401, 1050), divide(#0, 1050)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19583,
      "prediction_sha256": "de6f7e7b67052078e141c4b7cd579a55081b1216dc56bb9ec2c425c3fe72cde1",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(88.2, 450.4)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "083ed94dcf361f41d9c4bfcc3d75f117e6a90bca5297ab64d03f4d434479878a",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-scout",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(148.92, 135.02)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "24a98ee4b5f497c83bf161b0743436db2a4410185d68dc0bc5ba68da15af0ade",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"subtract(33, 19), add(#0, const_0)\",\n  \"evidence\": [\"table_2\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -29.23833,
      "prediction_sha256": "38d69df54705cd76d8a5a375e636fc23671e0538cf69ec89a5f889cbae993054",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"subtract(576, 814), divide(#0, 814), multiply(#1, const_100)\",\n  \"evidence\": [\"text_7\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 1373.658,
      "prediction_sha256": "bbbb9e00debea026d133ad4cba38ac85a1ea6cc56f04cd278c91f5dc528cf105",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"multiply(5283.3, const_0.26), add(#0, const_0)\",\n  \"evidence\": [\"text_1\", \"table_5\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 1.71332,
      "prediction_sha256": "1f9637365b224d7c8299de4925f8a30ce592cc3794475f6b5200e5b85e009d72",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "scale_difference_candidate",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\"program\": \"subtract(22203, 21829), divide(#0, 21829), multiply(#1, const_100)\", \"evidence\": [\"table_0\", \"table_3\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.0132,
      "prediction_sha256": "1a9368742637d106a28016a8005987d767b52f7441900bb6bdcee7234783bca3",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"subtract(531822, 524804), divide(#0, 531822)\",\n  \"evidence\": [\"table_4\", \"table_2\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 16.66159,
      "prediction_sha256": "bdea7802c7a9122318aeb5b3cbb12cf0f692413d69fb91edc44d3fe757c12328",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\n  \"program\": \"divide(3280, 19686), multiply(#0, const_100)\",\n  \"evidence\": [\"table_1\", \"table_7\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "0be223709fdee1a865f20d64a9307045dc990cdf1a847e221b21327663709831",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"divide(64, 549)\",\n  \"evidence\": [\"table_5\", \"table_10\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 28.125,
      "prediction_sha256": "1e405e0410f74467d86b8493540584f20e06c358c7c0831a135788bd18a18ff5",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\n  \"program\": \"subtract(205, 160), divide(#0, 160), multiply(#1, 100)\",\n  \"evidence\": [\"text_31\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.006,
      "prediction_sha256": "739fca7828aa9a4e8463e13e76e6ceebdff595afb76ac8a1168a94cf9110a352",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"subtract(15.7, 15.1), divide(#0, const_100)\",\n  \"evidence\": [\"table_6\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 14.1621,
      "prediction_sha256": "3923fc03287ad8814a5a64dc9571b221ed440adad76fead2c061da342d4a54de",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\n  \"program\": \"divide(35269, 249038), multiply(#0, const_100)\",\n  \"evidence\": [\"table_0\", \"table_6\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 20.74359,
      "prediction_sha256": "1024f4797de654e5b61d43e89723c39ca0c2b966a907444a5f902f99b633a5fb",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"multiply(431170, 48.11), divide(#0, const_1000000)\", \"evidence\": [\"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "73c98ac7ba981c231a4da4c7271fdefcc26484796501ff0a0ccd13a61149b44a",
      "status": "invalid_program",
      "evaluation_error": "unknown or empty table row",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"table_min(november 17 2011, none), subtract(#0, const_100)\",\n  \"evidence\": [\"table_1\", \"table_2\", \"table_3\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "3ca6cc76dc98aa01fd8b1994827ac720fb0f109a0c0b6bde88a9660dac20cd49",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"divide(47733, 247980)\",\n  \"evidence\": [\"table_6\", \"table_7\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.10042,
      "prediction_sha256": "efff403be4e206d603c59bc24fc66884b9cbcaea2c3d3fd278b2c7f8658ca616",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"divide(2077, 20684)\",\n  \"evidence\": [\"table_2\", \"table_7\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 11.97936,
      "prediction_sha256": "c42bda90968c83d95cd6e69b40a79a72794c60f2acddb36bd25f653fe38a42a7",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\n  \"program\": \"divide(325, 2713), multiply(#0, const_100)\",\n  \"evidence\": [\"table_5\", \"table_9\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -1.1867,
      "prediction_sha256": "37a1476b46c5d2cb7e42f0b1efc8df0b6aa09601d525b6d0d5d1ef77e1a1e827",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\n  \"program\": \"subtract(52708, 53341), divide(#0, 53341), multiply(#1, const_100)\",\n  \"evidence\": [\"table_1\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 91.16,
      "prediction_sha256": "eb69c779c5acc3f4d1b863d9d8540e37a6b8dd921dfcf92790c892722dc02124",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"table_min(allegion plc, none)\",\n  \"evidence\": [\"table_1\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 35100.0,
      "prediction_sha256": "c23c99ef30ae15c416b3b1299019d5b8beef98bfdd42e476d2619004b35b7818",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"subtract(1050, 1401), divide(#0, const_m1), multiply(#1, const_100)\",\n  \"evidence\": [\"table_1\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19583,
      "prediction_sha256": "610d07f9cdc3ad922e1baf1faa80ff815f4c8f5491034b47ec2dbd37ffe38c94",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"divide(88.2, 450.4)\", \"evidence\": [\"table_1\", \"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "96b28b6aa4224423884eb3af6b4b831a3166ad5a5eeec02d34b4592103d9733a",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "ministral-14b",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\n  \"program\": \"subtract(148.92, 135.02), add(#0, const_0)\",\n  \"evidence\": [\"table_1\", \"table_2\"]\n}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "4f1ca972fd9e3404dd65d03effee5c8ea1ee5abcf07a3db61bf4307ceb49484d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(33, 19)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 32.35294,
      "prediction_sha256": "27bd5cf0713bcb5f145a8265b91b393ec47d1a84f30207df0ee2b892682f4851",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(6.8, 4.6), divide(#0, 6.8), multiply(#1, const_100)\", \"evidence\": [\"table_3\", \"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 1373.658,
      "prediction_sha256": "57d4a6685423eb3b7f972dfdc4f08ea6cd40f2fe5c93be0ad0ae5c37ea3cfb94",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(5283.3, 0.26)\", \"evidence\": [\"text_1\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.0,
      "prediction_sha256": "bef002f6ae25b0fdc938e5a44e6966357be3347bacde9d6bd3d1012cb87219a9",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(21829, 21829), divide(#0, const_100)\", \"evidence\": [\"table_0\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": -0.05751,
      "prediction_sha256": "6e365c10d3779cf2dc4dae21173a9251ca541d66db94df9fe096cfd634a7d7f0",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(-30584, 531822)\", \"evidence\": [\"table_4\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.16662,
      "prediction_sha256": "b6827b82d8fd3a1e8ed8843c6247e71482bca55c16063e8a57ccf51c4149fcd6",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(3280, 19686)\", \"evidence\": [\"table_1\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "a68b7bf11bd740a7f6df2733f1424278001054f653ba3cd538c59b43cc5b710f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(64, 549)\", \"evidence\": [\"table_5\", \"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.28125,
      "prediction_sha256": "386714e4fa399584226d71c322fccd219f3bacae92e3826811f537a991c72004",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(205, 160), divide(#0, 160)\", \"evidence\": [\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -0.006,
      "prediction_sha256": "8c08ee1f3870b72e6b2a0ca4b4011146c2ea6473664b28b5bd2244571d22b741",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(0.151, 0.157)\", \"evidence\": [\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.14162,
      "prediction_sha256": "44db5bbdf450d7e2635bd284f92d6dd7c704bf906852313dc18758b0d4d443d9",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(35269, 249038)\", \"evidence\": [\"table_2\", \"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 20743588.7,
      "prediction_sha256": "a043be8dcdd22a2b8cf8ae9c5e99723b222a6c2349c4a8927ff872e1575860af",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(431170, 48.11)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.0098,
      "prediction_sha256": "3de5d311d0a20f429035f851ab6140c1cf0709a1f58f2ce4c39d20b262bf8054",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(100.98, 100.00), divide(#0, 100.00)\", \"evidence\": [\"table_0\", \"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "24d4d97362700356c9a059f1db2f17a1a832a87385e9a57395d6db052e923138",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(47733, 247980)\", \"evidence\": [\"table_6\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.10042,
      "prediction_sha256": "5c460bc0b14cf2eae947b477c1d6409a68d631c62e18772712ecd089c870f4ff",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(2077, 20684)\", \"evidence\": [\"table_2\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 11.97936,
      "prediction_sha256": "7c123543e14f3dfacc1533001519a76d9366b73c44f0fa251e8efc2d1fbf6a49",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"divide(325, 2713), multiply(#0, const_100)\", \"evidence\": [\"table_5\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": -0.01187,
      "prediction_sha256": "025b549920476f04c01dc1a9e511a50353256f6fd5e4c61dc7670f7a337a3210",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(52708, 53341), divide(#0, 53341)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "3990691ee088fd390edbfece6f14ad7239f6c1093c2161cd26c40552926fb1b8",
      "status": "invalid_program",
      "evaluation_error": "table operation requires none",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_min(allegion plc, none), table_min(s&p 500, none), table_min(s&p 400 capital goods, none), table_min(#0, #1), table_min(#3, #2)\", \"evidence\": [\"table_1\", \"table_2\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 25.05353,
      "prediction_sha256": "1ac52ea0fb0b07dfe4ec9ceb8a9d6961d526187d09c9034472a05d7308bdd156",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(1401, 1050), divide(#0, 1401), multiply(#1, const_100)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 17.4,
      "prediction_sha256": "2cba63f31c7c408c9d9922d232c30f04f45f9baaa35ba15cc180991e606ab9bd",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"add(17.4, const_0)\", \"evidence\": [\"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "26f206ad4cecb6012d261d65458092929780237be6bb76b4f75b806ec7e61d02",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3-235b-a22b",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(148.92, 135.02)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "32105e94b37f044ff8ffcff56385205a27f96e43a3d458b124a74b676519fb62",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(33, 19)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.61829,
      "prediction_sha256": "a862391d75a133785dc4ddf84ac9a672d9dc16100c885cf0d79bac3e3b719be4",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(814, 503), divide(#0, 503)\", \"evidence\": [\"text_6\", \"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "3729188e5b189d843f76195a0b15e1836f8dc2b68ae66e753329b353ed048266",
      "status": "invalid_program",
      "evaluation_error": "unknown or empty table row",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "invalid_program",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_sum(total revenue, none), multiply(#0, const_0.26)\", \"evidence\": [\"table_5\", \"text_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.0,
      "prediction_sha256": "5c7120a2f8827b30e13ec88352d4d6ad3609fcaaa319882f5718483d7cb38bab",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"add(21829, 249), add(#0, 125), subtract(22203, #1), divide(#2, #1), multiply(#3, const_100)\", \"evidence\": [\"table_0\", \"table_1\", \"table_2\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": -0.05751,
      "prediction_sha256": "87ee904e5c72acdc9c8b2e6e2a73e864b30ecf3bfdfae10fcfaf6b2ab3ae593d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_sum(non-tower cash flow for the twelve months ended december 31 2005, none), divide(#0, 531822)\", \"evidence\": [\"table_5\", \"table_4\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 5.57084,
      "prediction_sha256": "6b4c3ccd8ed2a21484e3677b38f2be0973c8b5ff4aae2a8f76b75f81b5416fe4",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_sum(total freight revenues, none), divide(3280, #0), multiply(#1, const_100)\", \"evidence\": [\"table_7\", \"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.20036,
      "prediction_sha256": "e7770242c2fb9a7d15d50d27f7f2ba2858918348e4b815599185c8baf9362ca9",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_sum(goodwill, none), divide(#0, 549)\", \"evidence\": [\"table_5\", \"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.28125,
      "prediction_sha256": "f5d6577f2d41acd048653e8faf6e63f704829a3a9b6714705caaacd722b156d3",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(205, 160), divide(#0, 160)\", \"evidence\": [\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 60.0,
      "prediction_sha256": "149ec9757adf340567eb3f57b1a1118d2b0f5dbda9505957ce74685166b3283c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(15.7, 15.1), multiply(#0, const_100)\", \"evidence\": [\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.14162,
      "prediction_sha256": "b339d95796020cc408da9290233d7e01494518a2f1bca4c8246dbca7972ecec0",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_sum(total, none), divide(35269, #0)\", \"evidence\": [\"table_6\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 20743588.7,
      "prediction_sha256": "975026d204de87d069dab28a3cd6ea1902a1ba7b7e39ba43a9eceed3a334090c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(431170, 48.11)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "cad4ba18de95c67ebc373af10f67d8976df6a3f1ff5a4e9aabac4b204ca5e522",
      "status": "invalid_program",
      "evaluation_error": "unknown or empty table row",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_min(december 31 2011, none)\", \"evidence\": [\"table_1\", \"table_2\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "5cc54af1ab5575eeb39f534906aa98ee852058e6f7cdf35201a336e5a455da45",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(47733, 247980)\", \"evidence\": [\"table_6\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 3.52763,
      "prediction_sha256": "47e9c8a63e782724af633d53ae8398ed2f458e18d4e3d87cbffe7b1b8c79c53c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_sum(total freight revenues, none), divide(2077, #0), multiply(#1, const_100)\", \"evidence\": [\"table_2\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.05583,
      "prediction_sha256": "555e78c2a3529e6670f77d770a6166ec408342c22977a5474d5eca1f0eb683d5",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_sum(total accounts payable and othercurrent liabilities, none), divide(325, #0)\", \"evidence\": [\"table_9\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -1.1867,
      "prediction_sha256": "0edf14522bfd5c6a605bf2f1eadb13261c7290e3b244c5aea18931f3e02c905f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"subtract(52708, 53341), divide(#0, 53341), multiply(#1, const_100)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 91.16,
      "prediction_sha256": "8cb70f995f0f5506850c062c57f12884cbda5d0ee25f8350e2c4586aa8715078",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_min(allegion plc, none)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -25.05353,
      "prediction_sha256": "bbb52fc0bef0a4c649ff6ca50d018dc913f6e1249022512bc8d6d2ea4cbe8cd2",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(1050, 1401), divide(#0, 1401), multiply(#1, const_100)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19583,
      "prediction_sha256": "12afd8b9d0fb5095e4a4a3e0b8c983b84ab21b61dc54d9e5f855199359bc8162",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(88.2, 450.4)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 175.57,
      "prediction_sha256": "095b25a3218d9ef930442e3c9600fd66ae8290604754b524e12e5d195e67ef7d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama4-maverick",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_sum(united parcel service inc ., none), table_sum(s&p 500 index, none), subtract(#0, #1)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "c25d948a1e15c8405271972e3bea6b35b33702affe80641ff1ba6e761c85fd49",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(33, 19)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.47826,
      "prediction_sha256": "f7b381a4daa08c5d22bab86eadd9d17621a3d12640a5205b8d8f0b2c54483f3f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(0.068, 0.046), divide(#0, 0.046)\", \"evidence\": [\"text_7\", \"text_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 1373.658,
      "prediction_sha256": "34b384301950d1c52cf8578ff9e8a0cf309d78fe7fbbc13bcf69218f5a472c5c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(5283.3, 0.26)\", \"evidence\": [\"text_1\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.01713,
      "prediction_sha256": "8b7b032cd1b6511f5e6152ba7aeff104bb397de7f3cca67eda91169a93308fa8",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "official_match",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(22203, 21829), divide(#0, 21829)\", \"evidence\": [\"table_0\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": -0.05751,
      "prediction_sha256": "5187b0e7a65188c9bf8026c996887745b7f2537f3dc2458899c7f614a462b4f6",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(-30584, 531822)\", \"evidence\": [\"table_4\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.16662,
      "prediction_sha256": "186a5d8334440850abe6daf89d4ce44fd2dd8719749253321ea556770820c354",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(3280, 19686)\", \"evidence\": [\"table_1\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "3c1f30976d3915a17992161a1b8add9e9cf4535f5d2f3f3600a981378ef94a1d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(64, 549)\", \"evidence\": [\"table_0\", \"table_5\", \"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 28.125,
      "prediction_sha256": "27855b4526d0a2f6e5d39efca58d28e78b8908a5afbafdebd72ea862a71ae3f6",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"subtract(205, 160), divide(#0, 160), multiply(#1, const_100)\", \"evidence\": [\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.006,
      "prediction_sha256": "69454aafe9a31a1fefba766f0f1e4262cee7461f4d9da7aaab5936ad39e944b3",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(0.157, 0.151)\", \"evidence\": [\"table_6\", \"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.14162,
      "prediction_sha256": "61e378a1b40ec8991b466a85f2134864cb7d7c859d9fe7e42da1406cf2c2aa8e",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(35269, 249038)\", \"evidence\": [\"table_2\", \"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 20743588.7,
      "prediction_sha256": "cb80331e0379eedbc513e56522ef028ab64d98a5126285f6fcf436c2693aa611",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(431170, 48.11)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "7976471b5047ba645a75a90d37f604ecebee72d7d8917a8534a4dd1b8d6520fb",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_min(delphi automotive plc ( 1 ), none), add(#0, const_0)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "903e82edf5ccadf5c708f1233dcca9db8ad8f012203c6331b561887579ba54d1",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(47733, 247980)\", \"evidence\": [\"table_6\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.10042,
      "prediction_sha256": "10320c378796cf45be173f092b0e27d9acef6c9fcbd456d0f0ba56d4365da460",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(2077, 20684)\", \"evidence\": [\"table_2\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.11979,
      "prediction_sha256": "fec13bed705c00638fb1ef31169b4ee57bc586376524de83d20edc7b8e5d9553",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(325, 2713)\", \"evidence\": [\"table_5\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -0.01187,
      "prediction_sha256": "eed9e4f0d1780ffd6c308f50d10c834e106ff49e72f22be560ca0b2da8e9090d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(52708, 53341), divide(#0, 53341)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 91.16,
      "prediction_sha256": "79c1302697638705598c556c39669610d6c037dd7d582d6dc6859052d768bfaa",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_min(allegion plc, none)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -0.25054,
      "prediction_sha256": "586273693a70ef3cd26f0bade70e9fbc6eb2ade1d3d1b2df37c986802dc8cfaf",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(1050, 1401), divide(#0, 1401)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19583,
      "prediction_sha256": "1d25ed687ef044c9afe96bec9e3f3fc7af89186a239e398512665017bcc717b6",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(88.2, 450.4)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "b22ffd1fa9d3e49b04264563ce60948462f2a18785eca12441e787571f0f289f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b-bedrock",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(148.92, 135.02)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "397048eeaa57b6f98445e7c2bae7293b28787fd7c4dfcce81c699c1cd47923a5",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(33, 19)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -0.38206,
      "prediction_sha256": "8b2f739c7be6cabfc79e2a38d35911fa4c18a2e3243cacd0dc0c0c7dec1f3f27",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "official_match",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(503, 814), divide(#0, 814)\", \"evidence\": [\"text_6\", \"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 1373.658,
      "prediction_sha256": "3948da1ad233275315210a6d92fb269c591b41eb8af480cbd592f0f77c60a6dc",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(5283.3, 0.26)\", \"evidence\": [\"table_5\", \"text_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "9588fffbd894978f34a5d5a66cd339bbc13d7f8ebe85087884168de52406b61b",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "gemma-4-31b-bedrock",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"const_0\", \"evidence\": []}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": -0.05751,
      "prediction_sha256": "fcdae195b327a645a2f0b749e2314508d6b07bcdaaef79741fd3205a3537d91d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(-30584, 531822)\", \"evidence\": [\"table_4\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.16662,
      "prediction_sha256": "25bc91f658a1e5d77bd458e46e360f1241edbc40c2510bebfcec3b8ab683442f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(3280, 19686)\", \"evidence\": [\"table_1\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "f9c3aba8def2f883b873be446bf5e8190bf76430a36b1c38bbf3bfda0d77f7d2",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(64, 549)\", \"evidence\": [\"table_5\", \"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.28125,
      "prediction_sha256": "5742b676014f358263f1016bee33b6a63358f2f9c432ddc49de2dee8f1d3341f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(205, 160), divide(#0, 160)\", \"evidence\": [\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.006,
      "prediction_sha256": "f431756906e7d4ce744076757ea652714798efb336ae11b522b530cfddc0e70c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(0.157, 0.151)\", \"evidence\": [\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.14162,
      "prediction_sha256": "632896ea301284aad49a51e46695424f4082c034d45a9f4d30083f3d9059a186",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(35269, 249038)\", \"evidence\": [\"table_2\", \"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 20.74359,
      "prediction_sha256": "2c64a631638eb38fd3f2cd34f1acd50bca5039380b8ed5486d045361740abf5e",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(431170, 48.11), divide(#0, 1000000)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "8c74d7bbf143500f46e3d491634dd828faaaa8335ce433f530abcd02e868d92f",
      "status": "invalid_program",
      "evaluation_error": "unknown or empty table row",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_min(december 31 2011, none)\", \"evidence\": [\"table_0\", \"table_1\", \"table_2\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "1fa223de38ae0dd94032320f2d07df6a7b0137d92567218a4552acc367cf777b",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(47733, 247980)\", \"evidence\": [\"table_6\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.10042,
      "prediction_sha256": "6936cc479f93a6e97e412d3a4b3aaca244853463dd9048ec83c8846954f19c4f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(2077, 20684)\", \"evidence\": [\"table_2\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.11979,
      "prediction_sha256": "9948efe51d518afb83daa1edfe27b46009c14718d505396669e1355c3c785a0d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(325, 2713)\", \"evidence\": [\"table_5\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -0.01187,
      "prediction_sha256": "92a1cd5e68d68be2e156f24b2cd142bdaf6945b1e12bd7464fba7f33bc15e87a",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(52708, 53341), divide(#0, 53341)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "f5848069f9e8ae1bc202d7911a64c390351f5f0f56d98b7ab904a982d0acc380",
      "status": "invalid_program",
      "evaluation_error": "unknown or empty table row",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_min(\\\"december 31 2015\\\", none)\", \"evidence\": [\"table_1\", \"table_2\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.08224,
      "prediction_sha256": "1687f2f7b0378464ac92e057151e94ec8fbe4aa97918b77b9b51f1a715a259f0",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(107, 1301)\", \"evidence\": [\"text_25\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.174,
      "prediction_sha256": "677cd0c9b3e706c7c709207f98a86883d5798ff1a28f6adc9285bb2a4d34c475",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"add(0.174, const_0)\", \"evidence\": [\"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "4864692615370442c9e4ae11abe5a286039b7444581a1502c3a2a4c7a36221f4",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "gemma-4-31b-bedrock",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(148.92, 135.02)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "77f122ca2c260c2ef5c9f6ea1dadbeebadf86f562a26061f56014f34a16d1e21",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_2[2]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_2[2], table_2[0]), divide(#0, table_2[0])\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "ad9d7ff0e50700f5781883cde8dac66a3b8f810452aa09250ed77514df389690",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_3[2]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_3[2], table_3[0]), divide(#0, table_3[2]), multiply(const_100, #0), divide(const_100, #1)\", \"evidence\": [\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "b9a563e1d71c6640db93655c3e5130bab376decbe28f194b5cf25a4cef9e3594",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_2[1]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_2[1], table_5[1]), divide(#0, const_1000)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "67317ebfed3b65848fb3245d875ce4bd339c2aafd973d7d89428e4baf98793b4",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_0[1]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_0[1], table_1[1]), divide(#0, table_1[1]), subtract(table_0[1], table_2[1]), divide(#1, table_2[1])\", \"evidence\": [\"table_0\", \"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "de9b36d98f6f750f60cd1d624d313816df634895eb95050ea453a1b8fabb1780",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_4'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_4, table_2), divide(#0, table_1)\", \"evidence\": [\"table_1\", \"table_2\", \"table_4\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "b422a5535b4bca260fe967db8b0f6ad724f5d05119899ed94ffcbd7426f480a0",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(subtract(table_1[2012], table_1[2011]), divide(table_7[2012], const_100))\", \"evidence\": [\"table_1\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "a4d9326abdfd59461bf481a3741774e96b0aa7936a629af908954cb9b7a91302",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_5[1]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_5[1], table_5[0]), divide(#0, table_10[1])\", \"evidence\": [\"table_5\", \"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "b40cbfd7239001a356bb6ebdd1ab76061d388be6fab47f73bb4cbd0084fd6cdf",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_1[1]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_1[1], table_2[1]), divide(#0, table_1[1])\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "7750c9bc46afe072bb88b418aa5dba22fe60efce380b4f02c4af7c60bef1e3b9",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_6[2]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_6[2], table_6[1]), divide(#0, const_100)\" , \"evidence\": [\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "48cc93eefdc8d726e5c5ffae54e62d6e4c97cd4908133d997ab67267d8528b37",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_2[1]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_2[1], table_6[1]), divide(#0, table_6[1]), multiply(#1, const_100), add(#2, const_0)\", \"evidence\": [\"table_2\", \"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "b9e6257525ebeabd1b114ec67d8e09a7f4cbeec0e74c197a972920d49afa44c3",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_2[1]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_2[1], const_1000000), multiply(#0, const_51.42), add(#1, const_0)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "7bd5fd00dae70dfd0eecaa3e4e17598e22f3f089e70521749dcab8af28321779",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_1[4]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_1[4], table_1[1]), divide(#0, 100)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "45f2c721b74034002251b2e3a49bcbd2d7a13cefaefb8ceccebf9d72291ef83d",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(#2, #3), divide(#0, #1)\", \"evidence\": [\"table_6\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "70fc633b4ef5ec878b8f952329bc3149228cf963614b48bab18a9b123cc3decb",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(subtract(table_2[2013], const_0), table_sum(table_7[2013], const_0))\", \"evidence\": [\"table_1\", \"table_2\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "cb068fe996b452e0209ab3f0226be5a4696d9ede11e5915de2c995514b0d752b",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(subtract(#1, 677), subtract(#0, 2713)), greater(#0, 0)\", \"evidence\": [\"table_1\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "882c127d43da5ad87efdebb957bc507287bbc3433db670602a5149567f7b56ee",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_1[5]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_1[5], table_1[3]), divide(#0, const_100), multiply(#1, const_100), greater(#2, const_50)\" , \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "cf128505f80c8e5eb3bfcac4252901aa728d4f72204e6813e80b075c71169b30",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_1[4]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_1[4], table_1[1]), divide(#0, table_1[1])\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "8d550d56c85279d15b07a3df732c33050b715aab7b1e4aeda38b9f1e2e4a307c",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_1[1]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_1[1], table_2[1]), divide(#0, table_1[1])\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "d0c49740e995cb7e596d8bc40c1717a372b96111cfdfca9f738808dd06df6e1f",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_2[2014]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(table_2[2014], table_1[2014])\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "03c5ede5f7429d4d4b36d96cae4ba33b54a0b86ab4f2727ad91b81eb847280eb",
      "status": "invalid_program",
      "evaluation_error": "numeric operand required, received 'table_1[5]'",
      "evaluation_error_code": "unsupported_reference",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.1-8b",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(table_1[5], table_2[5]), divide(#0, table_1[0])\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "e0de6423524c54a79bda00e3f0f2a46fd1a9e778f83c0290e8ee32785f6eaa68",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(33, 19)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.47826,
      "prediction_sha256": "c4b11e0bcccae5037bbec2df41c8f0d17c30b10433f4413d2f6d3c1e652bd8a4",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(0.068, 0.046), divide(#0, 0.046)\", \"evidence\": [\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 1373.658,
      "prediction_sha256": "c0b1543b6aa0f073ac2f88645ea46e3145fa816abdae54b31d7d3e46d3fc7ea7",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(5283.3, 0.26)\", \"evidence\": [\"text_1\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -0.01684,
      "prediction_sha256": "799952c67464522745b5212449477cdf2fe9127c4558aba0ffe7d14bb47cc53e",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(21829, 22203), divide(#0, 22203)\", \"evidence\": [\"table_0\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": -0.05751,
      "prediction_sha256": "891e724215d6c26678603e0802f06ec27cc9e1bda9b433b81074daf373573585",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(-30584, 531822)\", \"evidence\": [\"table_4\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.16662,
      "prediction_sha256": "a81eaf128653d6ec5de873211c5c0e4065d4e6c0193074104c0249bf2b1a72d0",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(3280, 19686)\", \"evidence\": [\"table_1\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "da125417c0ddc9caa7ce699a336b317b50477de6b01930874f94fe0a7b7b67ed",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(64, 549)\", \"evidence\": [\"table_0\", \"table_5\", \"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.28125,
      "prediction_sha256": "6335fe2a47729dd589800a88c0f4959eb9075f0cb85c45d2fd383a3870c67795",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(205, 160), divide(#0, 160)\", \"evidence\": [\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.006,
      "prediction_sha256": "09563d0089106df7ce270ea83f5bce4ab195c2e46ddc0db3658398ebe04fbe5b",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(0.157, 0.151)\", \"evidence\": [\"table_6\", \"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.14162,
      "prediction_sha256": "9f56c6d22f1585b8c8042c8450053be5a9cc3e951e18988818352fe4d98ec188",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(35269, 249038)\", \"evidence\": [\"table_2\", \"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 20743588.7,
      "prediction_sha256": "937756206728d9d2ac841befe93f26bcfc37812ead03cfa17dac9c7bad0224cd",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(431170, 48.11)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "05705549294cbcc8fa0e9a41199d1e993360a034f3a0c2e1d35c67336f0e5bfc",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_min(delphi automotive plc ( 1 ), none), add(#0, const_0)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "14de97bb763ca127609754685a3bf9536a7a50a456109bc6dec5d672f4dd66cb",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(47733, 247980)\", \"evidence\": [\"table_6\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.10042,
      "prediction_sha256": "1c55180d0bc7b14b5124a1ba14c14eefee2c9c58143b8e0e34eda158974b3c86",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(2077, 20684)\", \"evidence\": [\"table_2\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.11979,
      "prediction_sha256": "fd2e4866760e32940cb4064114cd20450a927b48cc41019ee91583afa7d89df4",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(325, 2713)\", \"evidence\": [\"table_5\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -1.1867,
      "prediction_sha256": "6a90593274a1dc7621d5a49c1815a4f7c9330a8e389774ac2a9a58f102e04976",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"subtract(52708, 53341), divide(#0, 53341), multiply(#1, const_100)\", \"evidence\": [\"table_1\", \"text_19\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 91.16,
      "prediction_sha256": "e34c00550018f06172087e81254c00c5fafbe03e3eee151df48a248b1179e00b",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_min(allegion plc, none)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -0.25054,
      "prediction_sha256": "b5defed702f54e836e171915480960e1f66b44cd8f1a3df8f2f041bc274d7652",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(1050, 1401), divide(#0, 1401)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19583,
      "prediction_sha256": "5b417b5ea2a3a35cf65a5acffdfa5211d688d715fce0debe7ddd45ffdcf40d1c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(88.2, 450.4)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "7c91f14961ce1a766efd971729cb7e6a852ae4b8359ffa663765be7d38a37a47",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "llama-3.3-70b",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(148.92, 135.02)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "5b1ea8681a3671f339683f8397114d42266c771b9d9d89c84a18460d060e1425",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(33, 19)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -0.38206,
      "prediction_sha256": "fe651d8af1c1fd209f49f71cbd69673f589da72fb783d6cd6c2521b3fbee6f90",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "official_match",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n\"program\": \"subtract(503, 814), divide(#0, 814)\",\n\"evidence\": [\n\"text_6\",\n\"text_7\"\n]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 1373.658,
      "prediction_sha256": "fef97ed301708e110e07a175a846d4c60c67259ce4a722bb38d9dd045a278eba",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n\"program\": \"multiply(5283.3, 0.26)\",\n\"evidence\": [\n\"text_1\",\n\"table_5\"\n]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.01713,
      "prediction_sha256": "df81ee5b482f74e1b9b9d7030a932e17479fcba5948741669946b101cfd49fd5",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "official_match",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n\"program\": \"subtract(22203, 21829), divide(#0, 21829)\",\n\"evidence\": [\n\"table_0\",\n\"table_3\"\n]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": -0.05751,
      "prediction_sha256": "c5d9770b8a19313f51df10414a97289c6f14fc5185fcbfe94bd3067f4a7ac63f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n\"program\": \"divide(-30584, 531822)\",\n\"evidence\": [\n\"table_4\",\n\"table_5\"\n]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.16662,
      "prediction_sha256": "6248cdd9ea974aaa317d5ae7e250a5900d5fe4fb01b9c724b30de22d8844435f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(3280, 19686)\", \"evidence\": [\"table_1\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "32ccac8cc205ab7c307e87c6c1a1c86d48c6726065f046c7c7634b163428eeeb",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"divide(64, 549)\",\n  \"evidence\": [\n    \"table_5\",\n    \"table_10\"\n  ]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.28125,
      "prediction_sha256": "a19d7ad1f2026704c4164c9353281cf5821fef9df7b0a1e4b4293a0128592201",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n\"program\": \"subtract(205, 160), divide(#0, 160)\",\n\"evidence\": [\n\"text_31\"\n]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.6,
      "prediction_sha256": "aa41bcaaa5f29649fe64836cd561bffa11ccf389893ba7dd47c21cc0cee69fa2",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "scale_difference_candidate",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\n\"program\": \"subtract(15.7, 15.1)\",\n\"evidence\": [\n\"table_6\"\n]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.14162,
      "prediction_sha256": "cb9c50af0d785929ceac1b73d0a39ec470a00bbbb0a151523674929ed824d8bc",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n\"program\": \"divide(35269, 249038)\",\n\"evidence\": [\n\"table_2\",\n\"table_6\"\n]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 20.74359,
      "prediction_sha256": "397aefe422b4c6a5001fc6802ea96161d90fd0b3119d9661cfdc2f85ae95ede3",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"multiply(431170, 48.11), divide(#0, const_1000000)\",\n  \"evidence\": [\n    \"table_2\"\n  ]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": -10.73,
      "prediction_sha256": "1850d6ef6b2a64096489f8db4c408012b4f158a65586f0c4b322042b90926c4f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"subtract(89.27, 100.00)\",\n  \"evidence\": [\n    \"table_3\"\n  ]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "4b60f8775289dc60896b5f10e5e46ba1c9a98364fe1cec3dad49c0abab84349c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n\"program\": \"divide(47733, 247980)\",\n\"evidence\": [\n\"table_6\",\n\"table_7\"\n]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 10.04158,
      "prediction_sha256": "e5eb11b954f74d3414d90dc886a846282ad429b1fb28591cf3b82de976644e56",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(2077, 20684), multiply(#0, const_100)\", \"evidence\": [\"table_2\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.11979,
      "prediction_sha256": "d5d5fd00b9242fcd1ce2af81da3d35788bbcdb001a1af2525d5c91f18d05a8a8",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(325, 2713)\", \"evidence\": [\"table_5\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": -0.01187,
      "prediction_sha256": "d974e03e960cf6daff830ce7f846b316fe5487ffa7a54c2d2f6ed2612b3d5d41",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n\"program\": \"subtract(52708, 53341), divide(#0, 53341)\",\n\"evidence\": [\n\"table_1\"\n]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "3a4a711873ab55ad00c29dcfbc2f408d77553c69121d1dc45b79f1d3449c4e18",
      "status": "invalid_program",
      "evaluation_error": "unknown or empty table row",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "invalid_program",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n  \"program\": \"table_min(december 31 2015, none)\",\n  \"evidence\": [\n    \"table_1\",\n    \"table_2\",\n    \"table_3\"\n  ]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.08224,
      "prediction_sha256": "3d0a31ce625b4ee523141faa12575d5551a97493564257e6ddd277afbbd26385",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n\"program\": \"divide(107, 1301)\",\n\"evidence\": [\n\"text_25\",\n\"table_3\"\n]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19583,
      "prediction_sha256": "25eedace7ea57f9db8e97d0909328ad671aae3ae3587f5a7226c60cd0b48c0f3",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n\"program\": \"divide(88.2, 450.4)\",\n\"evidence\": [\n\"table_2\",\n\"table_1\"\n]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "6372ca5964b8487879d77bb02dd8ee3b82d0850a1ea1d3cfb80dc801abb92bd6",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "qwen3.6-27b",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\n\"program\": \"subtract(148.92, 135.02)\",\n\"evidence\": [\n\"table_1\",\n\"table_2\"\n]\n}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "a40dd62afc78257ac66738747765ad19c3ac49b09c0b9e510bb08e3945e186bc",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(33, 19)\", \"evidence\": [\"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 38.20639,
      "prediction_sha256": "e8ca1669cc872fc6aaa64cdeb792c7f0d3a76a85dad724de94de84ff938e2845",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(814, 503), divide(#0, 814), multiply(#1, const_100)\", \"evidence\": [\"table_2\", \"text_7\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "6fd6dcfbf2d88251f96776f689bbf7e59b19203113831ed316c270c9c4a42188",
      "status": "invalid_program",
      "evaluation_error": "Expecting value: line 1 column 1 (char 0)",
      "evaluation_error_code": "invalid_json",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "exaone-3.5-32b",
      "category": "invalid_program",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(table_2[2017], multiply(table_2[2017], const_0.26)), evidence\": [\"text_1\", \"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 98.28668,
      "prediction_sha256": "e927a511629d60bd9e61ba17a20f5e1f71ba3f08ca57e587c20a16c3937ecdc8",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(21829, 249), subtract(#0, 125), divide(#1, 21829), multiply(#2, 100)\", \"evidence\": [\"table_0\", \"table_1\", \"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "b29cc52e2cfa28a9523d1dff85016166bc9d1c0e7b4270b5aa9a98c71d5f650e",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(#4, #2), divide(#0, #4)\", \"evidence\": [\"table_2\", \"table_4\", \"table_1\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "09a0f334e5edd307446eb261e3cb324f76e5035a880b8172820bd16a7b13f221",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(#0, const_0) , divide(#1, #0) , multiply(#2, const_100)\", \"evidence\": [\"table_1\", \"table_7\", \"table_1\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "b7208babdcb264208924310c718884bd04048cd612836bfd5d10adeee06e9c2f",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"divide(#0, table_10[revised])\", \"evidence\": [\"table_10\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 28.125,
      "prediction_sha256": "3f9dfcc39512d6617407755458813855d43828e675c085be04e59180adec4b4d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\"program\": \"subtract(205, 160), divide(#0, 160), multiply(#1, const_100)\", \"evidence\": [\"text_31\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.6,
      "prediction_sha256": "f8b554e48de40998e69e19df2017d0b697f042b3ac4151e9886319cdc83fddbc",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "scale_difference_candidate",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\"program\": \"subtract(15.7, 15.1), add(#0, const_0)\", \"evidence\": [\"table_6\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "eccd31f6c35e82966c557c211f8fa69ca3ae708d6be3c38da92d34ad8acaae93",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(#0, #1), divide(#2, #0), multiply(#3, const_100)\", \"evidence\": [\"table_2\", \"table_1\", \"table_6\", \"table_0\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "ceed699d618a314a22ca657e486d47924c47aa565814ea800d2878a65e285bd4",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(#0, const_0)\", \"evidence\": [\"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "1b5e25058cf728362d69c44aeba577a2d65f00f2cc2427a02e3811b09515d00b",
      "status": "invalid_program",
      "evaluation_error": "unknown or empty table row",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"table_min(november 17 2011, none)\", \"evidence\": [\"table_1\", \"table_2\", \"table_3\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "5b1317e450e67d27880305961e1a63213bef1d7a174028e536c666e03ab6d443",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"divide(#0, #1)\", \"evidence\": [\"table_6\", \"table_7\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 895.85941,
      "prediction_sha256": "1ca3b5527d1aeac8e5b0d404733cee14482d9e88efcf1f9a0a42cfc56d4ffefb",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(20684, 2077), divide(#0, 2077), multiply(#1, const_100)\", \"evidence\": [\"table_1\", \"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "5a4370ba93f0b1be694b384f39a2b8a09441eb2a8a0794a36c506d0ffb90d985",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(#0, #1), divide(#2, #0)\", \"evidence\": [\"table_1\", \"table_5\", \"table_9\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -1.1867,
      "prediction_sha256": "88285f2d12b870ef114b2b6796220286cd81a7c56296964acc11ea935e119539",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "```json\n{\"program\": \"subtract(52708, 53341), divide(#0, 53341), multiply(#1, const_100)\", \"evidence\": [\"table_1\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 91.16,
      "prediction_sha256": "e85b6570bd274cc031c26f324388eb9f059a7622313f1474dec76148e05c547e",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"table_min(allegion plc, none)\", \"evidence\": [\"table_1\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -25.05353,
      "prediction_sha256": "3fcb3195fb134413ec766f7836b180ed510ac2c7fb20a47e86a2158d58d89805",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(1050, 1401), divide(#0, 1401), multiply(#1, 100)\", \"evidence\": [\"table_1\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "5aedf72eb505184c659c1d100e7f7f5f70e2bd017279619f7bd671476c447bd7",
      "status": "invalid_program",
      "evaluation_error": "forward reference",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "invalid_program",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"divide(#0, table_1[2014])\", \"evidence\": [\"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "10d7ee1a30ae21e04ff64a8415a221dbcee583635babb6e08626dd660f9ecaa7",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [
        "json_fence"
      ],
      "model": "exaone-3.5-32b",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "```json\n{\"program\": \"subtract(148.92, 135.02), add(#0, const_0)\", \"evidence\": [\"table_1\", \"table_2\"]}\n```",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "7eac532d9735ff2ad28d369fcb01ed3f998ebd9c7762cf639fe423da24ba556c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(33, 19)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -38.20639,
      "prediction_sha256": "c43d582aa76bf622ac7d30b736c40385b5aea48e3da525a03bd027a328fdd5a7",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "scale_difference_candidate",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"subtract(503, 814), divide(#0, 814), multiply(#1, const_100)\", \"evidence\": [\"text_6\", \"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 447.408,
      "prediction_sha256": "2ffa6ddec8baea753740446e66e3ba03d6e1247f56103d3107d1d4abba5ea224",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(1720.8, 0.26)\", \"evidence\": [\"table_2\", \"text_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 98.31554,
      "prediction_sha256": "bf1f38070cd65e43adacc2fe3019bb5f756e816ff46a506ba548e9c8d8f75c16",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(21829, 22203), multiply(#0, const_100)\", \"evidence\": [\"table_0\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": -0.05751,
      "prediction_sha256": "b047d89006ede5a5e6fa4a43830ab3eca00c0f56b6e805833a76d6d418cc5d3f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(-30584, 531822)\", \"evidence\": [\"table_4\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 16.66159,
      "prediction_sha256": "c37ea536b5f19cddad1e6a80a574c9047916ad1c05d1a9c8a9e9c1a80340c141",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"divide(3280, 19686), multiply(#0, const_100)\", \"evidence\": [\"table_1\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "99057e1017164e328319c68ffa14ed87f7947b127888e9327829e6ae29e9c1ad",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(64, 549)\", \"evidence\": [\"table_5\", \"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 28.125,
      "prediction_sha256": "0e157d339cbe8e7049f8f48c7fd311101f6bfb1d37e0422dcd8017af1ec2531a",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"subtract(205, 160), divide(#0, 160), multiply(#1, const_100)\", \"evidence\": [\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.6,
      "prediction_sha256": "9eb83e3ef2eb00415b11b0a7024a7c636c67e616f999ffb77c0f95574035861e",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "scale_difference_candidate",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"subtract(15.7, 15.1)\", \"evidence\": [\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 14.1621,
      "prediction_sha256": "300b7a162754d38d4fea4de691acbb4549020a6b599a40a2de97a03cf915b5b2",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"divide(35269, 249038), multiply(#0, const_100)\", \"evidence\": [\"table_0\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 20.74359,
      "prediction_sha256": "7bf975230c83292a0860c1c6f3071678994926aff4064746f61325f086bbec5b",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(431170, 48.11), divide(#0, const_1000000)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": -0.1073,
      "prediction_sha256": "1eeb2ab92adade6c51e7743d087730a2880eb28589de989f820cac5f23827e08",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "scale_difference_candidate",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [
        "multiply_100"
      ],
      "output_text": "{\"program\": \"subtract(89.27, 100), divide(#0, const_100)\", \"evidence\": [\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "08174affe69657aca66258d2c39747193729006ef16d75d004468117408188aa",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(47733, 247980)\", \"evidence\": [\"table_6\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 10.04158,
      "prediction_sha256": "74f627ac985fb9439a3a873492a006b9a238c0f01c4fbe74bcc20d460fe887b5",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(2077, 20684), multiply(#0, const_100)\", \"evidence\": [\"table_2\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 11.97936,
      "prediction_sha256": "771126abe1ed9cb52a792abd0393366cae87fbb30e6c27fab215a1ed6aac786e",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"divide(325, 2713), multiply(#0, const_100)\", \"evidence\": [\"table_5\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -1.1867,
      "prediction_sha256": "beafbeafdcb643904bef6931a2386ef370f713e933082cc38108024a9e57195e",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"subtract(52708, 53341), divide(#0, 53341), multiply(#1, const_100)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 99.9,
      "prediction_sha256": "61731d2b7d45d4eeb4d010c686015e3a424e501007774e226925794c479ca78d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"add(99.90, const_0)\", \"evidence\": [\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.08224,
      "prediction_sha256": "10d0d00e4ea15d0fb915dbad300f0774b08ab5d6145be38f344a812e2d48d076",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(107, 1301)\", \"evidence\": [\"text_25\", \"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 19.58259,
      "prediction_sha256": "98f794a5a59b86dde78ce105c91a71adbc6b27c7a9f84670366729ebf5a54bfd",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"divide(88.2, 450.4), multiply(#0, const_100)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "dc39ce0c6813cdb50a8a1d8f5981af69bd86d61b67f07a896461550627371098",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.3",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(148.92, 135.02)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MRO/2009/page_139.pdf-3",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 14.0,
      "prediction_sha256": "36ec54790f0d27eb27466c193ce1044d910cf8c3a23b044f8d6ccc4a4a3cbb62",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(33, 19)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-CB/2010/page_88.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.61829,
      "prediction_sha256": "b2e99ca8380abbdef0377ae883dfe23d07852e0c66ea3e66494661d99cd05cf5",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(814, 503), divide(#0, 503)\", \"evidence\": [\"text_6\", \"text_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FBHS/2017/page_23.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 447.408,
      "prediction_sha256": "950378a1aa5092b9d006472f7dcd9a0a6e4ca130f7cf070836aba840e1f5115a",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(1720.8, const_26), divide(#0, const_100)\", \"evidence\": [\"text_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-VRTX/2003/page_71.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 21829.0,
      "prediction_sha256": "bd03301e541e638e0dcea1ce921a4874df12cea35a8e97c6f7a30bb7d51cdf70",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"add(21829, const_0)\", \"evidence\": [\"table_0\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-AMT/2005/page_54.pdf-1",
      "correct": true,
      "evidence_exact": false,
      "execution_result": -0.05751,
      "prediction_sha256": "635b96935f139fd4f8fd37ecee9bb09a074c98a32836e0c6853e5b69e12dac8f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(-30584, 531822)\", \"evidence\": [\"table_4\", \"table_5\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 16.66159,
      "prediction_sha256": "69b7a024adf08278a6f5be1d0a6d6ca6efe8eb3aff0aebbb5e8afc3f934b163b",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"divide(3280, 19686), multiply(#0, const_100)\", \"evidence\": [\"table_1\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_60.pdf-2",
      "correct": true,
      "evidence_exact": false,
      "execution_result": 0.11658,
      "prediction_sha256": "63c45bf864b89385d104eb62a9da73e72d235f96ee44585ac885c2ed0d3a5d2e",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(64, 549)\", \"evidence\": [\"table_5\", \"table_10\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PM/2018/page_31.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.28125,
      "prediction_sha256": "3d27744be2d18ced04f234ec8bc51eb3c624db5432cd7ab77acd849346f6a831",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(205, 160), divide(#0, 160)\", \"evidence\": [\"text_31\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-MAS/2018/page_35.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": 0.6,
      "prediction_sha256": "39d1eb437245b92c989a0858f20eb1ea6e70680852126c4b83e86bd1a7e5961c",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "scale_difference_candidate",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"subtract(15.7, 15.1)\", \"evidence\": [\"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-FIS/2007/page_94.pdf-4",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.14162,
      "prediction_sha256": "8c3596d20ba138b7584edb5efd44f57647eae99bec3745eb6d49fd512d9440e9",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(35269, 249038)\", \"evidence\": [\"table_2\", \"table_6\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-BLL/2007/page_35.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 20.74359,
      "prediction_sha256": "3388bc6ec2bb84b79ad82d638d9384682ea068d02365daa3461f1d15d91f5a82",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"multiply(431170, 48.11), divide(#0, const_1000000)\", \"evidence\": [\"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APTV/2014/page_49.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": null,
      "prediction_sha256": "242f77987735c05c4f558e166cd1fc441f82657ce7b44b5e8aedfc1a32b06c19",
      "status": "invalid_program",
      "evaluation_error": "invalid program syntax",
      "evaluation_error_code": "invalid_program",
      "strict_correct": false,
      "strict_status": "invalid_program",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "invalid_program",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_min(automotive supplier peer group ( 3 ), none)\", \"evidence\": [\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-JPM/2018/page_110.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19249,
      "prediction_sha256": "b7f2458fd532098bdaa9be0e83cad9fe0ff4bb55f08cb891c591929af91668a0",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(47733, 247980)\", \"evidence\": [\"table_6\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2013/page_54.pdf-3",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 0.10042,
      "prediction_sha256": "6cf498676266c3507ba1a13a3608ed755ea1c5e30293cd618069494d73cb0427",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(2077, 20684)\", \"evidence\": [\"table_2\", \"table_7\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UNP/2011/page_76.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.11979,
      "prediction_sha256": "35ab8cd3659717adf759cfecd42e647ea59c95679de433b4acd3acb770aa468d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(325, 2713)\", \"evidence\": [\"table_5\", \"table_9\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-INTC/2013/page_33.pdf-1",
      "correct": false,
      "evidence_exact": true,
      "execution_result": -1.1867,
      "prediction_sha256": "bdba5f1bb495001ca0e844a2e73a02ff3aa80d74ec2f64c005e3cb20cd5e4687",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "scale_difference_candidate",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [
        "divide_100"
      ],
      "output_text": "{\"program\": \"subtract(52708, 53341), divide(#0, 53341), multiply(#1, const_100)\", \"evidence\": [\"table_1\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-ALLE/2016/page_43.pdf-1",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 99.9,
      "prediction_sha256": "d67fd5e461628a6ba15aa6e5f2b756b6ac88eb76249562316811ea408452b01d",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "ambiguous",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"table_min(s&p 400 capital goods, none)\", \"evidence\": [\"table_3\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-PNC/2009/page_41.pdf-2",
      "correct": false,
      "evidence_exact": false,
      "execution_result": 1.87719,
      "prediction_sha256": "7c81191ee13a23fdaa17aa01542b4e577bc3ab6d228462bb2f2bd4bd21c68aed",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": false,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "other_numeric_or_interpretation_mismatch",
      "document_quality": "annotation_issue",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"add(5400, 300), divide(107, #0), multiply(#1, const_100)\", \"evidence\": [\"text_24\", \"text_25\", \"text_26\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-APD/2014/page_39.pdf-1",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 0.19583,
      "prediction_sha256": "6a814978c9b7d61f7cab1ed58e4e31c8dc3b1c402dac075d8fbafc211fbeae96",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "official_match",
      "document_quality": "clear",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"divide(88.2, 450.4)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    },
    {
      "id": "finqa-dev-UPS/2006/page_32.pdf-3",
      "correct": true,
      "evidence_exact": true,
      "execution_result": 13.9,
      "prediction_sha256": "aa6ddf108ce24fa71dfb3307a7ccac0478c94103e62600c791116f981434694f",
      "status": "scored",
      "evidence_valid": true,
      "strict_correct": true,
      "strict_status": "scored",
      "normalizations": [],
      "model": "grok-4.6",
      "category": "official_match",
      "document_quality": "unit_convention",
      "diagnostic_scale_candidates": [],
      "output_text": "{\"program\": \"subtract(148.92, 135.02)\", \"evidence\": [\"table_1\", \"table_2\"]}",
      "review_limit": "Scale candidates are diagnostic, not validated correct answers. Other mismatches are not necessarily model errors; consult question audit."
    }
  ]
}