{
  "evaluation_id": "invoice-extraction-v0.1-deepseek-v4-flash-supplement",
  "completed_at": "2026-08-10T20:14:50.090Z",
  "status": "supplemental_candidate_scored_fixture_only",
  "headline": "DeepSeek V4 Flash was the cheapest and fastest hosted candidate, but returned only 6 of 12 records exactly under the locked contract",
  "relationship_to_original": {
    "type": "post-result_supplement",
    "original_evaluation_id": "invoice-extraction-v0.1",
    "original_result_unchanged": true,
    "note": "DeepSeek V4 Flash was registered after the original Luna, Sol, and local Gemma4 results were known. Its run is comparable on the locked cases and scorer, but it was not part of the original candidate set."
  },
  "decision": {
    "hosted_default_candidate": "gpt-5.6-luna",
    "reason": "Luna retained 12/12 whole-record exact match. DeepSeek V4 Flash was cheaper and faster, but failed the locked field contract on six records, so its lower variable cost does not justify routing this workload to it under this prompt.",
    "deepseek_next_step": "A stricter DeepSeek-specific schema or function-call protocol may address the repeated key-name failures, but that would be a new preregistered evaluation rather than a reinterpretation of this run.",
    "production_decision": "Not established. All results use 12 synthetic text records and exclude OCR, production variation, and an operational error budget."
  },
  "result": {
    "candidate_id": "deepseek-v4-flash-hosted",
    "model": "deepseek-v4-flash",
    "provider": "DeepSeek API",
    "route": "low_cost_hosted",
    "thinking": "disabled",
    "output_mode": "JSON object",
    "whole_records_correct": 6,
    "whole_records_total": 12,
    "whole_record_exact_match": 0.5,
    "locked_schema_contract_validity": 0.5833333333333334,
    "json_syntax_validity": 1,
    "errors": 0,
    "automatic_retries": 0,
    "input_tokens": 2148,
    "cached_input_tokens": 0,
    "cache_miss_input_tokens": 2148,
    "output_tokens": 561,
    "measured_cost_usd": 0.0004578,
    "median_latency_ms": 283,
    "minimum_latency_ms": 266,
    "maximum_latency_ms": 348,
    "run_id": "deepseek-v4-flash-hosted-2026-08-10T20-14-37.060Z",
    "artifacts": {
      "manifest": "runs/deepseek-v4-flash-hosted-2026-08-10T20-14-37.060Z/manifest.json",
      "predictions": "runs/deepseek-v4-flash-hosted-2026-08-10T20-14-37.060Z/predictions.jsonl",
      "raw_records": "runs/deepseek-v4-flash-hosted-2026-08-10T20-14-37.060Z/raw-records.jsonl",
      "score": "runs/deepseek-v4-flash-hosted-2026-08-10T20-14-37.060Z/score.json"
    }
  },
  "failure_analysis": {
    "wrong_amount_property_name": {
      "count": 5,
      "case_ids": ["inv-002", "inv-004", "inv-007", "inv-008", "inv-009"],
      "expected_property": "total_amount",
      "received_properties": ["amount_due", "amount_payable"],
      "note": "Every value was numerically correct, but the locked integration contract requires the exact total_amount key. JSON object mode guaranteed parseable JSON, not adherence to this JSON schema."
    },
    "wrong_supplier_variant": {
      "count": 1,
      "case_ids": ["inv-005"],
      "expected": "AOZORA TRADING CO., LTD.",
      "received": "株式会社 青空商事",
      "note": "The model selected the Japanese legal-name rendering instead of the expected English trading name shown alongside it. The remaining four fields were correct."
    }
  },
  "comparison_to_original_candidates": {
    "deepseek_cost_saving_vs_luna_percent": 62.71,
    "deepseek_cost_saving_vs_sol_percent": 98.51,
    "luna_to_deepseek_cost_ratio": 2.68,
    "sol_to_deepseek_cost_ratio": 67.05,
    "deepseek_median_latency_reduction_vs_luna_percent": 79.11,
    "deepseek_median_latency_reduction_vs_sol_percent": 82.04,
    "quality_tradeoff": "DeepSeek produced 6 fewer exact records than Luna or Sol and 5 fewer than local Gemma4.",
    "comparison_boundary": "Provider tokenizers differ and the supplemental run occurred later rather than simultaneously. Cost and latency comparisons are descriptive measurements from this fixture, not universal model characteristics."
  },
  "zero_spend_infrastructure_attempt": {
    "run_id": "deepseek-v4-flash-hosted-2026-08-10T20-14-02.831Z",
    "outcome": "All 12 requests failed locally with fetch failed before reaching the provider",
    "tokens": 0,
    "measured_cost_usd": 0,
    "retained": true,
    "artifacts": {
      "manifest": "runs/deepseek-v4-flash-hosted-2026-08-10T20-14-02.831Z/manifest.json",
      "raw_records": "runs/deepseek-v4-flash-hosted-2026-08-10T20-14-02.831Z/raw-records.jsonl",
      "score": "runs/deepseek-v4-flash-hosted-2026-08-10T20-14-02.831Z/score.json"
    }
  },
  "prices_locked_2026_08_10_usd_per_million_tokens": {
    "cache_hit_input": 0.0028,
    "cache_miss_input": 0.14,
    "output": 0.28,
    "source": "https://api-docs.deepseek.com/quick_start/pricing"
  },
  "boundaries": [
    "Twelve entirely synthetic text records test the harness; they do not establish production reliability.",
    "The fixture does not test image rendering or OCR quality.",
    "JSON syntax validity is distinct from validity against the locked five-property contract.",
    "The paid run used non-thinking mode, temperature 0, one attempt per case, and no automatic retries.",
    "The failed local-network attempt is retained separately and incurred zero measured token cost.",
    "No production error budget or pass threshold has been set."
  ]
}