{
  "evaluation_id": "invoice-extraction-v0.1-deepseek-v4-flash-supplement",
  "preregistered_at": "2026-08-10T20:11:24.8863771Z",
  "repository_commit_before_registration": "9f6c7500dac36587b7f0e0376b882ee1e3c850f5",
  "status": "locked_before_first_deepseek_scored_model_call",
  "relationship_to_original": {
    "type": "post-result_supplemental_candidate",
    "original_evaluation_id": "invoice-extraction-v0.1",
    "original_preregistration": "preregistration.json",
    "reason": "The DeepSeek V4 Flash candidate and wallet access were offered after the original three-candidate evaluation had completed. It is therefore reported as a supplemental run, not retroactively inserted into the original preregistration."
  },
  "scope": {
    "task": "Extract supplier_name, invoice_number, issue_date, currency, and total_amount from OCR-like invoice text",
    "modality": "text only",
    "case_count": 12,
    "data_boundary": "The same entirely synthetic records used in invoice-extraction-v0.1; no image OCR and no production supplier data",
    "production_readiness_claim": false
  },
  "locked_inputs": {
    "cases": {
      "path": "cases.jsonl",
      "sha256": "ff56f65b12804a96eda91b8439ca48d0d10fd16ea355abe1668b44bc69d0be9a"
    },
    "scorer": {
      "path": "score.mjs",
      "sha256": "3048b9f62bf58b5d01df1a0689dfe90cea2a9bc9206dfb9ccabf286d146b785a"
    },
    "protocol": {
      "path": "protocol.json",
      "sha256": "3224775ed33c3d0ef615a381640bf65937b8a6fd69ac6ffc69b6503f0906372d"
    },
    "original_preregistration": {
      "path": "preregistration.json",
      "sha256": "30432d4d0089c896159421fe587c3cec547d19934084e7a0341accd0dc263640"
    },
    "runner": {
      "path": "run-eval.mjs",
      "sha256": "4f9ce678933d749ebecb9b4fe8cd6720f9b236332ca86e7a516270b4000b3102"
    }
  },
  "prompt": {
    "system": "You extract invoice fields. Return only the five fields required by the supplied JSON schema. Use the supplier's trading name shown on the invoice, preserve the invoice number exactly, convert the issue date to YYYY-MM-DD, use the three-letter ISO 4217 currency code, and return the final amount payable including tax as a number. Do not infer a missing value.",
    "user_template": "Extract the required fields from this invoice text:\n\n<invoice>\n{{INPUT}}\n</invoice>",
    "schema": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "supplier_name": { "type": "string" },
        "invoice_number": { "type": "string" },
        "issue_date": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" },
        "currency": { "type": "string", "pattern": "^[A-Z]{3}$" },
        "total_amount": { "type": "number" }
      },
      "required": [
        "supplier_name",
        "invoice_number",
        "issue_date",
        "currency",
        "total_amount"
      ]
    }
  },
  "scored_candidates": [
    {
      "candidate_id": "deepseek-v4-flash-hosted",
      "route": "low_cost_hosted",
      "provider": "DeepSeek API",
      "endpoint": "https://api.deepseek.com/chat/completions",
      "model": "deepseek-v4-flash",
      "thinking": "disabled",
      "temperature": 0,
      "structured_output": "JSON object mode; the locked field contract is stated in the prompt and checked by the deterministic scorer",
      "max_output_tokens": 256,
      "price_usd_per_million_tokens": {
        "input": 0.14,
        "cached_input": 0.0028,
        "output": 0.28
      },
      "cost_guard": {
        "maximum_billable_input_tokens_per_request": 2000,
        "maximum_billable_output_tokens_per_request": 256,
        "maximum_request_cost_usd": 0.00035168,
        "planned_maximum_for_12_requests_usd": 0.00422016,
        "basis": "A deliberately conservative 2,000-token input allowance per locked sub-220-byte invoice, priced entirely as cache misses, plus the configured 256-token output maximum."
      }
    }
  ],
  "execution": {
    "candidate_order": ["deepseek-v4-flash-hosted"],
    "output_directory": "supplements/deepseek-v4-flash/runs",
    "attempts_per_case": 1,
    "automatic_retries": 0,
    "transport_errors": "Retained and scored as unsuccessful predictions",
    "raw_artifacts": "Retain sanitised request bodies, complete provider responses, parsed outputs, timing, token usage, and every error; never persist the authorization header or API key",
    "hosted_cost_ceiling_usd": 0.01,
    "stop_condition": "Reserve the preregistered maximum request cost before each call and stop before the reserve would take cumulative spend above $0.01"
  },
  "metrics": {
    "primary": "whole_record_exact_match",
    "secondary": [
      "field_exact_match",
      "schema_validity",
      "missing_prediction_count",
      "latency_ms",
      "measured_token_cost_usd"
    ],
    "pass_threshold": "Unset: this 12-case synthetic fixture cannot establish production readiness"
  },
  "comparison_rule": {
    "quality": "Apply the original deterministic scorer without modification and report whole-record exact match first.",
    "cost": "Report measured DeepSeek cache-hit, cache-miss, and output tokens at the price locked here. Compare with the original candidates descriptively because this is a later supplemental run.",
    "routing": "Do not replace the original hosted recommendation merely because a later candidate ties on 12 synthetic cases. Treat a tie as evidence that DeepSeek V4 Flash merits a larger representative evaluation."
  },
  "sources_checked_2026_08_10": {
    "models_and_pricing": "https://api-docs.deepseek.com/quick_start/pricing",
    "chat_completion_and_json_output": "https://api-docs.deepseek.com/api/create-chat-completion",
    "thinking_mode_toggle": "https://api-docs.deepseek.com/guides/thinking_mode"
  },
  "not_claimed": [
    "DeepSeek V4 Flash has not been run on this fixture at registration time",
    "The supplemental run was not part of the original three-candidate preregistration",
    "The fixture does not validate image OCR performance",
    "The fixture is too small and synthetic to establish production readiness"
  ]
}