{
  "evaluation_id": "invoice-extraction-v0.1",
  "preregistered_at": "2026-08-10T19:05:01.6796344Z",
  "repository_commit_before_runs": "df8d971a29c19ba7bfd8504faa7d26ee602c9579",
  "status": "locked_before_first_scored_model_call",
  "scope": {
    "task": "Extract supplier_name, invoice_number, issue_date, currency, and total_amount from OCR-like invoice text",
    "modality": "text only",
    "case_count": 12,
    "data_boundary": "Entirely synthetic records; no image OCR and no production supplier data",
    "production_readiness_claim": false
  },
  "locked_inputs": {
    "cases": {
      "path": "cases.jsonl",
      "sha256": "ff56f65b12804a96eda91b8439ca48d0d10fd16ea355abe1668b44bc69d0be9a"
    },
    "scorer": {
      "path": "score.mjs",
      "sha256": "3048b9f62bf58b5d01df1a0689dfe90cea2a9bc9206dfb9ccabf286d146b785a"
    },
    "protocol": {
      "path": "protocol.json",
      "sha256": "3224775ed33c3d0ef615a381640bf65937b8a6fd69ac6ffc69b6503f0906372d"
    }
  },
  "prompt": {
    "system": "You extract invoice fields. Return only the five fields required by the supplied JSON schema. Use the supplier's trading name shown on the invoice, preserve the invoice number exactly, convert the issue date to YYYY-MM-DD, use the three-letter ISO 4217 currency code, and return the final amount payable including tax as a number. Do not infer a missing value.",
    "user_template": "Extract the required fields from this invoice text:\n\n<invoice>\n{{INPUT}}\n</invoice>",
    "schema": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "supplier_name": { "type": "string" },
        "invoice_number": { "type": "string" },
        "issue_date": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" },
        "currency": { "type": "string", "pattern": "^[A-Z]{3}$" },
        "total_amount": { "type": "number" }
      },
      "required": [
        "supplier_name",
        "invoice_number",
        "issue_date",
        "currency",
        "total_amount"
      ]
    }
  },
  "scored_candidates": [
    {
      "candidate_id": "luna-hosted",
      "route": "cheap_hosted",
      "provider": "OpenAI API",
      "model": "gpt-5.6-luna",
      "reasoning_effort": "none",
      "structured_output": "strict JSON schema",
      "max_output_tokens": 256,
      "price_usd_per_million_tokens": {
        "input": 0.2,
        "cached_input": 0.02,
        "output": 1.2
      }
    },
    {
      "candidate_id": "sol-hosted",
      "route": "frontier_hosted",
      "provider": "OpenAI API",
      "model": "gpt-5.6-sol",
      "reasoning_effort": "none",
      "structured_output": "strict JSON schema",
      "max_output_tokens": 256,
      "price_usd_per_million_tokens": {
        "input": 5,
        "cached_input": 0.5,
        "output": 30
      }
    },
    {
      "candidate_id": "gemma4-local",
      "route": "laptop_open_weight",
      "provider": "Local Ollama",
      "model": "gemma4:12b",
      "artifact": {
        "parameters": "11.9B",
        "quantization": "Q4_K_M",
        "model_blob_sha256": "1278394b693672ac2799eadc9a83fd98259a6a88a40acfb1dcaa6c6fc895a606",
        "model_layer_bytes": 7381382048,
        "license_reported_by_runtime": "Apache-2.0"
      },
      "runtime": "Ollama 0.32.7",
      "thinking": false,
      "temperature": 0,
      "seed": 42,
      "context_tokens": 4096,
      "max_output_tokens": 256,
      "structured_output": "JSON schema"
    }
  ],
  "hardware": {
    "operating_system": "Windows",
    "cpu": "AMD Ryzen AI 9 HX 470 with Radeon 890M",
    "system_memory_gib": 31.32,
    "available_system_memory_gib_at_preregistration": 12.1,
    "gpu": "NVIDIA GeForce RTX 5060 Laptop GPU",
    "gpu_memory_mib": 8151,
    "gpu_driver": "592.25"
  },
  "open_weight_hardware_gate": {
    "model": "meta-models/Muse-Glimmer-30B-GGUF",
    "artifact": "muse-glimmer-30B-kquant-17gb.gguf",
    "artifact_bytes": 16756681056,
    "artifact_sha256": "7e9b74b7c8875e9e265695df9613bf6290f2392e479ce740495a129019c488d8",
    "vendor_target": "24 GB VRAM for the 17 GB K-Quant build",
    "local_gpu_memory_mib": 8151,
    "decision": "excluded_from_scored_candidates_on_this_machine",
    "reason": "The laptop has roughly one third of the vendor-target VRAM. The installed Ollama 0.32.7 build also predates the released GGUF import path; support is announced for 0.32.8. A 16.76 GB download is not a useful scored test on this configuration.",
    "revisit_when": "A supported runtime and at least the vendor-target 24 GB memory envelope are available"
  },
  "execution": {
    "candidate_order": ["luna-hosted", "sol-hosted", "gemma4-local"],
    "attempts_per_case": 1,
    "automatic_retries": 0,
    "transport_errors": "Retained and scored as unsuccessful predictions",
    "raw_artifacts": "Retain sanitised request bodies, complete provider responses, parsed outputs, timing, token usage, and every error",
    "hosted_cost_ceiling_usd": 0.25,
    "stop_condition": "Stop before a request that would take cumulative hosted spend above the ceiling"
  },
  "metrics": {
    "primary": "whole_record_exact_match",
    "secondary": [
      "field_exact_match",
      "schema_validity",
      "missing_prediction_count",
      "latency_ms",
      "measured_token_cost_usd"
    ],
    "pass_threshold": "Unset: this 12-case synthetic fixture cannot establish production readiness"
  },
  "decision_rule": {
    "hosted_route": "Rank by whole-record exact match, then schema validity, then measured variable cost, then median latency. Recommend the cheaper hosted route when it ties the frontier route on quality.",
    "frontier_escalation": "Recommend frontier escalation only for cases where Sol succeeds and Luna fails, or when a production error budget later requires evidence not present here.",
    "local_route": "Report local quality and latency separately. Do not describe zero API fees as zero total cost, because hardware, energy, maintenance, and concurrency are not measured.",
    "claim_boundary": "The result may identify the next routing candidate for this fixture, not a universal best model or production pass."
  }
}