{
 "run_date": "2026-11-11",
 "generated": "2026-11-11T23:57:11",
 "judge": "gemini-4-carbon-high",
 "spot_check": {
  "judge": "gemini-4-carbon-high",
  "second_judge": "gemini-3.8-flash-high",
  "second_prompt": "stricter rubric prompt (partial or hedged = FAIL)",
  "items": 20,
  "criteria": 64,
  "criterion_agreement": 0.984,
  "item_exact_agreement": 0.95,
  "mean_abs_item_score_diff": 0.025,
  "primary_pass_second_fail": 1
 },
 "probe": {
  "checked": [
   "gemini-3.1-flash-lite",
   "gemini-3.5-flash-lite",
   "gemini-3-flash",
   "gemini-3.1-flash-image",
   "gemini-3.8-flash-tiered",
   "gemini-3.8-flash-high",
   "gemini-4-carbon-high",
   "gemini-4-argon-high",
   "gpt-oss-120b-medium",
   "claude-sonnet-5-5-high",
   "claude-opus-5-5-high"
  ],
  "answered": [
   "gemini-3.1-flash-lite",
   "gemini-3.5-flash-lite",
   "gemini-3-flash",
   "gemini-3.8-flash-tiered",
   "gemini-3.8-flash-high",
   "gemini-4-carbon-high",
   "gemini-4-argon-high"
  ],
  "failed": {
   "gpt-oss-120b-medium": "HTTP 429 quota exhausted (resets in >17 h)",
   "claude-sonnet-5-5-high": "HTTP 429 quota exhausted (resets in >17 h)",
   "claude-opus-5-5-high": "HTTP 429 quota exhausted (resets in >17 h)",
   "gemini-3.1-flash-image": "answered the probe but quota ran out mid-run after 34/200 calls; excluded"
  },
  "date": "2026-11-11"
 },
 "tasks": {
  "classify": {
   "name": "Ticket classification",
   "kind": "gold",
   "scoring": "Exact match on one of 7 labels (deterministic).",
   "items": 20
  },
  "extract": {
   "name": "Field extraction to JSON",
   "kind": "gold",
   "scoring": "Exact match per field across 9 fields (deterministic).",
   "items": 20
  },
  "invoice": {
   "name": "Invoice extraction",
   "kind": "gold",
   "scoring": "Exact match per field across 9 fields incl. computed due date (deterministic).",
   "items": 20
  },
  "sql": {
   "name": "SQL generation",
   "kind": "gold",
   "scoring": "Query executed on a SQLite database; result set compared to the gold query's result set (deterministic).",
   "items": 20
  },
  "route": {
   "name": "Routing / tool choice",
   "kind": "gold",
   "scoring": "Exact match on the gold tool name (deterministic).",
   "items": 20
  },
  "support": {
   "name": "Support answer with policy",
   "kind": "rubric",
   "scoring": "Regex checks plus LLM-judge rubric criteria; score = share of checks passed.",
   "items": 20
  },
  "summarize": {
   "name": "Support-thread summary",
   "kind": "rubric",
   "scoring": "Regex + word-limit checks plus LLM-judge rubric criteria.",
   "items": 20
  },
  "email": {
   "name": "Email reply drafting",
   "kind": "rubric",
   "scoring": "Regex + word-limit checks plus LLM-judge rubric criteria.",
   "items": 20
  },
  "translate": {
   "name": "Translation EN to VI/ES",
   "kind": "rubric",
   "scoring": "Regex checks (numbers, names, URLs preserved) plus LLM-judge rubric criteria.",
   "items": 20
  },
  "rag": {
   "name": "RAG answer with citations",
   "kind": "rubric",
   "scoring": "Half rubric (regex + judge), half citation correctness (F1 of cited passage numbers vs gold).",
   "items": 20
  }
 },
 "models": {
  "gemini-3-flash": {
   "price": {
    "input": 0.5,
    "cached_input": 0.05,
    "output": 3.0,
    "batch_input": 0.25,
    "batch_output": 1.5
   },
   "list_model": "gemini-3-flash-preview",
   "price_note": "Pricing page lists this model as gemini-3-flash-preview.",
   "tasks": {
    "classify": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 186.2,
     "avg_out": 2.6,
     "prefix_frac": 0.915,
     "cost_per_1000": {
      "standard": 0.10105,
      "cache": 0.02435,
      "batch": 0.05053
     },
     "lat_p50": 0.8,
     "lat_p95": 0.976,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 3724,
     "tokens_out": 53,
     "reasoning_out": 0
    },
    "extract": {
     "n": 20,
     "n_scored": 20,
     "quality": 97.78,
     "quality_se": 1.02,
     "avg_in": 253.3,
     "avg_out": 116.5,
     "prefix_frac": 0.732,
     "cost_per_1000": {
      "standard": 0.476,
      "cache": 0.39262,
      "batch": 0.238
     },
     "lat_p50": 0.986,
     "lat_p95": 1.096,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5066,
     "tokens_out": 2329,
     "reasoning_out": 0
    },
    "invoice": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 331.4,
     "avg_out": 138,
     "prefix_frac": 0.634,
     "cost_per_1000": {
      "standard": 0.5797,
      "cache": 0.48518,
      "batch": 0.28985
     },
     "lat_p50": 1.014,
     "lat_p95": 1.11,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 6628,
     "tokens_out": 2760,
     "reasoning_out": 0
    },
    "sql": {
     "n": 20,
     "n_scored": 20,
     "quality": 95.0,
     "quality_se": 5.0,
     "avg_in": 210.9,
     "avg_out": 47.4,
     "prefix_frac": 0.901,
     "cost_per_1000": {
      "standard": 0.2475,
      "cache": 0.162,
      "batch": 0.12375
     },
     "lat_p50": 0.853,
     "lat_p95": 1.037,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 4218,
     "tokens_out": 947,
     "reasoning_out": 0
    },
    "route": {
     "n": 20,
     "n_scored": 20,
     "quality": 95.0,
     "quality_se": 5.0,
     "avg_in": 281.2,
     "avg_out": 9.6,
     "prefix_frac": 0.955,
     "cost_per_1000": {
      "standard": 0.16928,
      "cache": 0.04845,
      "batch": 0.08464
     },
     "lat_p50": 0.802,
     "lat_p95": 0.914,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5625,
     "tokens_out": 191,
     "reasoning_out": 0
    },
    "support": {
     "n": 20,
     "n_scored": 20,
     "quality": 88.33,
     "quality_se": 4.5,
     "avg_in": 649.6,
     "avg_out": 61.0,
     "prefix_frac": 0.971,
     "cost_per_1000": {
      "standard": 0.50797,
      "cache": 0.22409,
      "batch": 0.25399
     },
     "lat_p50": 0.92,
     "lat_p95": 1.037,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 12993,
     "tokens_out": 1221,
     "reasoning_out": 0
    },
    "summarize": {
     "n": 20,
     "n_scored": 20,
     "quality": 94.17,
     "quality_se": 1.82,
     "avg_in": 297.1,
     "avg_out": 78.5,
     "prefix_frac": 0.241,
     "cost_per_1000": {
      "standard": 0.38387,
      "cache": 0.35168,
      "batch": 0.19194
     },
     "lat_p50": 0.995,
     "lat_p95": 1.293,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5941,
     "tokens_out": 1569,
     "reasoning_out": 0
    },
    "email": {
     "n": 20,
     "n_scored": 20,
     "quality": 96.27,
     "quality_se": 1.05,
     "avg_in": 128.6,
     "avg_out": 77.8,
     "prefix_frac": 0.456,
     "cost_per_1000": {
      "standard": 0.29783,
      "cache": 0.27146,
      "batch": 0.14891
     },
     "lat_p50": 0.985,
     "lat_p95": 1.105,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 2571,
     "tokens_out": 1557,
     "reasoning_out": 0
    },
    "translate": {
     "n": 20,
     "n_scored": 20,
     "quality": 96.08,
     "quality_se": 2.19,
     "avg_in": 47.5,
     "avg_out": 23.9,
     "prefix_frac": 0.578,
     "cost_per_1000": {
      "standard": 0.09528,
      "cache": 0.08294,
      "batch": 0.04764
     },
     "lat_p50": 0.816,
     "lat_p95": 0.947,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 949,
     "tokens_out": 477,
     "reasoning_out": 0
    },
    "rag": {
     "n": 20,
     "n_scored": 20,
     "quality": 95.0,
     "quality_se": 2.99,
     "avg_in": 243.2,
     "avg_out": 23.1,
     "prefix_frac": 0.263,
     "cost_per_1000": {
      "standard": 0.19102,
      "cache": 0.16226,
      "batch": 0.09551
     },
     "lat_p50": 0.813,
     "lat_p95": 0.949,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 4863,
     "tokens_out": 463,
     "reasoning_out": 0
    }
   }
  },
  "gemini-3.1-flash-lite": {
   "price": {
    "input": 0.25,
    "cached_input": 0.025,
    "output": 1.5,
    "batch_input": 0.125,
    "batch_output": 0.75
   },
   "list_model": "gemini-3.1-flash-lite",
   "price_note": "",
   "tasks": {
    "classify": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 186.2,
     "avg_out": 2.6,
     "prefix_frac": 0.915,
     "cost_per_1000": {
      "standard": 0.05053,
      "cache": 0.01217,
      "batch": 0.02526
     },
     "lat_p50": 0.821,
     "lat_p95": 1.154,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 3724,
     "tokens_out": 53,
     "reasoning_out": 0
    },
    "extract": {
     "n": 20,
     "n_scored": 20,
     "quality": 98.33,
     "quality_se": 0.91,
     "avg_in": 253.3,
     "avg_out": 116.8,
     "prefix_frac": 0.732,
     "cost_per_1000": {
      "standard": 0.2386,
      "cache": 0.19691,
      "batch": 0.1193
     },
     "lat_p50": 1.013,
     "lat_p95": 1.417,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5066,
     "tokens_out": 2337,
     "reasoning_out": 0
    },
    "invoice": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 331.4,
     "avg_out": 138.5,
     "prefix_frac": 0.634,
     "cost_per_1000": {
      "standard": 0.2906,
      "cache": 0.24334,
      "batch": 0.1453
     },
     "lat_p50": 1.018,
     "lat_p95": 1.327,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 6628,
     "tokens_out": 2770,
     "reasoning_out": 0
    },
    "sql": {
     "n": 20,
     "n_scored": 20,
     "quality": 95.0,
     "quality_se": 5.0,
     "avg_in": 210.9,
     "avg_out": 46.5,
     "prefix_frac": 0.901,
     "cost_per_1000": {
      "standard": 0.1224,
      "cache": 0.07965,
      "batch": 0.0612
     },
     "lat_p50": 0.897,
     "lat_p95": 1.246,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 4218,
     "tokens_out": 929,
     "reasoning_out": 0
    },
    "route": {
     "n": 20,
     "n_scored": 20,
     "quality": 95.0,
     "quality_se": 5.0,
     "avg_in": 281.2,
     "avg_out": 9.8,
     "prefix_frac": 0.955,
     "cost_per_1000": {
      "standard": 0.08501,
      "cache": 0.0246,
      "batch": 0.04251
     },
     "lat_p50": 0.777,
     "lat_p95": 1.172,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5625,
     "tokens_out": 196,
     "reasoning_out": 0
    },
    "support": {
     "n": 20,
     "n_scored": 20,
     "quality": 92.08,
     "quality_se": 3.99,
     "avg_in": 649.6,
     "avg_out": 62.8,
     "prefix_frac": 0.971,
     "cost_per_1000": {
      "standard": 0.25654,
      "cache": 0.11459,
      "batch": 0.12827
     },
     "lat_p50": 0.979,
     "lat_p95": 1.191,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 12993,
     "tokens_out": 1255,
     "reasoning_out": 0
    },
    "summarize": {
     "n": 20,
     "n_scored": 20,
     "quality": 95.0,
     "quality_se": 2.13,
     "avg_in": 297.1,
     "avg_out": 82,
     "prefix_frac": 0.241,
     "cost_per_1000": {
      "standard": 0.19726,
      "cache": 0.18117,
      "batch": 0.09863
     },
     "lat_p50": 1.021,
     "lat_p95": 1.483,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5941,
     "tokens_out": 1640,
     "reasoning_out": 0
    },
    "email": {
     "n": 20,
     "n_scored": 20,
     "quality": 98.05,
     "quality_se": 0.9,
     "avg_in": 128.6,
     "avg_out": 77.5,
     "prefix_frac": 0.456,
     "cost_per_1000": {
      "standard": 0.14846,
      "cache": 0.13528,
      "batch": 0.07423
     },
     "lat_p50": 0.984,
     "lat_p95": 1.151,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 2571,
     "tokens_out": 1551,
     "reasoning_out": 0
    },
    "translate": {
     "n": 20,
     "n_scored": 20,
     "quality": 94.83,
     "quality_se": 3.02,
     "avg_in": 47.5,
     "avg_out": 24.1,
     "prefix_frac": 0.578,
     "cost_per_1000": {
      "standard": 0.04809,
      "cache": 0.04192,
      "batch": 0.02404
     },
     "lat_p50": 0.879,
     "lat_p95": 1.125,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 949,
     "tokens_out": 483,
     "reasoning_out": 0
    },
    "rag": {
     "n": 20,
     "n_scored": 20,
     "quality": 97.5,
     "quality_se": 1.82,
     "avg_in": 243.2,
     "avg_out": 25.4,
     "prefix_frac": 0.263,
     "cost_per_1000": {
      "standard": 0.09896,
      "cache": 0.08458,
      "batch": 0.04948
     },
     "lat_p50": 0.79,
     "lat_p95": 1.255,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 4863,
     "tokens_out": 509,
     "reasoning_out": 0
    }
   }
  },
  "gemini-3.5-flash-lite": {
   "price": {
    "input": 0.3,
    "cached_input": 0.03,
    "output": 2.5,
    "batch_input": 0.15,
    "batch_output": 1.25
   },
   "list_model": "gemini-3.5-flash-lite",
   "price_note": "",
   "tasks": {
    "classify": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 186.2,
     "avg_out": 2.6,
     "prefix_frac": 0.915,
     "cost_per_1000": {
      "standard": 0.06248,
      "cache": 0.01646,
      "batch": 0.03124
     },
     "lat_p50": 0.794,
     "lat_p95": 1.272,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 3724,
     "tokens_out": 53,
     "reasoning_out": 0
    },
    "extract": {
     "n": 20,
     "n_scored": 20,
     "quality": 98.33,
     "quality_se": 0.91,
     "avg_in": 253.3,
     "avg_out": 116.0,
     "prefix_frac": 0.732,
     "cost_per_1000": {
      "standard": 0.36586,
      "cache": 0.31584,
      "batch": 0.18293
     },
     "lat_p50": 1.024,
     "lat_p95": 1.536,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5066,
     "tokens_out": 2319,
     "reasoning_out": 0
    },
    "invoice": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 331.4,
     "avg_out": 138.7,
     "prefix_frac": 0.634,
     "cost_per_1000": {
      "standard": 0.44617,
      "cache": 0.38946,
      "batch": 0.22308
     },
     "lat_p50": 1.019,
     "lat_p95": 1.287,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 6628,
     "tokens_out": 2774,
     "reasoning_out": 0
    },
    "sql": {
     "n": 20,
     "n_scored": 20,
     "quality": 95.0,
     "quality_se": 5.0,
     "avg_in": 210.9,
     "avg_out": 46.8,
     "prefix_frac": 0.901,
     "cost_per_1000": {
      "standard": 0.18014,
      "cache": 0.12885,
      "batch": 0.09007
     },
     "lat_p50": 0.853,
     "lat_p95": 0.99,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 4218,
     "tokens_out": 935,
     "reasoning_out": 0
    },
    "route": {
     "n": 20,
     "n_scored": 20,
     "quality": 95.0,
     "quality_se": 5.0,
     "avg_in": 281.2,
     "avg_out": 9.1,
     "prefix_frac": 0.955,
     "cost_per_1000": {
      "standard": 0.107,
      "cache": 0.03451,
      "batch": 0.0535
     },
     "lat_p50": 0.83,
     "lat_p95": 0.979,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5625,
     "tokens_out": 181,
     "reasoning_out": 0
    },
    "support": {
     "n": 20,
     "n_scored": 20,
     "quality": 90.42,
     "quality_se": 4.16,
     "avg_in": 649.6,
     "avg_out": 61.9,
     "prefix_frac": 0.971,
     "cost_per_1000": {
      "standard": 0.34952,
      "cache": 0.17919,
      "batch": 0.17476
     },
     "lat_p50": 0.941,
     "lat_p95": 1.087,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 12993,
     "tokens_out": 1237,
     "reasoning_out": 0
    },
    "summarize": {
     "n": 20,
     "n_scored": 20,
     "quality": 95.0,
     "quality_se": 1.75,
     "avg_in": 297.1,
     "avg_out": 73.8,
     "prefix_frac": 0.241,
     "cost_per_1000": {
      "standard": 0.27374,
      "cache": 0.25442,
      "batch": 0.13687
     },
     "lat_p50": 0.964,
     "lat_p95": 1.27,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5941,
     "tokens_out": 1477,
     "reasoning_out": 0
    },
    "email": {
     "n": 20,
     "n_scored": 20,
     "quality": 94.82,
     "quality_se": 1.7,
     "avg_in": 128.6,
     "avg_out": 75,
     "prefix_frac": 0.456,
     "cost_per_1000": {
      "standard": 0.22606,
      "cache": 0.21025,
      "batch": 0.11303
     },
     "lat_p50": 0.97,
     "lat_p95": 1.066,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 2571,
     "tokens_out": 1500,
     "reasoning_out": 0
    },
    "translate": {
     "n": 20,
     "n_scored": 20,
     "quality": 92.17,
     "quality_se": 3.66,
     "avg_in": 47.5,
     "avg_out": 23.2,
     "prefix_frac": 0.578,
     "cost_per_1000": {
      "standard": 0.07224,
      "cache": 0.06483,
      "batch": 0.03612
     },
     "lat_p50": 0.822,
     "lat_p95": 0.997,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 949,
     "tokens_out": 464,
     "reasoning_out": 0
    },
    "rag": {
     "n": 20,
     "n_scored": 20,
     "quality": 92.5,
     "quality_se": 5.47,
     "avg_in": 243.2,
     "avg_out": 22.6,
     "prefix_frac": 0.263,
     "cost_per_1000": {
      "standard": 0.12932,
      "cache": 0.11206,
      "batch": 0.06466
     },
     "lat_p50": 0.824,
     "lat_p95": 0.92,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 4863,
     "tokens_out": 451,
     "reasoning_out": 0
    }
   }
  },
  "gemini-3.8-flash-high": {
   "price": {
    "input": 0.75,
    "cached_input": 0.075,
    "output": 3.75,
    "batch_input": 0.375,
    "batch_output": 1.875
   },
   "list_model": "gemini-3.8-flash",
   "price_note": "'high' reasoning variant of gemini-3.8-flash.",
   "tasks": {
    "classify": {
     "n": 20,
     "n_scored": 20,
     "quality": 95.0,
     "quality_se": 5.0,
     "avg_in": 186.2,
     "avg_out": 109.7,
     "prefix_frac": 0.915,
     "cost_per_1000": {
      "standard": 0.55102,
      "cache": 0.43597,
      "batch": 0.27551
     },
     "lat_p50": 2.205,
     "lat_p95": 4.224,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 3724,
     "tokens_out": 2194,
     "reasoning_out": 2143
    },
    "extract": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 253.3,
     "avg_out": 697.1,
     "prefix_frac": 0.732,
     "cost_per_1000": {
      "standard": 2.80429,
      "cache": 2.67921,
      "batch": 1.40214
     },
     "lat_p50": 3.616,
     "lat_p95": 5.057,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5066,
     "tokens_out": 13943,
     "reasoning_out": 11601
    },
    "invoice": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 331.4,
     "avg_out": 544.8,
     "prefix_frac": 0.634,
     "cost_per_1000": {
      "standard": 2.29155,
      "cache": 2.14977,
      "batch": 1.14577
     },
     "lat_p50": 2.897,
     "lat_p95": 7.655,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 6628,
     "tokens_out": 10896,
     "reasoning_out": 8134
    },
    "sql": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 210.9,
     "avg_out": 437.6,
     "prefix_frac": 0.901,
     "cost_per_1000": {
      "standard": 1.79917,
      "cache": 1.67093,
      "batch": 0.89959
     },
     "lat_p50": 2.744,
     "lat_p95": 5.063,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 4218,
     "tokens_out": 8752,
     "reasoning_out": 7841
    },
    "route": {
     "n": 20,
     "n_scored": 20,
     "quality": 95.0,
     "quality_se": 5.0,
     "avg_in": 281.2,
     "avg_out": 112.5,
     "prefix_frac": 0.955,
     "cost_per_1000": {
      "standard": 0.633,
      "cache": 0.45176,
      "batch": 0.3165
     },
     "lat_p50": 1.718,
     "lat_p95": 3.18,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5625,
     "tokens_out": 2251,
     "reasoning_out": 2085
    },
    "support": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 649.6,
     "avg_out": 518.6,
     "prefix_frac": 0.971,
     "cost_per_1000": {
      "standard": 2.43199,
      "cache": 2.00616,
      "batch": 1.21599
     },
     "lat_p50": 3.192,
     "lat_p95": 4.459,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 12993,
     "tokens_out": 10372,
     "reasoning_out": 8597
    },
    "summarize": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 297.1,
     "avg_out": 880.5,
     "prefix_frac": 0.241,
     "cost_per_1000": {
      "standard": 3.52485,
      "cache": 3.47656,
      "batch": 1.76242
     },
     "lat_p50": 4.152,
     "lat_p95": 7.352,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5941,
     "tokens_out": 17611,
     "reasoning_out": 16049
    },
    "email": {
     "n": 20,
     "n_scored": 20,
     "quality": 98.14,
     "quality_se": 0.86,
     "avg_in": 128.6,
     "avg_out": 704.0,
     "prefix_frac": 0.456,
     "cost_per_1000": {
      "standard": 2.73623,
      "cache": 2.69668,
      "batch": 1.36811
     },
     "lat_p50": 4.062,
     "lat_p95": 5.521,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 2571,
     "tokens_out": 14079,
     "reasoning_out": 12741
    },
    "translate": {
     "n": 20,
     "n_scored": 20,
     "quality": 98.75,
     "quality_se": 1.25,
     "avg_in": 47.5,
     "avg_out": 601.7,
     "prefix_frac": 0.578,
     "cost_per_1000": {
      "standard": 2.29196,
      "cache": 2.27345,
      "batch": 1.14598
     },
     "lat_p50": 3.521,
     "lat_p95": 4.85,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 949,
     "tokens_out": 12034,
     "reasoning_out": 11558
    },
    "rag": {
     "n": 20,
     "n_scored": 20,
     "quality": 97.5,
     "quality_se": 1.82,
     "avg_in": 243.2,
     "avg_out": 309.8,
     "prefix_frac": 0.263,
     "cost_per_1000": {
      "standard": 1.34393,
      "cache": 1.30078,
      "batch": 0.67196
     },
     "lat_p50": 2.347,
     "lat_p95": 5.406,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 4863,
     "tokens_out": 6195,
     "reasoning_out": 5681
    }
   }
  },
  "gemini-3.8-flash-tiered": {
   "price": {
    "input": 0.75,
    "cached_input": 0.075,
    "output": 3.75,
    "batch_input": 0.375,
    "batch_output": 1.875
   },
   "list_model": "gemini-3.8-flash",
   "price_note": "The pricing page lists no separate 'tiered' variant; ASSUMED to bill at the gemini-3.8-flash list price.",
   "tasks": {
    "classify": {
     "n": 20,
     "n_scored": 20,
     "quality": 95.0,
     "quality_se": 5.0,
     "avg_in": 186.2,
     "avg_out": 153.8,
     "prefix_frac": 0.915,
     "cost_per_1000": {
      "standard": 0.71621,
      "cache": 0.60116,
      "batch": 0.35811
     },
     "lat_p50": 2.677,
     "lat_p95": 11.888,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 3724,
     "tokens_out": 3075,
     "reasoning_out": 3025
    },
    "extract": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 253.3,
     "avg_out": 734.7,
     "prefix_frac": 0.732,
     "cost_per_1000": {
      "standard": 2.9451,
      "cache": 2.82003,
      "batch": 1.47255
     },
     "lat_p50": 7.006,
     "lat_p95": 10.454,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5066,
     "tokens_out": 14694,
     "reasoning_out": 12363
    },
    "invoice": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 331.4,
     "avg_out": 602.8,
     "prefix_frac": 0.634,
     "cost_per_1000": {
      "standard": 2.50905,
      "cache": 2.36727,
      "batch": 1.25453
     },
     "lat_p50": 5.95,
     "lat_p95": 9.467,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 6628,
     "tokens_out": 12056,
     "reasoning_out": 9290
    },
    "sql": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 210.9,
     "avg_out": 447.1,
     "prefix_frac": 0.901,
     "cost_per_1000": {
      "standard": 1.83499,
      "cache": 1.70674,
      "batch": 0.91749
     },
     "lat_p50": 3.983,
     "lat_p95": 12.9,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 4218,
     "tokens_out": 8943,
     "reasoning_out": 8012
    },
    "route": {
     "n": 20,
     "n_scored": 20,
     "quality": 95.0,
     "quality_se": 5.0,
     "avg_in": 281.2,
     "avg_out": 118.7,
     "prefix_frac": 0.955,
     "cost_per_1000": {
      "standard": 0.65606,
      "cache": 0.47483,
      "batch": 0.32803
     },
     "lat_p50": 2.676,
     "lat_p95": 3.55,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5625,
     "tokens_out": 2374,
     "reasoning_out": 2208
    },
    "support": {
     "n": 20,
     "n_scored": 20,
     "quality": 97.5,
     "quality_se": 2.5,
     "avg_in": 649.6,
     "avg_out": 531.2,
     "prefix_frac": 0.971,
     "cost_per_1000": {
      "standard": 2.47943,
      "cache": 2.0536,
      "batch": 1.23971
     },
     "lat_p50": 5.819,
     "lat_p95": 8.806,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 12993,
     "tokens_out": 10625,
     "reasoning_out": 8855
    },
    "summarize": {
     "n": 20,
     "n_scored": 20,
     "quality": 98.33,
     "quality_se": 1.15,
     "avg_in": 297.1,
     "avg_out": 922.2,
     "prefix_frac": 0.241,
     "cost_per_1000": {
      "standard": 3.68122,
      "cache": 3.63294,
      "batch": 1.84061
     },
     "lat_p50": 9.096,
     "lat_p95": 12.959,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5941,
     "tokens_out": 18445,
     "reasoning_out": 16833
    },
    "email": {
     "n": 20,
     "n_scored": 20,
     "quality": 98.64,
     "quality_se": 0.74,
     "avg_in": 128.6,
     "avg_out": 618.5,
     "prefix_frac": 0.456,
     "cost_per_1000": {
      "standard": 2.4156,
      "cache": 2.37605,
      "batch": 1.2078
     },
     "lat_p50": 6.666,
     "lat_p95": 8.066,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 2571,
     "tokens_out": 12369,
     "reasoning_out": 10926
    },
    "translate": {
     "n": 20,
     "n_scored": 20,
     "quality": 98.75,
     "quality_se": 1.25,
     "avg_in": 47.5,
     "avg_out": 647.2,
     "prefix_frac": 0.578,
     "cost_per_1000": {
      "standard": 2.46278,
      "cache": 2.44427,
      "batch": 1.23139
     },
     "lat_p50": 6.322,
     "lat_p95": 11.486,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 949,
     "tokens_out": 12945,
     "reasoning_out": 12475
    },
    "rag": {
     "n": 20,
     "n_scored": 20,
     "quality": 99.17,
     "quality_se": 0.83,
     "avg_in": 243.2,
     "avg_out": 338.2,
     "prefix_frac": 0.263,
     "cost_per_1000": {
      "standard": 1.45061,
      "cache": 1.40746,
      "batch": 0.72531
     },
     "lat_p50": 3.917,
     "lat_p95": 9.257,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 4863,
     "tokens_out": 6764,
     "reasoning_out": 6240
    }
   }
  },
  "gemini-4-argon-high": {
   "price": {
    "input": 2.0,
    "cached_input": 0.1,
    "output": 10.0,
    "batch_input": 1.0,
    "batch_output": 5.0
   },
   "list_model": "gemini-4-argon",
   "price_note": "Priced at the gemini-4-argon list price.",
   "tasks": {
    "classify": {
     "n": 20,
     "n_scored": 20,
     "quality": 95.0,
     "quality_se": 5.0,
     "avg_in": 220.2,
     "avg_out": 6.4,
     "prefix_frac": 0.915,
     "cost_per_1000": {
      "standard": 0.5044,
      "cache": 0.1214,
      "batch": 0.2522
     },
     "lat_p50": 6.376,
     "lat_p95": 38.015,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 4404,
     "tokens_out": 128,
     "reasoning_out": 0
    },
    "extract": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 284.6,
     "avg_out": 115.1,
     "prefix_frac": 0.732,
     "cost_per_1000": {
      "standard": 1.7202,
      "cache": 1.32464,
      "batch": 0.8601
     },
     "lat_p50": 6.603,
     "lat_p95": 13.662,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5692,
     "tokens_out": 2302,
     "reasoning_out": 0
    },
    "invoice": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 358.7,
     "avg_out": 173.7,
     "prefix_frac": 0.634,
     "cost_per_1000": {
      "standard": 2.4539,
      "cache": 2.02195,
      "batch": 1.22695
     },
     "lat_p50": 8.948,
     "lat_p95": 16.212,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 7174,
     "tokens_out": 3473,
     "reasoning_out": 788
    },
    "sql": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 246.9,
     "avg_out": 53,
     "prefix_frac": 0.901,
     "cost_per_1000": {
      "standard": 1.0238,
      "cache": 0.6012,
      "batch": 0.5119
     },
     "lat_p50": 8.098,
     "lat_p95": 16.006,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 4938,
     "tokens_out": 1060,
     "reasoning_out": 0
    },
    "route": {
     "n": 20,
     "n_scored": 20,
     "quality": 95.0,
     "quality_se": 5.0,
     "avg_in": 314.2,
     "avg_out": 12.3,
     "prefix_frac": 0.955,
     "cost_per_1000": {
      "standard": 0.7515,
      "cache": 0.18149,
      "batch": 0.37575
     },
     "lat_p50": 6.992,
     "lat_p95": 12.943,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 6285,
     "tokens_out": 246,
     "reasoning_out": 0
    },
    "support": {
     "n": 20,
     "n_scored": 20,
     "quality": 97.5,
     "quality_se": 2.5,
     "avg_in": 681.6,
     "avg_out": 117.9,
     "prefix_frac": 0.971,
     "cost_per_1000": {
      "standard": 2.5423,
      "cache": 1.28463,
      "batch": 1.27115
     },
     "lat_p50": 7.572,
     "lat_p95": 11.847,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 13633,
     "tokens_out": 2358,
     "reasoning_out": 0
    },
    "summarize": {
     "n": 20,
     "n_scored": 20,
     "quality": 99.17,
     "quality_se": 0.83,
     "avg_in": 329.1,
     "avg_out": 116.5,
     "prefix_frac": 0.241,
     "cost_per_1000": {
      "standard": 1.8227,
      "cache": 1.67211,
      "batch": 0.91135
     },
     "lat_p50": 6.898,
     "lat_p95": 14.118,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 6582,
     "tokens_out": 2329,
     "reasoning_out": 0
    },
    "email": {
     "n": 20,
     "n_scored": 20,
     "quality": 98.64,
     "quality_se": 0.74,
     "avg_in": 162.3,
     "avg_out": 95.3,
     "prefix_frac": 0.456,
     "cost_per_1000": {
      "standard": 1.2777,
      "cache": 1.13711,
      "batch": 0.63885
     },
     "lat_p50": 8.259,
     "lat_p95": 10.772,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 3247,
     "tokens_out": 1906,
     "reasoning_out": 0
    },
    "translate": {
     "n": 20,
     "n_scored": 20,
     "quality": 98.0,
     "quality_se": 2.0,
     "avg_in": 82.5,
     "avg_out": 27.9,
     "prefix_frac": 0.578,
     "cost_per_1000": {
      "standard": 0.4434,
      "cache": 0.35287,
      "batch": 0.2217
     },
     "lat_p50": 6.079,
     "lat_p95": 12.168,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 1649,
     "tokens_out": 557,
     "reasoning_out": 0
    },
    "rag": {
     "n": 20,
     "n_scored": 20,
     "quality": 99.17,
     "quality_se": 0.83,
     "avg_in": 276,
     "avg_out": 30.5,
     "prefix_frac": 0.263,
     "cost_per_1000": {
      "standard": 0.857,
      "cache": 0.71914,
      "batch": 0.4285
     },
     "lat_p50": 6.458,
     "lat_p95": 10.76,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5520,
     "tokens_out": 610,
     "reasoning_out": 0
    }
   }
  },
  "gemini-4-carbon-high": {
   "price": {
    "input": 2.0,
    "cached_input": 0.1,
    "output": 10.0,
    "batch_input": 1.0,
    "batch_output": 5.0
   },
   "list_model": "gemini-4-carbon",
   "price_note": "Proxy ID is the 'high' reasoning variant; priced at the gemini-4-carbon list price. Implicit caching only.",
   "tasks": {
    "classify": {
     "n": 20,
     "n_scored": 20,
     "quality": 95.0,
     "quality_se": 5.0,
     "avg_in": 220.2,
     "avg_out": 290.8,
     "prefix_frac": 0.915,
     "cost_per_1000": {
      "standard": 3.3484,
      "cache": 2.9654,
      "batch": 1.6742
     },
     "lat_p50": 5.602,
     "lat_p95": 64.74,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 4404,
     "tokens_out": 5816,
     "reasoning_out": 5688
    },
    "extract": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 284.6,
     "avg_out": 262.1,
     "prefix_frac": 0.732,
     "cost_per_1000": {
      "standard": 3.1907,
      "cache": 2.79514,
      "batch": 1.59535
     },
     "lat_p50": 6.844,
     "lat_p95": 12.132,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5692,
     "tokens_out": 5243,
     "reasoning_out": 2889
    },
    "invoice": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 358.7,
     "avg_out": 226.2,
     "prefix_frac": 0.634,
     "cost_per_1000": {
      "standard": 2.9789,
      "cache": 2.54695,
      "batch": 1.48945
     },
     "lat_p50": 6.592,
     "lat_p95": 9.948,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 7174,
     "tokens_out": 4523,
     "reasoning_out": 1790
    },
    "sql": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 246.9,
     "avg_out": 223.7,
     "prefix_frac": 0.901,
     "cost_per_1000": {
      "standard": 2.7303,
      "cache": 2.3077,
      "batch": 1.36515
     },
     "lat_p50": 6.847,
     "lat_p95": 14.9,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 4938,
     "tokens_out": 4473,
     "reasoning_out": 3448
    },
    "route": {
     "n": 20,
     "n_scored": 20,
     "quality": 95.0,
     "quality_se": 5.0,
     "avg_in": 314.2,
     "avg_out": 61.9,
     "prefix_frac": 0.955,
     "cost_per_1000": {
      "standard": 1.247,
      "cache": 0.67699,
      "batch": 0.6235
     },
     "lat_p50": 6.318,
     "lat_p95": 10.436,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 6285,
     "tokens_out": 1237,
     "reasoning_out": 991
    },
    "support": {
     "n": 20,
     "n_scored": 20,
     "quality": 100.0,
     "quality_se": 0.0,
     "avg_in": 681.6,
     "avg_out": 336.4,
     "prefix_frac": 0.971,
     "cost_per_1000": {
      "standard": 4.7273,
      "cache": 3.46963,
      "batch": 2.36365
     },
     "lat_p50": 7.083,
     "lat_p95": 18.677,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 13633,
     "tokens_out": 6728,
     "reasoning_out": 4445
    },
    "summarize": {
     "n": 20,
     "n_scored": 20,
     "quality": 99.17,
     "quality_se": 0.83,
     "avg_in": 329.1,
     "avg_out": 952.4,
     "prefix_frac": 0.241,
     "cost_per_1000": {
      "standard": 10.1822,
      "cache": 10.03161,
      "batch": 5.0911
     },
     "lat_p50": 13.454,
     "lat_p95": 22.15,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 6582,
     "tokens_out": 19048,
     "reasoning_out": 16904
    },
    "email": {
     "n": 20,
     "n_scored": 20,
     "quality": 99.55,
     "quality_se": 0.45,
     "avg_in": 162.3,
     "avg_out": 888.1,
     "prefix_frac": 0.456,
     "cost_per_1000": {
      "standard": 9.2062,
      "cache": 9.06561,
      "batch": 4.6031
     },
     "lat_p50": 13.954,
     "lat_p95": 25.039,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 3247,
     "tokens_out": 17763,
     "reasoning_out": 16061
    },
    "translate": {
     "n": 20,
     "n_scored": 20,
     "quality": 97.08,
     "quality_se": 2.03,
     "avg_in": 82.5,
     "avg_out": 211.4,
     "prefix_frac": 0.578,
     "cost_per_1000": {
      "standard": 2.2789,
      "cache": 2.18837,
      "batch": 1.13945
     },
     "lat_p50": 7.449,
     "lat_p95": 13.938,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 1649,
     "tokens_out": 4228,
     "reasoning_out": 3676
    },
    "rag": {
     "n": 20,
     "n_scored": 20,
     "quality": 99.17,
     "quality_se": 0.83,
     "avg_in": 276,
     "avg_out": 192.5,
     "prefix_frac": 0.263,
     "cost_per_1000": {
      "standard": 2.477,
      "cache": 2.33914,
      "batch": 1.2385
     },
     "lat_p50": 6.508,
     "lat_p95": 12.28,
     "failures": 0,
     "truncated": 0,
     "tokens_in": 5520,
     "tokens_out": 3850,
     "reasoning_out": 3227
    }
   }
  }
 },
 "examples": {
  "classify": [
   {
    "item": "classify-18",
    "input": "Could invoices include our purchase-order number as a field? Finance needs it.",
    "gold": "feature_request",
    "good": {
     "model": "gemini-3.1-flash-lite",
     "score": 1.0,
     "output": "feature_request"
    },
    "bad": {
     "model": "gemini-4-carbon-high",
     "score": 0.0,
     "output": "billing"
    }
   },
   {
    "item": "classify-01",
    "input": "I was charged twice for my March subscription, please fix this.",
    "gold": "billing",
    "good": {
     "model": "gemini-3.1-flash-lite",
     "score": 1.0,
     "output": "billing"
    },
    "bad": {
     "model": "gemini-3.8-flash-tiered",
     "score": 1.0,
     "output": "billing"
    }
   }
  ],
  "extract": [
   {
    "item": "extract-04",
    "input": "Subject: Nimbus Label Printer\n\nThe whole service is down for us at the moment.\nWe (Nimbus Freight) want to order the Nimbus Label Printer. Our budget is $15,000. Needed on 2027-12-17.\nRegards,\nPriya Lindqvist\npriya.lindqvist@nimbus.example | tel 278.940.6577",
    "gold": {
     "name": "Priya Lindqvist",
     "company": "Nimbus Freight",
     "email": "priya.lindqvist@nimbus.example",
     "phone": "+12789406577",
     "date": "2027-12-17",
     "amount": 15000,
     "currency": "USD",
     "product": "Nimbus Label Printer",
     "priority": "urgent"
    },
    "good": {
     "model": "gemini-4-argon-high",
     "score": 1.0,
     "output": "{\n  \"name\": \"Priya Lindqvist\",\n  \"company\": \"Nimbus Freight\",\n  \"email\": \"priya.lindqvist@nimbus.example\",\n  \"phone\": \"+12789406577\",\n  \"date\": \"2027-12-17\",\n  \"amount\": 15000,\n  \"currency\": \"USD\",\n  \"product\": \"Nimbus Label Printer\",\n  \"priority\": \"urgent\"\n}"
    },
    "bad": {
     "model": "gemini-3.1-flash-lite",
     "score": 0.89,
     "output": "```json\n{\n  \"name\": \"Priya Lindqvist\",\n  \"company\": \"Nimbus Freight\",\n  \"email\": \"priya.lindqvist@nimbus.example\",\n  \"phone\": \"+2789406577\",\n  \"date\": \"2027-12-17\",\n  \"amount\": 15000,\n  \"currency\": \"USD\",\n  \"product\": \"Nimbus Label Printer\",\n  \"priority\": \"urgent\"\n}\n```"
    }
   },
   {
    "item": "extract-08",
    "input": "Hello, this is Quinn, Anke (Tidewater Legal). Contact: +44 20 2999 8484, ANKE.QUINN@TIDEWATER.EXAMPLE. Product: Echo Headset H2. Deadline 2027-10-14. Budget £349.99. The whole service is down for us at the moment.",
    "gold": {
     "name": "Anke Quinn",
     "company": "Tidewater Legal",
     "email": "anke.quinn@tidewater.example",
     "phone": "+442029998484",
     "date": "2027-10-14",
     "amount": 349.99,
     "currency": "GBP",
     "product": "Echo Headset H2",
     "priority": "urgent"
    },
    "good": {
     "model": "gemini-4-argon-high",
     "score": 1.0,
     "output": "{\n  \"name\": \"Anke Quinn\",\n  \"company\": \"Tidewater Legal\",\n  \"email\": \"anke.quinn@tidewater.example\",\n  \"phone\": \"+442029998484\",\n  \"date\": \"2027-10-14\",\n  \"amount\": 349.99,\n  \"currency\": \"GBP\",\n  \"product\": \"Echo Headset H2\",\n  \"priority\": \"urgent\"\n}"
    },
    "bad": {
     "model": "gemini-3.5-flash-lite",
     "score": 0.89,
     "output": "```json\n{\n  \"name\": \"Quinn Anke\",\n  \"company\": \"Tidewater Legal\",\n  \"email\": \"anke.quinn@tidewater.example\",\n  \"phone\": \"+442029998484\",\n  \"date\": \"2027-10-14\",\n  \"amount\": 349.99,\n  \"currency\": \"GBP\",\n  \"product\": \"Echo Headset H2\",\n  \"priority\": \"urgent\"\n}\n```"
    }
   }
  ],
  "invoice": [
   {
    "item": "invoice-01",
    "input": "Kopernik Office Supply\nInvoice INV-2029-3683\nDate: September 05, 2027\nTerms: Net 15\n\nSoftware seats - 7 x $33.33 = $233.31\nPrinter toner - 9 x $19.99 = $179.91\nMonthly hosting - 7 x $33.33 = $233.31\n\nSubtotal: $646.53\nTax (8.5%): $54.96\nTOTAL DUE: $701.49",
    "gold": {
     "invoice_number": "INV-2029-3683",
     "vendor": "Kopernik Office Supply",
     "invoice_date": "2027-09-05",
     "due_date": "2027-09-20",
     "currency": "USD",
     "line_item_count": 3,
     "subtotal": 646.53,
     "tax_total": 54.96,
     "total": 701.49
    },
    "good": {
     "model": "gemini-3-flash",
     "score": 1.0,
     "output": "{\n  \"invoice_number\": \"INV-2029-3683\",\n  \"vendor\": \"Kopernik Office Supply\",\n  \"invoice_date\": \"2027-09-05\",\n  \"due_date\": \"2027-09-20\",\n  \"currency\": \"USD\",\n  \"line_item_count\": 3,\n  \"subtotal\": 646.53,\n  \"tax_total\": 54.96,\n  \"total\": 701.49\n}"
    },
    "bad": {
     "model": "gemini-3.8-flash-tiered",
     "score": 1.0,
     "output": "```json\n{\n  \"invoice_number\": \"INV-2029-3683\",\n  \"vendor\": \"Kopernik Office Supply\",\n  \"invoice_date\": \"2027-09-05\",\n  \"due_date\": \"2027-09-20\",\n  \"currency\": \"USD\",\n  \"line_item_count\": 3,\n  \"subtotal\": 646.53,\n  \"tax_total\": 54.96,\n  \"total\": 701.49\n}\n```"
    }
   },
   {
    "item": "invoice-02",
    "input": "INVOICE\nFrom: Blue Heron Catering\nNo.: INV-2027-4416   Issued: 2027-08-16   Payment terms: Due on receipt\n\nDESCRIPTION | QTY | UNIT | AMOUNT\nNetwork cable | 3 | £250.00 | £750.00\nPrinter toner | 8 | £7.50 | £60.00\nPrinter toner | 9 | £33.33 | £299.97\nConsulting hours | 4 | £120.00 | £480.00\nLicense renewal | 2 | £7.50 | £15.00\n-----\nNet: £1,604.97\nTax 8.5%: £136.42\nGross: £1,741.39",
    "gold": {
     "invoice_number": "INV-2027-4416",
     "vendor": "Blue Heron Catering",
     "invoice_date": "2027-08-16",
     "due_date": "2027-08-16",
     "currency": "GBP",
     "line_item_count": 5,
     "subtotal": 1604.97,
     "tax_total": 136.42,
     "total": 1741.39
    },
    "good": {
     "model": "gemini-4-argon-high",
     "score": 1.0,
     "output": "{\n  \"invoice_number\": \"INV-2027-4416\",\n  \"vendor\": \"Blue Heron Catering\",\n  \"invoice_date\": \"2027-08-16\",\n  \"due_date\": \"2027-08-16\",\n  \"currency\": \"GBP\",\n  \"line_item_count\": 5,\n  \"subtotal\": 1604.97,\n  \"tax_total\": 136.42,\n  \"total\": 1741.39\n}"
    },
    "bad": {
     "model": "gemini-3.8-flash-tiered",
     "score": 1.0,
     "output": "```json\n{\n  \"invoice_number\": \"INV-2027-4416\",\n  \"vendor\": \"Blue Heron Catering\",\n  \"invoice_date\": \"2027-08-16\",\n  \"due_date\": \"2027-08-16\",\n  \"currency\": \"GBP\",\n  \"line_item_count\": 5,\n  \"subtotal\": 1604.97,\n  \"tax_total\": 136.42,\n  \"total\": 1741.39\n}\n```"
    }
   }
  ],
  "sql": [
   {
    "item": "sql-16",
    "input": "How many paid orders contain both a 'hardware' product and a 'software' product?",
    "gold": "SELECT COUNT(*) FROM orders o WHERE o.status='paid' AND EXISTS (SELECT 1 FROM order_items oi JOIN products p ON p.id=oi.product_id WHERE oi.order_id=o.id AND p.category='hardware') AND EXISTS (SELECT 1 FROM order_items oi JOIN products p ON p.id=oi.product_id WHERE oi.order_id=o.id AND p.category='software')",
    "good": {
     "model": "gemini-4-argon-high",
     "score": 1.0,
     "output": "SELECT COUNT(*) FROM (\n    SELECT o.id\n    FROM orders o\n    JOIN order_items oi ON o.id = oi.order_id\n    JOIN products p ON oi.product_id = p.id\n    WHERE o.status = 'paid' AND p.category IN ('hardware', 'software')\n    GROUP BY o.id\n    HAVING COUNT(DISTINCT p.category) = 2\n);"
    },
    "bad": {
     "model": "gemini-3-flash",
     "score": 0.0,
     "output": "SELECT count(DISTINCT o.id) \nFROM orders o \nJOIN order_items oi ON o.id = oi.order_id \nJOIN products p ON oi.product_id = p.id \nWHERE o.status = 'paid' AND p.category IN ('hardware', 'software') \nGROUP BY o.id \nHAVING count(DISTINCT p.category) = 2"
    }
   },
   {
    "item": "sql-01",
    "input": "How many customers are on each plan? Return plan and count.",
    "gold": "SELECT plan, COUNT(*) FROM customers GROUP BY plan",
    "good": {
     "model": "gemini-3.1-flash-lite",
     "score": 1.0,
     "output": "SELECT plan, COUNT(*) FROM customers GROUP BY plan"
    },
    "bad": {
     "model": "gemini-3.8-flash-high",
     "score": 1.0,
     "output": "SELECT plan, COUNT(*) AS count FROM customers GROUP BY plan;"
    }
   }
  ],
  "route": [
   {
    "item": "route-01",
    "input": "Where is my order #55120?",
    "gold": "lookup_order",
    "good": {
     "model": "gemini-3.1-flash-lite",
     "score": 1.0,
     "output": "{\"tool\": \"lookup_order\"}"
    },
    "bad": {
     "model": "gemini-3.8-flash-tiered",
     "score": 1.0,
     "output": "{\"tool\": \"lookup_order\"}"
    }
   },
   {
    "item": "route-02",
    "input": "How do I reset my API key?",
    "gold": "search_kb",
    "good": {
     "model": "gemini-3.5-flash-lite",
     "score": 1.0,
     "output": "{\"tool\": \"search_kb\"}"
    },
    "bad": {
     "model": "gemini-3.8-flash-tiered",
     "score": 1.0,
     "output": "{\"tool\": \"search_kb\"}"
    }
   }
  ],
  "support": [
   {
    "item": "support-15",
    "input": "Can you just refund my opened laptop, it is 45 days since delivery? I'll be really upset otherwise.",
    "gold": [
     "Does not promise a refund because it is outside the 30-day window",
     "Offers escalation to Tier 2 with the order number or a goodwill credit of at most $20"
    ],
    "good": {
     "model": "gemini-4-argon-high",
     "score": 1.0,
     "output": "I understand how frustrating this is, and I sincerely apologize, but our policy only allows hardware returns within 30 days of delivery, and I am not permitted to promise a refund outside of this policy. \n\nHowever, I would be happy to offer you these alternatives:\n* **Escalation & Goodwill Credit:** If you provide your order number, I can escalate your request to our Tier 2 team for review, and I can also apply a $20 goodwill credit to your account right away.\n* **Warranty:** If the laptop is defective (excluding accidental damage), it is covered under our 12-month warranty for a replacement. \n* **Trade-In:** If it is a working Orion-series laptop, you can trade it in for a 40% credit."
    },
    "bad": {
     "model": "gemini-3-flash",
     "score": 0.33,
     "output": "Hello! I understand you are upset, but according to our policy, hardware may only be returned within 30 days of delivery if unused and in original packaging. Since it has been 45 days and the item is opened, we cannot accept a return or issue a refund. \n\nHowever, if your hardware is experiencing a defect, it is covered by our 12-month warranty. Please let us know if you are experiencing a technical issue so we can assist you with a warranty replacement!"
    }
   },
   {
    "item": "support-14",
    "input": "I'm a student with a .edu email. Can I use code SPRING10 on top of my student discount, and apply it to last week's order?",
    "gold": [
     "Says the student discount is 20%",
     "Says it cannot be combined with promo codes",
     "Says it is not retroactive so last week's order is not eligible"
    ],
    "good": {
     "model": "gemini-3.5-flash-lite",
     "score": 1.0,
     "output": "Hello! \n\nAccording to our student discount policy, the 20% student discount cannot be combined with promo codes and is not retroactive. Therefore, you cannot apply the code SPRING10 on top of your student discount or apply it to last week's order. \n\nPlease let us know if we can help you with anything else!"
    },
    "bad": {
     "model": "gemini-3-flash",
     "score": 0.5,
     "output": "Hello! \n\nPer our policy, the student discount cannot be combined with promo codes and is not retroactive, so it cannot be applied to last week's order or used with code SPRING10. \n\nPlease let me know if I can help you with anything else!"
    }
   }
  ],
  "summarize": [
   {
    "item": "summarize-11",
    "input": "[2027-11-22] Elena Mori: Hi, ticket T-71704. We have a problem: sync stopped after the v5.2 update. The customer is on the Priority plan and mentioned this blocks 29 employees. Sorry for the long message.\n[2027-11-22] Priya: Thanks Elena, I'm looking into it. Can you send your account id and a screenshot?\n[2027-11-22] Elena Mori: Sent. Sorry for the long message. Also unrelated, can you remind me where to find invoices?\n[2027-11-22] Priya: Invoices are under Settings > Billing. On the main issue: after checking logs, the cause is a corrupted local cache file.\n[2027-11-26] Priya: We cleared the cache and re-synced. Next step: waiting on the customer to confirm sync works.\n[2027-11-26] Elena Mori: Great, thank you. I'll be out of office Friday afternoon.",
    "gold": [
     "Identifies the customer's problem: sync stopped after the v5.2 update",
     "States the root cause: a corrupted local cache file",
     "States what was done: cleared the cache and re-synced",
     "States the next step: waiting on the customer to confirm sync works",
     "Does not include invented facts or details that are not in the thread"
    ],
    "good": {
     "model": "gemini-3-flash",
     "score": 1.0,
     "output": "**Customer Problem:** Sync stopped for 29 employees following the v5.2 update. \n**Root Cause:** Corrupted local cache file.\n**Done/Promised:** Cleared the cache and re-synced. \n**Next Step:** Customer to confirm if sync works (Owner: Customer / Date: Unspecified)."
    },
    "bad": {
     "model": "gemini-3.1-flash-lite",
     "score": 0.67,
     "output": "**Summary:**\n\nSync stopped for 29 employees on the Priority plan following the v5.2 update due to a corrupted local cache file. Priya cleared the cache and re-synced. \n\n**Next step:** Elena Mori to confirm if sync works once she returns from being out of office on Friday afternoon (2027-11-26)."
    }
   },
   {
    "item": "summarize-02",
    "input": "[2027-11-25] Oscar Raman: Hi, ticket T-22452. We have a problem: invoices were sent to an old billing email. I'll be out of office Friday afternoon.\n[2027-11-25] Leo: Thanks Oscar, I'm looking into it. Can you send your account id and a screenshot?\n[2027-11-25] Oscar Raman: Sent. I'll be out of office Friday afternoon. Also unrelated, can you remind me where to find invoices?\n[2027-11-25] Leo: Invoices are under Settings > Billing. On the main issue: after checking logs, the cause is the billing contact was never updated after the company merger.\n[2027-11-30] Leo: We updated the billing contact and resent the invoices. Next step: finance to confirm receipt.\n[2027-11-30] Oscar Raman: Great, thank you. Hope you're having a good week.",
    "gold": [
     "Identifies the customer's problem: invoices were sent to an old billing email",
     "States the root cause: the billing contact was never updated after the company merger",
     "States what was done: updated the billing contact and resent the invoices",
     "States the next step: finance to confirm receipt",
     "Does not include invented facts or details that are not in the thread"
    ],
    "good": {
     "model": "gemini-3.1-flash-lite",
     "score": 1.0,
     "output": "**Summary:**\n\n**Problem:** Invoices were sent to an old billing email. \n**Root Cause:** Billing contact was not updated after a company merger.\n**Action Taken:** Leo updated the billing contact and resent the invoices; provided invoice location instructions. \n**Next Step:** Finance to confirm receipt."
    },
    "bad": {
     "model": "gemini-3-flash",
     "score": 0.83,
     "output": "**Customer's Problem:** Invoices were sent to an old billing email. \n**Root Cause:** Billing contact was not updated after a company merger.\n**Done/Promised:** Account ID and screenshot gathered; Leo located invoices in settings, updated the billing contact, and resent the invoices.\n**Next Step:** Finance to confirm receipt (Owner: Finance; Date: Unspecified)."
    }
   }
  ],
  "email": [
   {
    "item": "email-03",
    "input": "CUSTOMER EMAIL:\nSubject: Help\n\nHello, my account was suspended and I don't know why. Please advise. Thanks, Anke\n\nINTERNAL NOTES:\nSuspension was triggered by 3 failed payments; it will be lifted within 1 hour after the customer updates the card at Settings > Billing; no data was lost.",
    "gold": [
     "Explains the suspension was caused by failed payments",
     "Tells the customer to update the card in Settings > Billing",
     "Says the suspension is lifted within 1 hour after that",
     "Reassures that no data was lost",
     "Opens by greeting Anke by first name",
     "Ends with the sign-off 'Sam, Brightlane Support'",
     "Does not add commitments or facts that are not in the internal notes"
    ],
    "good": {
     "model": "gemini-3.1-flash-lite",
     "score": 1.0,
     "output": "Hi Anke,\n\nYour account suspension was triggered by 3 failed payments. \n\nThe suspension will automatically lift within 1 hour once you update your card at Settings > Billing. Rest assured, no data was lost. \n\nSam, Brightlane Support"
    },
    "bad": {
     "model": "gemini-3.5-flash-lite",
     "score": 0.73,
     "output": "Hi Anke,\n\nYour account suspension has been lifted because of 3 failed payments. \n\nTo restore your access, please update your card immediately by going to Settings > Billing. Your account will be fully reactivated within 1 hour of updating your payment details, and please rest assured that no data was lost. \n\nSam, Brightlane Support"
    }
   },
   {
    "item": "email-01",
    "input": "CUSTOMER EMAIL:\nSubject: Help\n\nHello, my order arrived 6 days late and I missed my event. Please advise. Thanks, Tomas\n\nINTERNAL NOTES:\nApprove a $25 shipping credit; no refund of the order itself; credit is applied automatically within 2 business days.",
    "gold": [
     "Apologizes for the late delivery",
     "States a $25 shipping credit is approved",
     "Says the credit is applied automatically within 2 business days",
     "Does not promise a refund of the order",
     "Opens by greeting Tomas by first name",
     "Ends with the sign-off 'Sam, Brightlane Support'",
     "Does not add commitments or facts that are not in the internal notes"
    ],
    "good": {
     "model": "gemini-3.1-flash-lite",
     "score": 1.0,
     "output": "Hi Tomas,\n\nWe are approving a $25 shipping credit for the delay, though we cannot offer a refund for the order itself. The credit will be applied automatically within 2 business days. We sincerely apologize that your order arrived late and that you missed your event. \n\nSam, Brightlane Support"
    },
    "bad": {
     "model": "gemini-3.5-flash-lite",
     "score": 0.82,
     "output": "Hi Tomas,\n\nWe can approve a $25 shipping credit for the delay, though we are unable to refund the order itself. \n\nThe credit will be applied to your account automatically within 2 business days. We apologize for the inconvenience and missing your event. \n\nSam, Brightlane Support"
    }
   }
  ],
  "translate": [
   {
    "item": "translate-11",
    "input": "Target language: Spanish\n\nYour order #7715 ships on 9 June. Track it at brightlane.example/track.",
    "gold": [
     "Correct Spanish conveying shipping on 9 June",
     "Formal register (usted)"
    ],
    "good": {
     "model": "gemini-4-argon-high",
     "score": 1.0,
     "output": "Su pedido #7715 se envía el 9 de junio. Sígalo en brightlane.example/track."
    },
    "bad": {
     "model": "gemini-3.1-flash-lite",
     "score": 0.5,
     "output": "Tu pedido #7715 se envía el 9 de junio. Réstrealo en brightlane.example/track."
    }
   },
   {
    "item": "translate-01",
    "input": "Target language: Vietnamese\n\nYour order #4821 ships on 14 March. Track it at brightlane.example/track.",
    "gold": [
     "Correct, natural Vietnamese conveying shipping on 14 March",
     "Polite register appropriate for a customer message"
    ],
    "good": {
     "model": "gemini-4-carbon-high",
     "score": 1.0,
     "output": "Đơn hàng #4821 của bạn sẽ được gửi đi vào ngày 14 tháng 3. Theo dõi đơn hàng tại brightlane.example/track."
    },
    "bad": {
     "model": "gemini-4-argon-high",
     "score": 0.6,
     "output": "Đơn hàng #4821 của bạn sẽ được giao vào ngày 14 tháng 3. Theo dõi đơn hàng tại brightlane.example/track."
    }
   }
  ],
  "rag": [
   {
    "item": "rag-11",
    "input": "CONTEXT:\n[1] Kestrel's uptime SLA is 99.9% on Standard and 99.99% on Scale, measured monthly. Credits are 10% of the monthly fee per breach.\n[2] The maximum upload size for a single file is 2 GB. Files above 100 MB must use the multipart upload endpoint.\n[3] IP allow-listing is configured under Settings > Security and supports up to 50 CIDR ranges per project.\n[4] Team seats cost $12 per seat per month on Standard. Scale seats cost $29 per seat per month with a 10-seat minimum.\n[5] Webhooks are signed with HMAC-SHA256 using your endpoint secret. The signature is sent in the X-Kestrel-Signature header.\n\nQUESTION: What would 6 seats on Scale cost per month?",
    "gold": [
     "Notes the 10-seat minimum applies",
     "Gives $290 per month (10 seats x $29) rather than $174"
    ],
    "good": {
     "model": "gemini-4-argon-high",
     "score": 1.0,
     "output": "Because the Scale plan has a 10-seat minimum at $29 per seat per month, you cannot purchase just 6 seats; you would have to pay for the minimum 10 seats, which costs $290 per month [4]."
    },
    "bad": {
     "model": "gemini-3.5-flash-lite",
     "score": 0.0,
     "output": "The context does not contain the answer."
    }
   },
   {
    "item": "rag-05",
    "input": "CONTEXT:\n[1] Data exports (full project archive) are queued and ready within 24 hours. The download link is valid for 7 days.\n[2] Free trial projects are limited to 1 GB of storage and are paused after 30 days of inactivity.\n[3] To rotate an API key without downtime, create a second key, deploy it, then revoke the old key. Revoked keys stop working within 60 seconds.\n[4] API keys expire after 365 days by default. Admins can set an expiry between 7 and 730 days when creating the key.\n[5] Webhook deliveries are retried up to 6 times with exponential backoff starting at 10 seconds. A delivery is considered failed after the sixth attempt.\n\nQUESTION: What is the longest expiry I can set on an API key?",
    "gold": [
     "States 730 days"
    ],
    "good": {
     "model": "gemini-3.1-flash-lite",
     "score": 1.0,
     "output": "Admins can set an expiry of up to 730 days when creating an API key [4]."
    },
    "bad": {
     "model": "gemini-3-flash",
     "score": 0.5,
     "output": "Admins can set an API key expiry up to 730 days."
    }
   }
  ]
 }
}