{ "agreement_rule": { "beat_definition": "0 accepted errors, strictly more accepted than Solar + Perplexity (53), and a lower two-run charge than $0.03749436; counted by pairs_more_and_cheaper", "best_beating": { "accepted": 57, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-005", "DEV-006", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort fresh1", "model": "deepseek/deepseek-v4.1-flash", "pass": "fresh1", "run_id": "extended-openrouter-paid-deepseek-v41-flash-low-fresh1-p0" }, "run_b": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on fresh1", "model": "google/gemma-4-26b-a4b-it", "pass": "fresh1", "run_id": "extended-openrouter-paid-gemma4-26b-a4b-on-fresh1-p0" }, "two_run_charge_usd": 0.03674717 }, "budget_answers": [ { "accepted": 51, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 9, "deferred_ids": [ "DEV-006", "DEV-010", "DEV-013", "DEV-014", "DEV-027", "DEV-029", "DEV-030", "DEV-053", "DEV-059" ], "question": "Cheapest pair with 0 accepted errors and at least 50 accepted", "run_a": { "category": "general", "condition": "P0", "effort": "off", "label": "DeepSeek V4.1 Flash thinking off original", "model": "deepseek/deepseek-v4.1-flash", "pass": "original", "run_id": "openrouter-paid-deepseek-v41-flash-off" }, "run_b": { "category": "decision", "condition": "P0", "effort": "na", "label": "jev-1.13.0 fresh1", "model": "jev-1.13.0", "pass": "fresh1", "run_id": "extended-typesafe-jev113-native-prompts-fresh1-p0" }, "two_run_charge_usd": 0.00857206 }, { "accepted": 54, "accepted_error_ids": [ "DEV-014" ], "accepted_errors": 1, "charge_basis": "observed or known provider charges", "deferred": 6, "deferred_ids": [ "DEV-005", "DEV-006", "DEV-010", "DEV-013", "DEV-030", "DEV-053" ], "question": "Cheapest pair with at most 1 accepted error and at least 54 accepted", "run_a": { "category": "general", "condition": "P0", "effort": "off", "label": "Gemma 4 31B thinking off original", "model": "google/gemma-4-31b-it", "pass": "original", "run_id": "openrouter-paid-gemma4-31b-off" }, "run_b": { "category": "general", "condition": "P0", "effort": "off", "label": "DeepSeek V4.1 Flash thinking off original", "model": "deepseek/deepseek-v4.1-flash", "pass": "original", "run_id": "openrouter-paid-deepseek-v41-flash-off" }, "two_run_charge_usd": 0.00823614 }, { "accepted": 57, "accepted_error_ids": [ "DEV-005" ], "accepted_errors": 1, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-006", "DEV-013", "DEV-014" ], "question": "Cheapest pair with at most 1 accepted error and at least 56 accepted", "run_a": { "category": "general", "condition": "P0", "effort": "off", "label": "Gemma 4 31B thinking off original", "model": "google/gemma-4-31b-it", "pass": "original", "run_id": "openrouter-paid-gemma4-31b-off" }, "run_b": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on fresh1", "model": "google/gemma-4-26b-a4b-it", "pass": "fresh1", "run_id": "extended-openrouter-paid-gemma4-26b-a4b-on-fresh1-p0" }, "two_run_charge_usd": 0.02486758 }, { "accepted": 55, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 5, "deferred_ids": [ "DEV-006", "DEV-013", "DEV-027", "DEV-029", "DEV-030" ], "question": "Best native + general pair with 0 accepted errors", "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort original", "model": "deepseek/deepseek-v4.1-flash", "pass": "original", "run_id": "openrouter-paid-deepseek-v41-flash-low" }, "run_b": { "category": "decision", "condition": "P0", "effort": "na", "label": "Solar Decide fresh1", "model": "upstage/solar-decide", "pass": "fresh1", "run_id": "solar-decide-native-fresh1-p0" }, "two_run_charge_usd": 0.03720239 } ], "charged_cross_pairs": 895, "charged_runs": 43, "charged_runs_by_category": { "decision": 9, "general": 34 }, "charged_zero_error_pairs": 446, "confidence": { "note": "Counts are exact and look back at the same development reviews. Any ranking of pairs is descriptive-only, and no pair is a recommended configuration.", "tag": "solid" }, "denominator": 60, "explorer_pairs": [ { "accepted": 58, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 2, "deferred_ids": [ "DEV-005", "DEV-013" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "Qwen3.8 27B low effort (fp4) original", "model": "qwen/qwen3.8-27b", "pass": "original", "run_id": "openrouter-qwen27-low-darkbloom-fp4" }, "run_b": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on fresh1", "model": "google/gemma-4-26b-a4b-it", "pass": "fresh1", "run_id": "extended-openrouter-paid-gemma4-26b-a4b-on-fresh1-p0" }, "two_run_charge_usd": 0.06854978 }, { "accepted": 58, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 2, "deferred_ids": [ "DEV-013", "DEV-022" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "Qwen3.8 27B low effort (fp4) original", "model": "qwen/qwen3.8-27b", "pass": "original", "run_id": "openrouter-qwen27-low-darkbloom-fp4" }, "run_b": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on original", "model": "google/gemma-4-26b-a4b-it", "pass": "original", "run_id": "openrouter-paid-gemma4-26b-a4b-on" }, "two_run_charge_usd": 0.07038578 }, { "accepted": 57, "accepted_error_ids": [ "DEV-013" ], "accepted_errors": 1, "charge_basis": "unavailable: at least one run has no per-pass charge", "deferred": 3, "deferred_ids": [ "DEV-002", "DEV-014", "DEV-048" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "Qwen3.8 27B low effort (fp4) original", "model": "qwen/qwen3.8-27b", "pass": "original", "run_id": "openrouter-qwen27-low-darkbloom-fp4" }, "run_b": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on fresh2", "model": "google/gemma-4-26b-a4b-it", "pass": "fresh2", "run_id": "extended-openrouter-paid-gemma4-26b-a4b-on-fresh2-p0" }, "two_run_charge_usd": null }, { "accepted": 59, "accepted_error_ids": [ "DEV-013" ], "accepted_errors": 1, "charge_basis": "unavailable: at least one run has no per-pass charge", "deferred": 1, "deferred_ids": [ "DEV-005" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "Qwen3.8 27B low effort (fp4) original", "model": "qwen/qwen3.8-27b", "pass": "original", "run_id": "openrouter-qwen27-low-darkbloom-fp4" }, "run_b": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on fresh3", "model": "google/gemma-4-26b-a4b-it", "pass": "fresh3", "run_id": "extended-openrouter-paid-gemma4-26b-a4b-on-fresh3-p0" }, "two_run_charge_usd": null }, { "accepted": 58, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 2, "deferred_ids": [ "DEV-013", "DEV-022" ], "run_a": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on original", "model": "google/gemma-4-26b-a4b-it", "pass": "original", "run_id": "openrouter-paid-gemma4-26b-a4b-on" }, "run_b": { "category": "general", "condition": "P0", "effort": "low", "label": "Qwen3.8 27B low effort (fp4) original", "model": "qwen/qwen3.8-27b", "pass": "original", "run_id": "openrouter-qwen27-low-darkbloom-fp4" }, "two_run_charge_usd": 0.07038578 }, { "accepted": 57, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-005", "DEV-006", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort fresh1", "model": "deepseek/deepseek-v4.1-flash", "pass": "fresh1", "run_id": "extended-openrouter-paid-deepseek-v41-flash-low-fresh1-p0" }, "run_b": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on fresh1", "model": "google/gemma-4-26b-a4b-it", "pass": "fresh1", "run_id": "extended-openrouter-paid-gemma4-26b-a4b-on-fresh1-p0" }, "two_run_charge_usd": 0.03674717 }, { "accepted": 57, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-006", "DEV-022", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on original", "model": "google/gemma-4-26b-a4b-it", "pass": "original", "run_id": "openrouter-paid-gemma4-26b-a4b-on" }, "run_b": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort fresh1", "model": "deepseek/deepseek-v4.1-flash", "pass": "fresh1", "run_id": "extended-openrouter-paid-deepseek-v41-flash-low-fresh1-p0" }, "two_run_charge_usd": 0.03858317 }, { "accepted": 57, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-006", "DEV-013", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort original", "model": "deepseek/deepseek-v4.1-flash", "pass": "original", "run_id": "openrouter-paid-deepseek-v41-flash-low" }, "run_b": { "category": "general", "condition": "P0", "effort": "high", "label": "DeepSeek V4.1 Flash high effort fresh1", "model": "deepseek/deepseek-v4.1-flash", "pass": "fresh1", "run_id": "extended-openrouter-paid-deepseek-v41-flash-high-authority-v3-current-price-fresh1-p0" }, "two_run_charge_usd": 0.049460181 }, { "accepted": 57, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-006", "DEV-013", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "high", "label": "DeepSeek V4.1 Flash high effort fresh1", "model": "deepseek/deepseek-v4.1-flash", "pass": "fresh1", "run_id": "extended-openrouter-paid-deepseek-v41-flash-high-authority-v3-current-price-fresh1-p0" }, "run_b": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort fresh1", "model": "deepseek/deepseek-v4.1-flash", "pass": "fresh1", "run_id": "extended-openrouter-paid-deepseek-v41-flash-low-fresh1-p0" }, "two_run_charge_usd": 0.051759381 }, { "accepted": 57, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-006", "DEV-013", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort original", "model": "deepseek/deepseek-v4.1-flash", "pass": "original", "run_id": "openrouter-paid-deepseek-v41-flash-low" }, "run_b": { "category": "general", "condition": "P0", "effort": "low", "label": "Qwen3.8 27B low effort (fp4) original", "model": "qwen/qwen3.8-27b", "pass": "original", "run_id": "openrouter-qwen27-low-darkbloom-fp4" }, "two_run_charge_usd": 0.06437259 }, { "accepted": 57, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-006", "DEV-013", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "Qwen3.8 27B low effort (fp4) original", "model": "qwen/qwen3.8-27b", "pass": "original", "run_id": "openrouter-qwen27-low-darkbloom-fp4" }, "run_b": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort fresh1", "model": "deepseek/deepseek-v4.1-flash", "pass": "fresh1", "run_id": "extended-openrouter-paid-deepseek-v41-flash-low-fresh1-p0" }, "two_run_charge_usd": 0.06667179 }, { "accepted": 57, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-006", "DEV-013", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort original", "model": "deepseek/deepseek-v4.1-flash", "pass": "original", "run_id": "openrouter-paid-deepseek-v41-flash-low" }, "run_b": { "category": "general", "condition": "P0", "effort": "xhigh", "label": "Qwen3.8 27B xhigh effort fresh1", "model": "qwen/qwen3.8-27b", "pass": "fresh1", "run_id": "extended-openrouter-paid-qwen3.8-27b-xhigh-fresh1-p0" }, "two_run_charge_usd": 0.070539165 }, { "accepted": 57, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-005", "DEV-006", "DEV-013" ], "run_a": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on fresh1", "model": "google/gemma-4-26b-a4b-it", "pass": "fresh1", "run_id": "extended-openrouter-paid-gemma4-26b-a4b-on-fresh1-p0" }, "run_b": { "category": "general", "condition": "P0", "effort": "xhigh", "label": "Qwen3.8 27B xhigh effort fresh1", "model": "qwen/qwen3.8-27b", "pass": "fresh1", "run_id": "extended-openrouter-paid-qwen3.8-27b-xhigh-fresh1-p0" }, "two_run_charge_usd": 0.074716355 }, { "accepted": 51, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 9, "deferred_ids": [ "DEV-006", "DEV-010", "DEV-013", "DEV-014", "DEV-027", "DEV-029", "DEV-030", "DEV-053", "DEV-059" ], "run_a": { "category": "general", "condition": "P0", "effort": "off", "label": "DeepSeek V4.1 Flash thinking off original", "model": "deepseek/deepseek-v4.1-flash", "pass": "original", "run_id": "openrouter-paid-deepseek-v41-flash-off" }, "run_b": { "category": "decision", "condition": "P0", "effort": "na", "label": "jev-1.13.0 fresh1", "model": "jev-1.13.0", "pass": "fresh1", "run_id": "extended-typesafe-jev113-native-prompts-fresh1-p0" }, "two_run_charge_usd": 0.00857206 }, { "accepted": 54, "accepted_error_ids": [ "DEV-014" ], "accepted_errors": 1, "charge_basis": "observed or known provider charges", "deferred": 6, "deferred_ids": [ "DEV-005", "DEV-006", "DEV-010", "DEV-013", "DEV-030", "DEV-053" ], "run_a": { "category": "general", "condition": "P0", "effort": "off", "label": "Gemma 4 31B thinking off original", "model": "google/gemma-4-31b-it", "pass": "original", "run_id": "openrouter-paid-gemma4-31b-off" }, "run_b": { "category": "general", "condition": "P0", "effort": "off", "label": "DeepSeek V4.1 Flash thinking off original", "model": "deepseek/deepseek-v4.1-flash", "pass": "original", "run_id": "openrouter-paid-deepseek-v41-flash-off" }, "two_run_charge_usd": 0.00823614 }, { "accepted": 57, "accepted_error_ids": [ "DEV-005" ], "accepted_errors": 1, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-006", "DEV-013", "DEV-014" ], "run_a": { "category": "general", "condition": "P0", "effort": "off", "label": "Gemma 4 31B thinking off original", "model": "google/gemma-4-31b-it", "pass": "original", "run_id": "openrouter-paid-gemma4-31b-off" }, "run_b": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on fresh1", "model": "google/gemma-4-26b-a4b-it", "pass": "fresh1", "run_id": "extended-openrouter-paid-gemma4-26b-a4b-on-fresh1-p0" }, "two_run_charge_usd": 0.02486758 }, { "accepted": 55, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 5, "deferred_ids": [ "DEV-006", "DEV-013", "DEV-027", "DEV-029", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort original", "model": "deepseek/deepseek-v4.1-flash", "pass": "original", "run_id": "openrouter-paid-deepseek-v41-flash-low" }, "run_b": { "category": "decision", "condition": "P0", "effort": "na", "label": "Solar Decide fresh1", "model": "upstage/solar-decide", "pass": "fresh1", "run_id": "solar-decide-native-fresh1-p0" }, "two_run_charge_usd": 0.03720239 }, { "accepted": 56, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 4, "deferred_ids": [ "DEV-005", "DEV-006", "DEV-013", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort original", "model": "deepseek/deepseek-v4.1-flash", "pass": "original", "run_id": "openrouter-paid-deepseek-v41-flash-low" }, "run_b": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on fresh1", "model": "google/gemma-4-26b-a4b-it", "pass": "fresh1", "run_id": "extended-openrouter-paid-gemma4-26b-a4b-on-fresh1-p0" }, "two_run_charge_usd": 0.03444797 }, { "accepted": 56, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 4, "deferred_ids": [ "DEV-006", "DEV-013", "DEV-022", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on original", "model": "google/gemma-4-26b-a4b-it", "pass": "original", "run_id": "openrouter-paid-gemma4-26b-a4b-on" }, "run_b": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort original", "model": "deepseek/deepseek-v4.1-flash", "pass": "original", "run_id": "openrouter-paid-deepseek-v41-flash-low" }, "two_run_charge_usd": 0.03628397 } ], "feeds": [ "public-site/native-agreement-policy-v1.json", "public-site/disputed-reviews-v1.json", "public-site/data.json", "public-site/extended-cases-v1.json" ], "full_policy_routing": { "denominator": 60, "overlap": { "concern_and_insufficient": 0, "deferred_and_concern": 1, "deferred_and_insufficient": 3 }, "pair": "Solar Decide + Perplexity Decider", "reference_serious_concern_yes": 25, "rows": [ { "count": 7, "ids": [ "DEV-006", "DEV-013", "DEV-027", "DEV-028", "DEV-029", "DEV-030", "DEV-035" ], "step": "Deferred: the two answers differ" }, { "count": 24, "ids": [ "DEV-008", "DEV-009", "DEV-011", "DEV-012", "DEV-025", "DEV-031", "DEV-032", "DEV-036", "DEV-037", "DEV-039", "DEV-041", "DEV-043", "DEV-044", "DEV-045", "DEV-046", "DEV-047", "DEV-049", "DEV-050", "DEV-051", "DEV-052", "DEV-054", "DEV-056", "DEV-058", "DEV-059" ], "step": "Accepted, escalated: either model says serious concern" }, { "count": 4, "ids": [ "DEV-005", "DEV-022", "DEV-048", "DEV-060" ], "step": "Accepted, sent for clarification: either model says insufficient information" }, { "count": 35, "ids": [ "DEV-005", "DEV-006", "DEV-008", "DEV-009", "DEV-011", "DEV-012", "DEV-013", "DEV-022", "DEV-025", "DEV-027", "DEV-028", "DEV-029", "DEV-030", "DEV-031", "DEV-032", "DEV-035", "DEV-036", "DEV-037", "DEV-039", "DEV-041", "DEV-043", "DEV-044", "DEV-045", "DEV-046", "DEV-047", "DEV-048", "DEV-049", "DEV-050", "DEV-051", "DEV-052", "DEV-054", "DEV-056", "DEV-058", "DEV-059", "DEV-060" ], "step": "Reaches a person (union)" }, { "count": 25, "ids": [ "DEV-001", "DEV-002", "DEV-003", "DEV-004", "DEV-007", "DEV-010", "DEV-014", "DEV-015", "DEV-016", "DEV-017", "DEV-018", "DEV-019", "DEV-020", "DEV-021", "DEV-023", "DEV-024", "DEV-026", "DEV-033", "DEV-034", "DEV-038", "DEV-040", "DEV-042", "DEV-053", "DEV-055", "DEV-057" ], "step": "Accepted with no routing" } ] }, "implication": "Pick a pair for complementary failures and price, and budget for the deferred reviews; agreement still cannot catch a blind spot both models share.", "key_numbers": [ { "format": "count", "label": [ "Solar + Perplexity accepted, with ", { "format": "count", "path": "agreement_rule.solar_perplexity.accepted_errors" }, " errors" ], "of": "agreement_rule.denominator", "share": null, "value": "agreement_rule.solar_perplexity.accepted" }, { "format": "usd", "label": [ "Solar + Perplexity two-run charge" ], "of": null, "share": null, "value": "agreement_rule.solar_perplexity.two_run_charge_usd" }, { "format": "count", "label": [ { "format": "text", "path": "agreement_rule.best_beating.run_a.label" }, " + ", { "format": "text", "path": "agreement_rule.best_beating.run_b.label" }, " accepted, with ", { "format": "count", "path": "agreement_rule.best_beating.accepted_errors" }, " errors" ], "of": "agreement_rule.denominator", "share": null, "value": "agreement_rule.best_beating.accepted" }, { "format": "usd", "label": [ "Its two-run charge" ], "of": null, "share": null, "value": "agreement_rule.best_beating.two_run_charge_usd" }, { "format": "count", "label": [ "Charged pairs with no accepted errors that accept more reviews than Solar + Perplexity at a lower charge" ], "of": "agreement_rule.charged_cross_pairs", "share": null, "value": "agreement_rule.pairs_more_and_cheaper" }, { "format": "count", "label": [ "Full policy: Solar + Perplexity reviews that reach a person" ], "of": "agreement_rule.denominator", "share": null, "value": "agreement_rule.full_policy_routing.rows[3].count" } ], "loose_beat_definition": "Same rule with at least as many accepted (53 or more), which adds the ties; counted by pairs_beating_solar_perplexity", "native_routing": [ { "accepted_no_routing": 18, "clarification_beyond_deferral": 1, "concern_any": 25, "deferred": 27, "escalated_beyond_deferral": 14, "insufficient_any": 10, "left": "liquid-d1-native-fresh1-p0", "reaches_person": 42, "right": "tev1-4b-native-fresh1-p0" }, { "accepted_no_routing": 26, "clarification_beyond_deferral": 2, "concern_any": 25, "deferred": 17, "escalated_beyond_deferral": 15, "insufficient_any": 7, "left": "liquid-d1-native-fresh1-p0", "reaches_person": 34, "right": "solar-decide-native-fresh1-p0" }, { "accepted_no_routing": 27, "clarification_beyond_deferral": 2, "concern_any": 26, "deferred": 15, "escalated_beyond_deferral": 16, "insufficient_any": 5, "left": "liquid-d1-native-fresh1-p0", "reaches_person": 33, "right": "clef-openrouter-native-fresh1-p0" }, { "accepted_no_routing": 25, "clarification_beyond_deferral": 2, "concern_any": 25, "deferred": 20, "escalated_beyond_deferral": 13, "insufficient_any": 5, "left": "liquid-d1-native-fresh1-p0", "reaches_person": 35, "right": "clef-flash-openrouter-native-fresh1-p0" }, { "accepted_no_routing": 23, "clarification_beyond_deferral": 1, "concern_any": 25, "deferred": 22, "escalated_beyond_deferral": 14, "insufficient_any": 7, "left": "liquid-d1-native-fresh1-p0", "reaches_person": 37, "right": "luna-decisions-openrouter-native-fresh1-p0" }, { "accepted_no_routing": 28, "clarification_beyond_deferral": 2, "concern_any": 25, "deferred": 14, "escalated_beyond_deferral": 16, "insufficient_any": 5, "left": "liquid-d1-native-fresh1-p0", "reaches_person": 32, "right": "perplexity-decider-native-fresh1-p0" }, { "accepted_no_routing": 19, "clarification_beyond_deferral": 3, "concern_any": 25, "deferred": 15, "escalated_beyond_deferral": 23, "insufficient_any": 9, "left": "tev1-4b-native-fresh1-p0", "reaches_person": 41, "right": "solar-decide-native-fresh1-p0" }, { "accepted_no_routing": 18, "clarification_beyond_deferral": 3, "concern_any": 26, "deferred": 18, "escalated_beyond_deferral": 21, "insufficient_any": 9, "left": "tev1-4b-native-fresh1-p0", "reaches_person": 42, "right": "clef-openrouter-native-fresh1-p0" }, { "accepted_no_routing": 19, "clarification_beyond_deferral": 2, "concern_any": 25, "deferred": 21, "escalated_beyond_deferral": 18, "insufficient_any": 10, "left": "tev1-4b-native-fresh1-p0", "reaches_person": 41, "right": "clef-flash-openrouter-native-fresh1-p0" }, { "accepted_no_routing": 18, "clarification_beyond_deferral": 3, "concern_any": 25, "deferred": 18, "escalated_beyond_deferral": 21, "insufficient_any": 11, "left": "tev1-4b-native-fresh1-p0", "reaches_person": 42, "right": "luna-decisions-openrouter-native-fresh1-p0" }, { "accepted_no_routing": 19, "clarification_beyond_deferral": 3, "concern_any": 25, "deferred": 16, "escalated_beyond_deferral": 22, "insufficient_any": 10, "left": "tev1-4b-native-fresh1-p0", "reaches_person": 41, "right": "perplexity-decider-native-fresh1-p0" }, { "accepted_no_routing": 25, "clarification_beyond_deferral": 4, "concern_any": 26, "deferred": 8, "escalated_beyond_deferral": 23, "insufficient_any": 6, "left": "solar-decide-native-fresh1-p0", "reaches_person": 35, "right": "clef-openrouter-native-fresh1-p0" }, { "accepted_no_routing": 22, "clarification_beyond_deferral": 3, "concern_any": 25, "deferred": 16, "escalated_beyond_deferral": 19, "insufficient_any": 7, "left": "solar-decide-native-fresh1-p0", "reaches_person": 38, "right": "clef-flash-openrouter-native-fresh1-p0" }, { "accepted_no_routing": 23, "clarification_beyond_deferral": 3, "concern_any": 25, "deferred": 11, "escalated_beyond_deferral": 23, "insufficient_any": 8, "left": "solar-decide-native-fresh1-p0", "reaches_person": 37, "right": "luna-decisions-openrouter-native-fresh1-p0" }, { "accepted_no_routing": 25, "clarification_beyond_deferral": 4, "concern_any": 25, "deferred": 7, "escalated_beyond_deferral": 24, "insufficient_any": 7, "left": "solar-decide-native-fresh1-p0", "reaches_person": 35, "right": "perplexity-decider-native-fresh1-p0" }, { "accepted_no_routing": 25, "clarification_beyond_deferral": 3, "concern_any": 26, "deferred": 13, "escalated_beyond_deferral": 19, "insufficient_any": 5, "left": "clef-openrouter-native-fresh1-p0", "reaches_person": 35, "right": "clef-flash-openrouter-native-fresh1-p0" }, { "accepted_no_routing": 25, "clarification_beyond_deferral": 3, "concern_any": 26, "deferred": 10, "escalated_beyond_deferral": 22, "insufficient_any": 6, "left": "clef-openrouter-native-fresh1-p0", "reaches_person": 35, "right": "luna-decisions-openrouter-native-fresh1-p0" }, { "accepted_no_routing": 27, "clarification_beyond_deferral": 4, "concern_any": 26, "deferred": 7, "escalated_beyond_deferral": 22, "insufficient_any": 5, "left": "clef-openrouter-native-fresh1-p0", "reaches_person": 33, "right": "perplexity-decider-native-fresh1-p0" }, { "accepted_no_routing": 20, "clarification_beyond_deferral": 2, "concern_any": 25, "deferred": 20, "escalated_beyond_deferral": 18, "insufficient_any": 7, "left": "clef-flash-openrouter-native-fresh1-p0", "reaches_person": 40, "right": "luna-decisions-openrouter-native-fresh1-p0" }, { "accepted_no_routing": 23, "clarification_beyond_deferral": 3, "concern_any": 25, "deferred": 15, "escalated_beyond_deferral": 19, "insufficient_any": 6, "left": "clef-flash-openrouter-native-fresh1-p0", "reaches_person": 37, "right": "perplexity-decider-native-fresh1-p0" }, { "accepted_no_routing": 23, "clarification_beyond_deferral": 3, "concern_any": 25, "deferred": 12, "escalated_beyond_deferral": 22, "insufficient_any": 7, "left": "luna-decisions-openrouter-native-fresh1-p0", "reaches_person": 37, "right": "perplexity-decider-native-fresh1-p0" } ], "pairs_beating_solar_perplexity": 41, "pairs_more_and_cheaper": 25, "qwen_gemma": { "accepted": 58, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 2, "deferred_ids": [ "DEV-005", "DEV-013" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "Qwen3.8 27B low effort (fp4) original", "model": "qwen/qwen3.8-27b", "pass": "original", "run_id": "openrouter-qwen27-low-darkbloom-fp4" }, "run_b": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on fresh1", "model": "google/gemma-4-26b-a4b-it", "pass": "fresh1", "run_id": "extended-openrouter-paid-gemma4-26b-a4b-on-fresh1-p0" }, "two_run_charge_usd": 0.06854978 }, "qwen_gemma_pass_sensitivity": [ { "accepted": 58, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 2, "deferred_ids": [ "DEV-013", "DEV-022" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "Qwen3.8 27B low effort (fp4) original", "model": "qwen/qwen3.8-27b", "pass": "original", "run_id": "openrouter-qwen27-low-darkbloom-fp4" }, "run_b": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on original", "model": "google/gemma-4-26b-a4b-it", "pass": "original", "run_id": "openrouter-paid-gemma4-26b-a4b-on" }, "two_run_charge_usd": 0.07038578 }, { "accepted": 58, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 2, "deferred_ids": [ "DEV-005", "DEV-013" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "Qwen3.8 27B low effort (fp4) original", "model": "qwen/qwen3.8-27b", "pass": "original", "run_id": "openrouter-qwen27-low-darkbloom-fp4" }, "run_b": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on fresh1", "model": "google/gemma-4-26b-a4b-it", "pass": "fresh1", "run_id": "extended-openrouter-paid-gemma4-26b-a4b-on-fresh1-p0" }, "two_run_charge_usd": 0.06854978 }, { "accepted": 57, "accepted_error_ids": [ "DEV-013" ], "accepted_errors": 1, "charge_basis": "unavailable: at least one run has no per-pass charge", "deferred": 3, "deferred_ids": [ "DEV-002", "DEV-014", "DEV-048" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "Qwen3.8 27B low effort (fp4) original", "model": "qwen/qwen3.8-27b", "pass": "original", "run_id": "openrouter-qwen27-low-darkbloom-fp4" }, "run_b": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on fresh2", "model": "google/gemma-4-26b-a4b-it", "pass": "fresh2", "run_id": "extended-openrouter-paid-gemma4-26b-a4b-on-fresh2-p0" }, "two_run_charge_usd": null }, { "accepted": 59, "accepted_error_ids": [ "DEV-013" ], "accepted_errors": 1, "charge_basis": "unavailable: at least one run has no per-pass charge", "deferred": 1, "deferred_ids": [ "DEV-005" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "Qwen3.8 27B low effort (fp4) original", "model": "qwen/qwen3.8-27b", "pass": "original", "run_id": "openrouter-qwen27-low-darkbloom-fp4" }, "run_b": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on fresh3", "model": "google/gemma-4-26b-a4b-it", "pass": "fresh3", "run_id": "extended-openrouter-paid-gemma4-26b-a4b-on-fresh3-p0" }, "two_run_charge_usd": null } ], "rule": "Accept a review only when both runs return the identical valid four-field answer; otherwise defer to a person.", "scripts": [ "docs/talk/scripts/s11_agreement_general.py", "docs/talk/scripts/s12_full_policy_routing.py" ], "solar_perplexity": { "accepted": 53, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 7, "deferred_ids": [ "DEV-006", "DEV-013", "DEV-027", "DEV-028", "DEV-029", "DEV-030", "DEV-035" ], "run_a": { "category": "decision", "condition": "P0", "effort": "na", "label": "Solar Decide fresh1", "model": "upstage/solar-decide", "pass": "fresh1", "run_id": "solar-decide-native-fresh1-p0" }, "run_b": { "category": "decision", "condition": "P0", "effort": "na", "label": "Perplexity Decider fresh1", "model": "perplexity/pplx-decider-v1-27b", "pass": "fresh1", "run_id": "perplexity-decider-native-fresh1-p0" }, "two_run_charge_usd": 0.03749436 }, "title": [ "Some cheap general LLM pairs accepted more reviews than Solar + Perplexity at a lower charge" ], "top_zero_error_pairs": [ { "accepted": 58, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 2, "deferred_ids": [ "DEV-005", "DEV-013" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "Qwen3.8 27B low effort (fp4) original", "model": "qwen/qwen3.8-27b", "pass": "original", "run_id": "openrouter-qwen27-low-darkbloom-fp4" }, "run_b": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on fresh1", "model": "google/gemma-4-26b-a4b-it", "pass": "fresh1", "run_id": "extended-openrouter-paid-gemma4-26b-a4b-on-fresh1-p0" }, "two_run_charge_usd": 0.06854978 }, { "accepted": 58, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 2, "deferred_ids": [ "DEV-013", "DEV-022" ], "run_a": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on original", "model": "google/gemma-4-26b-a4b-it", "pass": "original", "run_id": "openrouter-paid-gemma4-26b-a4b-on" }, "run_b": { "category": "general", "condition": "P0", "effort": "low", "label": "Qwen3.8 27B low effort (fp4) original", "model": "qwen/qwen3.8-27b", "pass": "original", "run_id": "openrouter-qwen27-low-darkbloom-fp4" }, "two_run_charge_usd": 0.07038578 }, { "accepted": 57, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-005", "DEV-006", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort fresh1", "model": "deepseek/deepseek-v4.1-flash", "pass": "fresh1", "run_id": "extended-openrouter-paid-deepseek-v41-flash-low-fresh1-p0" }, "run_b": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on fresh1", "model": "google/gemma-4-26b-a4b-it", "pass": "fresh1", "run_id": "extended-openrouter-paid-gemma4-26b-a4b-on-fresh1-p0" }, "two_run_charge_usd": 0.03674717 }, { "accepted": 57, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-006", "DEV-022", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on original", "model": "google/gemma-4-26b-a4b-it", "pass": "original", "run_id": "openrouter-paid-gemma4-26b-a4b-on" }, "run_b": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort fresh1", "model": "deepseek/deepseek-v4.1-flash", "pass": "fresh1", "run_id": "extended-openrouter-paid-deepseek-v41-flash-low-fresh1-p0" }, "two_run_charge_usd": 0.03858317 }, { "accepted": 57, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-006", "DEV-013", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort original", "model": "deepseek/deepseek-v4.1-flash", "pass": "original", "run_id": "openrouter-paid-deepseek-v41-flash-low" }, "run_b": { "category": "general", "condition": "P0", "effort": "high", "label": "DeepSeek V4.1 Flash high effort fresh1", "model": "deepseek/deepseek-v4.1-flash", "pass": "fresh1", "run_id": "extended-openrouter-paid-deepseek-v41-flash-high-authority-v3-current-price-fresh1-p0" }, "two_run_charge_usd": 0.049460181 }, { "accepted": 57, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-006", "DEV-013", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "high", "label": "DeepSeek V4.1 Flash high effort fresh1", "model": "deepseek/deepseek-v4.1-flash", "pass": "fresh1", "run_id": "extended-openrouter-paid-deepseek-v41-flash-high-authority-v3-current-price-fresh1-p0" }, "run_b": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort fresh1", "model": "deepseek/deepseek-v4.1-flash", "pass": "fresh1", "run_id": "extended-openrouter-paid-deepseek-v41-flash-low-fresh1-p0" }, "two_run_charge_usd": 0.051759381 }, { "accepted": 57, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-006", "DEV-013", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort original", "model": "deepseek/deepseek-v4.1-flash", "pass": "original", "run_id": "openrouter-paid-deepseek-v41-flash-low" }, "run_b": { "category": "general", "condition": "P0", "effort": "low", "label": "Qwen3.8 27B low effort (fp4) original", "model": "qwen/qwen3.8-27b", "pass": "original", "run_id": "openrouter-qwen27-low-darkbloom-fp4" }, "two_run_charge_usd": 0.06437259 }, { "accepted": 57, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-006", "DEV-013", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "Qwen3.8 27B low effort (fp4) original", "model": "qwen/qwen3.8-27b", "pass": "original", "run_id": "openrouter-qwen27-low-darkbloom-fp4" }, "run_b": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort fresh1", "model": "deepseek/deepseek-v4.1-flash", "pass": "fresh1", "run_id": "extended-openrouter-paid-deepseek-v41-flash-low-fresh1-p0" }, "two_run_charge_usd": 0.06667179 }, { "accepted": 57, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-006", "DEV-013", "DEV-030" ], "run_a": { "category": "general", "condition": "P0", "effort": "low", "label": "DeepSeek V4.1 Flash low effort original", "model": "deepseek/deepseek-v4.1-flash", "pass": "original", "run_id": "openrouter-paid-deepseek-v41-flash-low" }, "run_b": { "category": "general", "condition": "P0", "effort": "xhigh", "label": "Qwen3.8 27B xhigh effort fresh1", "model": "qwen/qwen3.8-27b", "pass": "fresh1", "run_id": "extended-openrouter-paid-qwen3.8-27b-xhigh-fresh1-p0" }, "two_run_charge_usd": 0.070539165 }, { "accepted": 57, "accepted_error_ids": [], "accepted_errors": 0, "charge_basis": "observed or known provider charges", "deferred": 3, "deferred_ids": [ "DEV-005", "DEV-006", "DEV-013" ], "run_a": { "category": "general", "condition": "P0", "effort": "on", "label": "Gemma 4 26B thinking on fresh1", "model": "google/gemma-4-26b-a4b-it", "pass": "fresh1", "run_id": "extended-openrouter-paid-gemma4-26b-a4b-on-fresh1-p0" }, "run_b": { "category": "general", "condition": "P0", "effort": "xhigh", "label": "Qwen3.8 27B xhigh effort fresh1", "model": "qwen/qwen3.8-27b", "pass": "fresh1", "run_id": "extended-openrouter-paid-qwen3.8-27b-xhigh-fresh1-p0" }, "two_run_charge_usd": 0.074716355 } ] }, "calibration": { "bins_definition": [ [ 0.0, 0.5 ], [ 0.5, 0.7 ], [ 0.7, 0.9 ], [ 0.9, 0.99 ], [ 0.99, 1.0 ] ], "confidence": { "note": "Answers pooled across stages come from the same development reviews, so they are not independent. No abstention policy was run, and the two confident errors are anecdotal.", "tag": "descriptive-only" }, "feeds": [ "results/openjev/typesafe-development-v2-reconciled.jsonl", "results/clef-openrouter-v1/findings-v1/public-projection.json" ], "gap_threshold": 0.2, "hardest_count": 10, "hardest_ids": [ "DEV-030", "DEV-006", "DEV-013", "DEV-029", "DEV-005", "DEV-022", "DEV-018", "DEV-028", "DEV-059", "DEV-010" ], "implication": "If a vendor exposes two numbers, threshold on the option probability, and expect even the best-calibrated model to be confident about the wrong field on off-topic input.", "jev_confident_wrong": { "answers": [ { "chosen_probability": 0.97, "confidence": 0.96, "field": "testimonial_potential", "id": "DEV-027", "prediction": "no", "reference": "yes" }, { "chosen_probability": 0.94, "confidence": 0.91, "field": "serious_concern_reported", "id": "DEV-029", "prediction": "no", "reference": "insufficient_information" } ], "confidence_tag": "anecdotal", "stage": "direct/P0", "threshold": 0.9 }, "key_numbers": [ { "format": "ece", "label": [ "Jev expected calibration error on these ", { "format": "count", "path": "denominator" }, " reviews (pooled provider confidence)" ], "of": null, "share": null, "value": "calibration.models[0].ece_confidence" }, { "format": "count", "label": [ "Jev field answers" ], "of": null, "share": null, "value": "calibration.models[0].field_answers" }, { "format": "ece", "label": [ "Clef Flash expected calibration error" ], "of": null, "share": null, "value": "calibration.models[5].ece_confidence" }, { "format": "count", "label": [ "Clef Flash answers where confidence and option probability differ by more than ", { "format": "text", "path": "calibration.gap_threshold" } ], "of": "calibration.models[5].gap_answers", "share": "calibration.models[5].gap_over_0_2_share", "value": "calibration.models[5].gap_over_0_2" }, { "format": "count", "label": [ "Jev's least-confident reviews that are among the hardest" ], "of": "calibration.hardest_count", "share": null, "value": "calibration.models[0].least_confident_hard_overlap" } ], "models": [ { "accuracy": 0.956, "ece_chosen_probability": 0.015, "ece_confidence": 0.011, "field_answers": 720, "first_stage": "direct/P0", "gap_answers": 720, "gap_over_0_2": 4, "gap_over_0_2_share": 0.005556, "least_confident_10": [ "DEV-013", "DEV-056", "DEV-010", "DEV-029", "DEV-053", "DEV-057", "DEV-059", "DEV-030", "DEV-022", "DEV-005" ], "least_confident_hard_overlap": 7, "mean_confidence": 0.947, "model": "jev" }, { "accuracy": 0.95, "ece_chosen_probability": 0.041, "ece_confidence": 0.145, "field_answers": 2156, "first_stage": "fresh1/P0", "gap_answers": 2156, "gap_over_0_2": 525, "gap_over_0_2_share": 0.243506, "least_confident_10": [ "DEV-006", "DEV-055", "DEV-022", "DEV-030", "DEV-045", "DEV-058", "DEV-050", "DEV-023", "DEV-020", "DEV-038" ], "least_confident_hard_overlap": 3, "mean_confidence": 0.831, "model": "solar" }, { "accuracy": 0.915, "ece_chosen_probability": 0.047, "ece_confidence": 0.047, "field_answers": 2160, "first_stage": "fresh1/P0", "gap_answers": 2160, "gap_over_0_2": 0, "gap_over_0_2_share": 0.0, "least_confident_10": [ "DEV-057", "DEV-030", "DEV-008", "DEV-059", "DEV-055", "DEV-046", "DEV-017", "DEV-013", "DEV-018", "DEV-035" ], "least_confident_hard_overlap": 4, "mean_confidence": 0.897, "model": "tev" }, { "accuracy": 0.906, "ece_chosen_probability": 0.055, "ece_confidence": 0.035, "field_answers": 2160, "first_stage": "fresh1/P0", "gap_answers": 2160, "gap_over_0_2": 7, "gap_over_0_2_share": 0.003241, "least_confident_10": [ "DEV-013", "DEV-029", "DEV-043", "DEV-039", "DEV-056", "DEV-044", "DEV-041", "DEV-031", "DEV-046", "DEV-037" ], "least_confident_hard_overlap": 2, "mean_confidence": 0.939, "model": "liquid" }, { "accuracy": 0.944, "ece_chosen_probability": 0.077, "ece_confidence": 0.217, "field_answers": 2160, "first_stage": "fresh1/P0", "gap_answers": 2160, "gap_over_0_2": 702, "gap_over_0_2_share": 0.325, "least_confident_10": [ "DEV-013", "DEV-025", "DEV-018", "DEV-030", "DEV-059", "DEV-035", "DEV-053", "DEV-054", "DEV-027", "DEV-056" ], "least_confident_hard_overlap": 4, "mean_confidence": 0.727, "model": "clef" }, { "accuracy": 0.921, "ece_chosen_probability": 0.133, "ece_confidence": 0.358, "field_answers": 2156, "first_stage": "fresh1/P0", "gap_answers": 2156, "gap_over_0_2": 1192, "gap_over_0_2_share": 0.552876, "least_confident_10": [ "DEV-022", "DEV-025", "DEV-030", "DEV-013", "DEV-033", "DEV-046", "DEV-018", "DEV-027", "DEV-012", "DEV-058" ], "least_confident_hard_overlap": 4, "mean_confidence": 0.563, "model": "clef-flash" }, { "accuracy": 0.936, "ece_chosen_probability": 0.039, "ece_confidence": 0.074, "field_answers": 2160, "first_stage": "fresh1/P0", "gap_answers": 2160, "gap_over_0_2": 69, "gap_over_0_2_share": 0.031944, "least_confident_10": [ "DEV-022", "DEV-041", "DEV-056", "DEV-030", "DEV-042", "DEV-010", "DEV-012", "DEV-013", "DEV-049", "DEV-005" ], "least_confident_hard_overlap": 5, "mean_confidence": 0.876, "model": "luna-decisions" } ], "reliability_bins": { "clef-flash": [ { "accuracy": 0.841561, "correct": 733, "lower": 0.0, "mean_confidence": 0.269, "n": 871, "upper": 0.5 }, { "accuracy": 0.984334, "correct": 377, "lower": 0.5, "mean_confidence": 0.605, "n": 383, "upper": 0.7 }, { "accuracy": 0.973646, "correct": 665, "lower": 0.7, "mean_confidence": 0.793, "n": 683, "upper": 0.9 }, { "accuracy": 0.958904, "correct": 210, "lower": 0.9, "mean_confidence": 0.943, "n": 219, "upper": 0.99 }, { "accuracy": null, "correct": 0, "lower": 0.99, "mean_confidence": null, "n": 0, "upper": 1.0 } ], "jev": [ { "accuracy": 0.636364, "correct": 14, "lower": 0.0, "mean_confidence": 0.417, "n": 22, "upper": 0.5 }, { "accuracy": 0.588235, "correct": 10, "lower": 0.5, "mean_confidence": 0.601, "n": 17, "upper": 0.7 }, { "accuracy": 0.803279, "correct": 49, "lower": 0.7, "mean_confidence": 0.813, "n": 61, "upper": 0.9 }, { "accuracy": 0.963768, "correct": 133, "lower": 0.9, "mean_confidence": 0.955, "n": 138, "upper": 0.99 }, { "accuracy": 1.0, "correct": 482, "lower": 0.99, "mean_confidence": 0.998, "n": 482, "upper": 1.0 } ] }, "scripts": [ "docs/talk/scripts/s08_confidence.py" ], "title": [ "On these ", { "format": "count", "path": "denominator" }, " reviews, Jev's confidence is calibrated on average (expected calibration error ", { "format": "ece", "path": "calibration.models[0].ece_confidence" }, ", descriptive-only) and still ", { "format": "conf", "path": "calibration.jev_confident_wrong.answers[0].confidence" }, " on a wrong testimonial" ] }, "claim_rules": [ "Every number describes saved run-passes on one provisional reference, and the feed ranks no model.", "Prompts, effort and thinking are described as observed, with no causal claim.", "A missing cost stays unknown and never counts as zero; provider charges and API-equivalent estimates stay separate.", "The feed makes no speed claims.", "Run-passes reuse the same 60 reviews, so they are not independent samples." ], "corrections": { "count": 8, "date": "2026-10-08", "items": [ { "correct": [ { "format": "count", "path": "corrections.items[0].values.mixed" }, " of ", { "format": "count", "path": "corrections.items[0].values.models" }, " native decision models said mixed; ", { "format": "count", "path": "corrections.items[0].values.insufficient_information" }, " said insufficient information and ", { "format": "count", "path": "corrections.items[0].values.negative" }, " said negative." ], "id": "dev029-mixed", "said": "Five decision models called the off-topic soup review \"mixed\".", "source": "public-site/disputed-reviews-v1.json", "values": { "insufficient_information": 2, "mixed": 4, "models": 7, "negative": 1 } }, { "correct": [ { "format": "count", "path": "corrections.items[1].values.pairs" }, " of ", { "format": "count", "path": "corrections.items[1].values.of" }, " pairs kept zero accepted errors." ], "id": "zero-error-pairs", "said": "Four of 21 agreement pairs kept zero accepted errors.", "source": "public-site/native-agreement-policy-v1.json", "values": { "of": 21, "pair_names": [ "Liquid D1 + Tev 1 4B", "Liquid D1 + Solar Decide", "Solar Decide + Clef", "Solar Decide + Clef Flash", "Solar Decide + Perplexity Decider V1 27B" ], "pairs": 5 } }, { "correct": [ "The ", { "format": "count", "path": "corrections.items[2].values.pairs" }, " error-retaining pairs erred on ", { "format": "count", "path": "corrections.items[2].values.reviews" }, " distinct reviews." ], "id": "error-pair-reviews", "said": "The error-retaining pairs erred only on DEV-006, DEV-029 and DEV-030.", "source": "public-site/native-agreement-policy-v1.json", "values": { "pairs": 16, "review_ids": [ "DEV-005", "DEV-006", "DEV-013", "DEV-018", "DEV-027", "DEV-028", "DEV-029", "DEV-030", "DEV-035", "DEV-054", "DEV-056", "DEV-059" ], "reviews": 12 } }, { "correct": [ "The study cost ", { "format": "usd:4", "path": "corrections.items[3].values.total_usd" }, " in known provider charges over ", { "format": "count", "path": "corrections.items[3].values.runs" }, " runs, Clef's ", { "format": "usd:4", "path": "corrections.items[3].values.clef_known_usd" }, " included, plus up to ", { "format": "usd:2", "path": "corrections.items[3].values.unknown_upper_bound_usd" }, " unknown." ], "id": "seven-model-cost", "said": "The whole seven-model study cost under $1.20.", "source": "public-site/supplemental-decision-runs-v1.json; docs/talk/06-cost-check.md section 1", "values": { "clef_known_usd": 0.31128624, "runs": 63, "total_usd": 1.201282228, "unknown_by_model_usd": { "cloudflare/clef-flash": 0.02359296, "upstage/solar-decide": 0.1048576 }, "unknown_upper_bound_usd": 0.12845056 } }, { "correct": [ "Opus 5.5 high has ", { "format": "count", "path": "corrections.items[4].values.passes" }, " P0 passes: ", { "format": "count", "path": "corrections.items[4].values.scores[0]" }, ", ", { "format": "count", "path": "corrections.items[4].values.scores[1]" }, " and ", { "format": "count", "path": "corrections.items[4].values.scores[2]" }, " of ", { "format": "count", "path": "corrections.items[4].values.of" }, "." ], "id": "opus-passes", "said": "Opus 5.5 high has one P0 pass at 59 of 60.", "source": "docs/talk/scripts/s07_repeatability.py", "values": { "of": 60, "passes": 3, "scores": [ 59, 58, 58 ] } }, { "confidence_tag": "descriptive-only", "correct": [ "On these ", { "format": "count", "path": "corrections.items[5].values.reviews" }, " reviews, Jev's confidence is calibrated on average (expected calibration error ", { "format": "ece", "path": "corrections.items[5].values.ece" }, " over ", { "format": "count", "path": "corrections.items[5].values.field_answers" }, " pooled field answers that share the same ", { "format": "count", "path": "corrections.items[5].values.reviews" }, " texts; descriptive-only) and still ", { "format": "conf", "path": "corrections.items[5].values.confident_wrong[0].confidence" }, " on a wrong testimonial (", { "format": "text", "path": "corrections.items[5].values.confident_wrong[0].id" }, ") and ", { "format": "conf", "path": "corrections.items[5].values.confident_wrong[1].confidence" }, " on ", { "format": "text", "path": "corrections.items[5].values.confident_wrong[1].id" }, " serious concern, where it answered \"no\" and the reference is insufficient information." ], "id": "jev-calibration", "said": "Jev's confidence is not calibrated, and it called the soup review a serious concern at 0.91.", "source": "docs/talk/scripts/s08_confidence.py", "values": { "confident_wrong": [ { "chosen_probability": 0.97, "confidence": 0.96, "field": "testimonial_potential", "id": "DEV-027", "prediction": "no", "reference": "yes" }, { "chosen_probability": 0.94, "confidence": 0.91, "field": "serious_concern_reported", "id": "DEV-029", "prediction": "no", "reference": "insufficient_information" } ], "ece": 0.011, "field_answers": 720, "reviews": 60 } }, { "correct": [ { "format": "count", "path": "corrections.items[6].values.run_passes" }, " of ", { "format": "count", "path": "corrections.items[6].values.of" }, " strong run-passes miss nothing outside the three disputed labels; ", { "format": "count", "path": "corrections.items[6].values.top3" }, " counts only the three most common miss sets." ], "id": "miss-only-disputed", "said": "235 of 373 strong run-passes miss nothing outside the three disputed labels.", "source": "docs/talk/scripts/s09_quirks.py", "values": { "of": 373, "run_passes": 270, "top3": 235 } }, { "correct": [ "A definite label is given on ", { "format": "pct", "path": "corrections.items[7].values.follow" }, ", ", { "format": "pct", "path": "corrections.items[7].values.concern" }, " and ", { "format": "pct", "path": "corrections.items[7].values.testimonial" }, " of valid answers; the single nearest label, \"no\", takes ", { "format": "pct", "path": "corrections.items[7].values.follow_no" }, ", ", { "format": "pct", "path": "corrections.items[7].values.concern_no" }, " and ", { "format": "pct", "path": "corrections.items[7].values.testimonial_no" }, "." ], "id": "insufficient-range", "said": "Reference \"insufficient\" is answered as a definite label 22% to 26% of the time on follow-up, serious concern and testimonial.", "source": "docs/talk/scripts/s02_confusion.py", "values": { "concern": 0.311524, "concern_no": 0.258591, "follow": 0.248688, "follow_no": 0.220357, "testimonial": 0.263379, "testimonial_no": 0.213012 } } ] }, "cost_frontier": { "confidence": { "note": "Charges are exact and source-bound; frontier membership is descriptive-only; the gate-clearing pass is one pass, so the selection is anecdotal.", "tag": "solid" }, "denominator": 60, "feeds": [ "public-site/data.json", "public-site/extended-run-catalog-v1.json", "public-site/supplemental-decision-runs-v1.json", "public-site/subscription-price-estimates.json" ], "frontier": [ { "all_four": 49, "category": "general", "condition": "P0", "cost_kind": "known", "effort": "off", "label": "DeepSeek V4.1 Flash thinking off fresh2", "model": "deepseek/deepseek-v4.1-flash", "pass": "fresh2", "run_id": "extended-openrouter-paid-deepseek-v41-flash-off-fresh2-p0", "usd": 0.00265654, "valid": 60 }, { "all_four": 54, "category": "general", "condition": "P0", "cost_kind": "observed", "effort": "off", "label": "DeepSeek V4.1 Flash thinking off original", "model": "deepseek/deepseek-v4.1-flash", "pass": "original", "run_id": "openrouter-paid-deepseek-v41-flash-off", "usd": 0.00268114, "valid": 60 }, { "all_four": 55, "category": "general", "condition": "P0", "cost_kind": "known", "effort": "off", "label": "Gemma 4 31B thinking off repeat2", "model": "google/gemma-4-31b-it", "pass": "repeat2", "run_id": "extended-openrouter-paid-gemma4-31b-off-repeat2-p0", "usd": 0.0054888, "valid": 60 }, { "all_four": 56, "category": "general", "condition": "P0", "cost_kind": "observed", "effort": "off", "label": "Gemma 4 31B thinking off original", "model": "google/gemma-4-31b-it", "pass": "original", "run_id": "openrouter-paid-gemma4-31b-off", "usd": 0.005555, "valid": 60 }, { "all_four": 57, "category": "general", "condition": "P1", "cost_kind": "known", "effort": "low", "label": "DeepSeek V4.1 Flash low effort fresh1", "model": "deepseek/deepseek-v4.1-flash", "pass": "fresh1", "run_id": "extended-openrouter-paid-deepseek-v41-flash-low-fresh1-p1", "usd": 0.01165805, "valid": 60 }, { "all_four": 58, "category": "general", "condition": "P0", "cost_kind": "known", "effort": "on", "label": "Gemma 4 31B thinking on repeat3", "model": "google/gemma-4-31b-it", "pass": "repeat3", "run_id": "extended-openrouter-paid-gemma4-31b-on-repeat3-p0", "usd": 0.01292744, "valid": 60 }, { "all_four": 59, "category": "general", "condition": "P0", "cost_kind": "known", "effort": "on", "label": "Gemma 4 26B thinking on fresh1", "model": "google/gemma-4-26b-a4b-it", "pass": "fresh1", "run_id": "extended-openrouter-paid-gemma4-26b-a4b-on-fresh1-p0", "usd": 0.01931258, "valid": 60 } ], "frontier_points": 7, "gate": { "confidence_tag": "anecdotal", "definition": "serious concern recall 25 of 25 with 0 false positives, and all 10 reference insufficient_information cells matched", "passes": [ { "all_four": 54, "concern_tp_fp_fn": [ 25, 0, 0 ], "cost_kind": "observed", "insufficient_cells": 10, "pass": "original", "run_id": "openrouter-paid-deepseek-v41-flash-off", "testimonial_tp_fp_fn": [ 6, 0, 3 ], "usd": 0.00268114 }, { "all_four": 48, "concern_tp_fp_fn": [ 25, 0, 0 ], "cost_kind": "known", "insufficient_cells": 6, "pass": "fresh1", "run_id": "extended-openrouter-paid-deepseek-v41-flash-off-fresh1-p0", "testimonial_tp_fp_fn": [ 9, 4, 0 ], "usd": 0.0027523 }, { "all_four": 49, "concern_tp_fp_fn": [ 25, 0, 0 ], "cost_kind": "known", "insufficient_cells": 7, "pass": "fresh2", "run_id": "extended-openrouter-paid-deepseek-v41-flash-off-fresh2-p0", "testimonial_tp_fp_fn": [ 9, 4, 0 ], "usd": 0.00265654 }, { "all_four": 48, "concern_tp_fp_fn": [ 25, 0, 0 ], "cost_kind": "known", "insufficient_cells": 7, "pass": "fresh3", "run_id": "extended-openrouter-paid-deepseek-v41-flash-off-fresh3-p0", "testimonial_tp_fp_fn": [ 9, 4, 0 ], "usd": 0.00273904 } ], "qualifying_charged_run_passes": 22 }, "implication": "Select a configuration only after the gate holds across three passes.", "key_numbers": [ { "format": "count", "label": [ "Observed or known-charge frontier points" ], "of": "cost_frontier.run_passes.observed_or_known", "share": null, "value": "cost_frontier.frontier_points" }, { "format": "count", "label": [ "Cheapest gate-clearing pass, all four fields" ], "of": "cost_frontier.denominator", "share": null, "value": "cost_frontier.gate.passes[0].all_four" }, { "format": "usd", "label": [ "Its charge for ", { "format": "count", "path": "denominator" }, " reviews" ], "of": null, "share": null, "value": "cost_frontier.gate.passes[0].usd" }, { "format": "count", "label": [ "Same configuration, first fresh pass" ], "of": "cost_frontier.denominator", "share": null, "value": "cost_frontier.gate.passes[1].all_four" } ], "run_passes": { "estimate": 261, "observed_or_known": 304, "unknown": 439 }, "scripts": [ "docs/talk/scripts/s10_cost.py" ], "title": [ "The cheapest gate-clearing pass failed the insufficient-information gate when its configuration ran again" ], "unknown_cost_rule": "Unknown-cost run-passes stay unknown and are never plotted at zero." }, "denominator": 60, "determinism": { "confidence": { "note": "Counts are exact over every configuration with three passes. Any serving-stack explanation is descriptive-only.", "tag": "solid" }, "configurations": { "decision": 50, "general": 202 }, "configurations_total": 252, "feeds": [ "public-site/data.json", "public-site/extended-cases-v1.json", "public-site/additional-cases-v1.json" ], "implication": "A decision model gives a fixed wrong list you can audit once; a general LLM gives a different wrong list each batch, so monitoring should count per-review flips, since totals can stay flat while answers change.", "key_numbers": [ { "format": "count", "label": [ "Decision configurations with no changed answer across three passes" ], "of": "determinism.configurations.decision", "share": null, "value": "determinism.zero_change.decision" }, { "format": "count", "label": [ "General configurations with no changed answer" ], "of": "determinism.configurations.general", "share": null, "value": "determinism.zero_change.general" }, { "format": "count", "label": [ "Perfectly stable configurations at ", { "format": "count", "path": "determinism.stable_threshold" }, " or better, all general LLMs" ], "of": null, "share": null, "value": "determinism.stable_at_55_or_better" }, { "format": "count", "label": [ "Most-flipped reviews that are also among the hardest" ], "of": "determinism.unstable_hard_overlap.of", "share": null, "value": "determinism.unstable_hard_overlap.count" } ], "scripts": [ "docs/talk/scripts/s07_repeatability.py" ], "stable_at_55_categories": { "general": 7 }, "stable_at_55_or_better": 7, "stable_but_wrong": [ { "all_four": 58, "category": "general", "condition": "P2", "configuration": "openrouter-paid-gemma4-31b-on", "effort": "on", "family": "gemma-31b", "fixed_wrong": 2, "fixed_wrong_ids": [ "DEV-006", "DEV-013" ], "passes": 3 }, { "all_four": 58, "category": "general", "condition": "P1", "configuration": "codex-gpt-6-astra-medium", "effort": "medium", "family": "gpt-6-astra", "fixed_wrong": 2, "fixed_wrong_ids": [ "DEV-006", "DEV-013" ], "passes": 3 }, { "all_four": 58, "category": "general", "condition": "P1", "configuration": "codex-gpt-6-luna-xhigh-batch10", "effort": "xhigh", "family": "gpt-6-luna", "fixed_wrong": 2, "fixed_wrong_ids": [ "DEV-006", "DEV-030" ], "passes": 3 }, { "all_four": 57, "category": "general", "condition": "P2", "configuration": "gemini31-pro-preview-low-p0-openrouter-v3", "effort": "low", "family": "gemini-3.1-pro", "fixed_wrong": 3, "fixed_wrong_ids": [ "DEV-010", "DEV-013", "DEV-030" ], "passes": 3 }, { "all_four": 57, "category": "general", "condition": "P0", "configuration": "codex-gpt-5.6-sol-xhigh", "effort": "xhigh", "family": "gpt-5.6-sol", "fixed_wrong": 3, "fixed_wrong_ids": [ "DEV-006", "DEV-013", "DEV-030" ], "passes": 3 }, { "all_four": 57, "category": "general", "condition": "P2", "configuration": "codex-gpt-6-astra-xhigh", "effort": "xhigh", "family": "gpt-6-astra", "fixed_wrong": 3, "fixed_wrong_ids": [ "DEV-006", "DEV-013", "DEV-030" ], "passes": 3 }, { "all_four": 55, "category": "general", "condition": "P1", "configuration": "openrouter-paid-qwen3.8-27b-off", "effort": "off", "family": "qwen-27b", "fixed_wrong": 5, "fixed_wrong_ids": [ "DEV-010", "DEV-013", "DEV-014", "DEV-015", "DEV-053" ], "passes": 3 }, { "all_four": 54, "category": "decision", "condition": "P0", "configuration": "clef-openrouter", "effort": "na", "family": "clef", "fixed_wrong": 6, "fixed_wrong_ids": [ "DEV-006", "DEV-029", "DEV-030", "DEV-053", "DEV-054", "DEV-056" ], "passes": 3 }, { "all_four": 54, "category": "decision", "condition": "P0", "configuration": "perplexity-decider", "effort": "na", "family": "perplexity", "fixed_wrong": 6, "fixed_wrong_ids": [ "DEV-006", "DEV-013", "DEV-028", "DEV-029", "DEV-030", "DEV-035" ], "passes": 3 }, { "all_four": 54, "category": "decision", "condition": "P1", "configuration": "perplexity-decider", "effort": "na", "family": "perplexity", "fixed_wrong": 6, "fixed_wrong_ids": [ "DEV-006", "DEV-013", "DEV-028", "DEV-029", "DEV-030", "DEV-035" ], "passes": 3 }, { "all_four": 54, "category": "decision", "condition": "P2", "configuration": "perplexity-decider", "effort": "na", "family": "perplexity", "fixed_wrong": 6, "fixed_wrong_ids": [ "DEV-006", "DEV-013", "DEV-028", "DEV-029", "DEV-030", "DEV-035" ], "passes": 3 }, { "all_four": 54, "category": "general", "condition": "P2", "configuration": "openrouter-paid-qwen3.8-27b-off", "effort": "off", "family": "qwen-27b", "fixed_wrong": 6, "fixed_wrong_ids": [ "DEV-010", "DEV-013", "DEV-014", "DEV-015", "DEV-030", "DEV-053" ], "passes": 3 }, { "all_four": 53, "category": "decision", "condition": "P0", "configuration": "openjev-fixed", "effort": "na", "family": "openjev", "fixed_wrong": 7, "fixed_wrong_ids": [ "DEV-005", "DEV-006", "DEV-008", "DEV-011", "DEV-029", "DEV-030", "DEV-059" ], "passes": 3 }, { "all_four": 52, "category": "decision", "condition": "P0", "configuration": "openjev-adaptive", "effort": "na", "family": "openjev", "fixed_wrong": 8, "fixed_wrong_ids": [ "DEV-005", "DEV-006", "DEV-008", "DEV-011", "DEV-019", "DEV-020", "DEV-029", "DEV-030" ], "passes": 3 }, { "all_four": 51, "category": "decision", "condition": "P1", "configuration": "clef-openrouter", "effort": "na", "family": "clef", "fixed_wrong": 9, "fixed_wrong_ids": [ "DEV-006", "DEV-013", "DEV-014", "DEV-027", "DEV-029", "DEV-030", "DEV-035", "DEV-053", "DEV-054" ], "passes": 3 }, { "all_four": 51, "category": "decision", "condition": "P1", "configuration": "luna-decisions-openrouter", "effort": "na", "family": "luna-decisions", "fixed_wrong": 9, "fixed_wrong_ids": [ "DEV-001", "DEV-005", "DEV-013", "DEV-014", "DEV-026", "DEV-029", "DEV-030", "DEV-053", "DEV-057" ], "passes": 3 }, { "all_four": 49, "category": "decision", "condition": "P2", "configuration": "clef-openrouter", "effort": "na", "family": "clef", "fixed_wrong": 11, "fixed_wrong_ids": [ "DEV-006", "DEV-013", "DEV-014", "DEV-018", "DEV-027", "DEV-029", "DEV-030", "DEV-035", "DEV-053", "DEV-054", "DEV-056" ], "passes": 3 }, { "all_four": 49, "category": "decision", "condition": "P1", "configuration": "kev-openrouter-native-prompts", "effort": "na", "family": "kev", "fixed_wrong": 11, "fixed_wrong_ids": [ "DEV-001", "DEV-004", "DEV-006", "DEV-013", "DEV-027", "DEV-028", "DEV-029", "DEV-030", "DEV-033", "DEV-057", "DEV-059" ], "passes": 3 }, { "all_four": 49, "category": "decision", "condition": "P0", "configuration": "luna-decisions-openrouter", "effort": "na", "family": "luna-decisions", "fixed_wrong": 11, "fixed_wrong_ids": [ "DEV-001", "DEV-006", "DEV-013", "DEV-014", "DEV-022", "DEV-029", "DEV-030", "DEV-041", "DEV-053", "DEV-056", "DEV-057" ], "passes": 3 }, { "all_four": 49, "category": "decision", "condition": "P2", "configuration": "luna-decisions-openrouter", "effort": "na", "family": "luna-decisions", "fixed_wrong": 11, "fixed_wrong_ids": [ "DEV-001", "DEV-005", "DEV-010", "DEV-013", "DEV-014", "DEV-027", "DEV-029", "DEV-030", "DEV-041", "DEV-053", "DEV-057" ], "passes": 3 }, { "all_four": 47, "category": "decision", "condition": "P1", "configuration": "clef-flash-openrouter", "effort": "na", "family": "clef-flash", "fixed_wrong": 13, "fixed_wrong_ids": [ "DEV-005", "DEV-006", "DEV-017", "DEV-018", "DEV-020", "DEV-025", "DEV-029", "DEV-030", "DEV-033", "DEV-035", "DEV-056", "DEV-058", "DEV-059" ], "passes": 3 }, { "all_four": 47, "category": "decision", "condition": "P1", "configuration": "cloudflare-clef-flash-direct", "effort": "na", "family": "clef-flash", "fixed_wrong": 13, "fixed_wrong_ids": [ "DEV-005", "DEV-006", "DEV-017", "DEV-018", "DEV-020", "DEV-025", "DEV-029", "DEV-030", "DEV-033", "DEV-035", "DEV-056", "DEV-058", "DEV-059" ], "passes": 3 }, { "all_four": 46, "category": "decision", "condition": "P2", "configuration": "cloudflare-clef-flash-direct", "effort": "na", "family": "clef-flash", "fixed_wrong": 14, "fixed_wrong_ids": [ "DEV-005", "DEV-006", "DEV-017", "DEV-018", "DEV-020", "DEV-025", "DEV-027", "DEV-029", "DEV-030", "DEV-032", "DEV-033", "DEV-035", "DEV-056", "DEV-059" ], "passes": 3 }, { "all_four": 46, "category": "decision", "condition": "P2", "configuration": "kev-openrouter-native-prompts", "effort": "na", "family": "kev", "fixed_wrong": 14, "fixed_wrong_ids": [ "DEV-004", "DEV-005", "DEV-006", "DEV-013", "DEV-022", "DEV-027", "DEV-028", "DEV-029", "DEV-030", "DEV-033", "DEV-035", "DEV-041", "DEV-057", "DEV-059" ], "passes": 3 }, { "all_four": 45, "category": "decision", "condition": "P0", "configuration": "clef-flash-openrouter", "effort": "na", "family": "clef-flash", "fixed_wrong": 15, "fixed_wrong_ids": [ "DEV-005", "DEV-006", "DEV-017", "DEV-018", "DEV-020", "DEV-025", "DEV-027", "DEV-029", "DEV-030", "DEV-033", "DEV-035", "DEV-044", "DEV-056", "DEV-058", "DEV-059" ], "passes": 3 }, { "all_four": 45, "category": "decision", "condition": "P0", "configuration": "tev1-4b", "effort": "na", "family": "tev", "fixed_wrong": 15, "fixed_wrong_ids": [ "DEV-001", "DEV-002", "DEV-004", "DEV-013", "DEV-018", "DEV-020", "DEV-022", "DEV-024", "DEV-027", "DEV-028", "DEV-029", "DEV-030", "DEV-044", "DEV-046", "DEV-057" ], "passes": 3 }, { "all_four": 44, "category": "decision", "condition": "P1", "configuration": "tev1-4b", "effort": "na", "family": "tev", "fixed_wrong": 16, "fixed_wrong_ids": [ "DEV-001", "DEV-002", "DEV-004", "DEV-008", "DEV-013", "DEV-018", "DEV-020", "DEV-022", "DEV-024", "DEV-027", "DEV-028", "DEV-029", "DEV-030", "DEV-044", "DEV-046", "DEV-057" ], "passes": 3 }, { "all_four": 44, "category": "decision", "condition": "P2", "configuration": "tev1-4b", "effort": "na", "family": "tev", "fixed_wrong": 16, "fixed_wrong_ids": [ "DEV-001", "DEV-004", "DEV-006", "DEV-008", "DEV-013", "DEV-018", "DEV-020", "DEV-022", "DEV-024", "DEV-027", "DEV-028", "DEV-029", "DEV-030", "DEV-044", "DEV-057", "DEV-059" ], "passes": 3 }, { "all_four": 43, "category": "decision", "condition": "P0", "configuration": "liquid-d1", "effort": "na", "family": "liquid", "fixed_wrong": 17, "fixed_wrong_ids": [ "DEV-005", "DEV-006", "DEV-013", "DEV-027", "DEV-029", "DEV-030", "DEV-031", "DEV-032", "DEV-035", "DEV-036", "DEV-037", "DEV-039", "DEV-046", "DEV-054", "DEV-056", "DEV-059", "DEV-060" ], "passes": 3 }, { "all_four": 43, "category": "decision", "condition": "P2", "configuration": "semif-generated-fresh-v1", "effort": "na", "family": "semif", "fixed_wrong": 17, "fixed_wrong_ids": [ "DEV-004", "DEV-005", "DEV-010", "DEV-013", "DEV-014", "DEV-018", "DEV-019", "DEV-020", "DEV-022", "DEV-027", "DEV-029", "DEV-030", "DEV-038", "DEV-053", "DEV-055", "DEV-058", "DEV-060" ], "passes": 3 }, { "all_four": 41, "category": "decision", "condition": "P2", "configuration": "liquid-d1", "effort": "na", "family": "liquid", "fixed_wrong": 19, "fixed_wrong_ids": [ "DEV-005", "DEV-006", "DEV-013", "DEV-027", "DEV-029", "DEV-030", "DEV-031", "DEV-032", "DEV-035", "DEV-036", "DEV-037", "DEV-039", "DEV-041", "DEV-043", "DEV-046", "DEV-054", "DEV-056", "DEV-059", "DEV-060" ], "passes": 3 }, { "all_four": 39, "category": "decision", "condition": "P0", "configuration": "alex-openjev4b-native-p0-v1", "effort": "na", "family": "alex-openjev", "fixed_wrong": 21, "fixed_wrong_ids": [ "DEV-004", "DEV-005", "DEV-006", "DEV-008", "DEV-009", "DEV-013", "DEV-017", "DEV-018", "DEV-020", "DEV-021", "DEV-022", "DEV-029", "DEV-030", "DEV-031", "DEV-032", "DEV-041", "DEV-050", "DEV-053", "DEV-054", "DEV-057", "DEV-058" ], "passes": 3 }, { "all_four": 35, "category": "decision", "condition": "P0", "configuration": "semif-generated-fresh-v1", "effort": "na", "family": "semif", "fixed_wrong": 25, "fixed_wrong_ids": [ "DEV-001", "DEV-002", "DEV-004", "DEV-005", "DEV-006", "DEV-010", "DEV-013", "DEV-014", "DEV-018", "DEV-019", "DEV-020", "DEV-022", "DEV-027", "DEV-029", "DEV-030", "DEV-038", "DEV-040", "DEV-044", "DEV-049", "DEV-051", "DEV-053", "DEV-054", "DEV-055", "DEV-058", "DEV-060" ], "passes": 3 }, { "all_four": 26, "category": "decision", "condition": "P1", "configuration": "semif-generated-fresh-v1", "effort": "na", "family": "semif", "fixed_wrong": 34, "fixed_wrong_ids": [ "DEV-001", "DEV-003", "DEV-004", "DEV-005", "DEV-006", "DEV-009", "DEV-010", "DEV-014", "DEV-016", "DEV-017", "DEV-018", "DEV-019", "DEV-020", "DEV-022", "DEV-024", "DEV-026", "DEV-027", "DEV-029", "DEV-030", "DEV-031", "DEV-036", "DEV-038", "DEV-040", "DEV-042", "DEV-043", "DEV-044", "DEV-049", "DEV-050", "DEV-051", "DEV-053", "DEV-054", "DEV-055", "DEV-058", "DEV-060" ], "passes": 3 }, { "all_four": 6, "category": "decision", "condition": "P0", "configuration": "anyjev-qwen06-l1-direct-native-cv5", "effort": "na", "family": "anyjev", "fixed_wrong": 54, "fixed_wrong_ids": [ "DEV-001", "DEV-002", "DEV-004", "DEV-005", "DEV-006", "DEV-007", "DEV-008", "DEV-009", "DEV-010", "DEV-011", "DEV-012", "DEV-013", "DEV-014", "DEV-015", "DEV-016", "DEV-017", "DEV-018", "DEV-019", "DEV-020", "DEV-021", "DEV-022", "DEV-023", "DEV-024", "DEV-025", "DEV-026", "DEV-027", "DEV-028", "DEV-029", "DEV-030", "DEV-031", "DEV-032", "DEV-035", "DEV-037", "DEV-038", "DEV-039", "DEV-040", "DEV-041", "DEV-042", "DEV-045", "DEV-046", "DEV-047", "DEV-048", "DEV-049", "DEV-050", "DEV-051", "DEV-052", "DEV-053", "DEV-054", "DEV-055", "DEV-056", "DEV-057", "DEV-058", "DEV-059", "DEV-060" ], "passes": 3 }, { "all_four": 3, "category": "decision", "condition": "P0", "configuration": "alex-openjev08-native-p0-v1", "effort": "na", "family": "alex-openjev", "fixed_wrong": 57, "fixed_wrong_ids": [ "DEV-001", "DEV-002", "DEV-003", "DEV-004", "DEV-005", "DEV-006", "DEV-007", "DEV-008", "DEV-009", "DEV-011", "DEV-012", "DEV-013", "DEV-014", "DEV-015", "DEV-017", "DEV-018", "DEV-019", "DEV-020", "DEV-021", "DEV-022", "DEV-023", "DEV-024", "DEV-025", "DEV-026", "DEV-027", "DEV-028", "DEV-029", "DEV-030", "DEV-031", "DEV-032", "DEV-033", "DEV-034", "DEV-035", "DEV-036", "DEV-037", "DEV-039", "DEV-040", "DEV-041", "DEV-042", "DEV-043", "DEV-044", "DEV-045", "DEV-046", "DEV-047", "DEV-048", "DEV-049", "DEV-050", "DEV-051", "DEV-052", "DEV-053", "DEV-054", "DEV-055", "DEV-056", "DEV-057", "DEV-058", "DEV-059", "DEV-060" ], "passes": 3 }, { "all_four": 1, "category": "decision", "condition": "P2", "configuration": "anyjev-qwen06-generated-fresh-three", "effort": "na", "family": "anyjev", "fixed_wrong": 59, "fixed_wrong_ids": [ "DEV-001", "DEV-002", "DEV-003", "DEV-004", "DEV-005", "DEV-006", "DEV-007", "DEV-008", "DEV-009", "DEV-010", "DEV-011", "DEV-012", "DEV-013", "DEV-014", "DEV-015", "DEV-016", "DEV-017", "DEV-018", "DEV-019", "DEV-020", "DEV-021", "DEV-022", "DEV-023", "DEV-024", "DEV-025", "DEV-026", "DEV-027", "DEV-028", "DEV-029", "DEV-030", "DEV-031", "DEV-032", "DEV-033", "DEV-034", "DEV-035", "DEV-036", "DEV-037", "DEV-038", "DEV-040", "DEV-041", "DEV-042", "DEV-043", "DEV-044", "DEV-045", "DEV-046", "DEV-047", "DEV-048", "DEV-049", "DEV-050", "DEV-051", "DEV-052", "DEV-053", "DEV-054", "DEV-055", "DEV-056", "DEV-057", "DEV-058", "DEV-059", "DEV-060" ], "passes": 3 }, { "all_four": 0, "category": "decision", "condition": "P0", "configuration": "anyjev-qwen06-generated-fresh-three", "effort": "na", "family": "anyjev", "fixed_wrong": 60, "fixed_wrong_ids": [ "DEV-001", "DEV-002", "DEV-003", "DEV-004", "DEV-005", "DEV-006", "DEV-007", "DEV-008", "DEV-009", "DEV-010", "DEV-011", "DEV-012", "DEV-013", "DEV-014", "DEV-015", "DEV-016", "DEV-017", "DEV-018", "DEV-019", "DEV-020", "DEV-021", "DEV-022", "DEV-023", "DEV-024", "DEV-025", "DEV-026", "DEV-027", "DEV-028", "DEV-029", "DEV-030", "DEV-031", "DEV-032", "DEV-033", "DEV-034", "DEV-035", "DEV-036", "DEV-037", "DEV-038", "DEV-039", "DEV-040", "DEV-041", "DEV-042", "DEV-043", "DEV-044", "DEV-045", "DEV-046", "DEV-047", "DEV-048", "DEV-049", "DEV-050", "DEV-051", "DEV-052", "DEV-053", "DEV-054", "DEV-055", "DEV-056", "DEV-057", "DEV-058", "DEV-059", "DEV-060" ], "passes": 3 }, { "all_four": 0, "category": "decision", "condition": "P1", "configuration": "anyjev-qwen06-generated-fresh-three", "effort": "na", "family": "anyjev", "fixed_wrong": 60, "fixed_wrong_ids": [ "DEV-001", "DEV-002", "DEV-003", "DEV-004", "DEV-005", "DEV-006", "DEV-007", "DEV-008", "DEV-009", "DEV-010", "DEV-011", "DEV-012", "DEV-013", "DEV-014", "DEV-015", "DEV-016", "DEV-017", "DEV-018", "DEV-019", "DEV-020", "DEV-021", "DEV-022", "DEV-023", "DEV-024", "DEV-025", "DEV-026", "DEV-027", "DEV-028", "DEV-029", "DEV-030", "DEV-031", "DEV-032", "DEV-033", "DEV-034", "DEV-035", "DEV-036", "DEV-037", "DEV-038", "DEV-039", "DEV-040", "DEV-041", "DEV-042", "DEV-043", "DEV-044", "DEV-045", "DEV-046", "DEV-047", "DEV-048", "DEV-049", "DEV-050", "DEV-051", "DEV-052", "DEV-053", "DEV-054", "DEV-055", "DEV-056", "DEV-057", "DEV-058", "DEV-059", "DEV-060" ], "passes": 3 }, { "all_four": 0, "category": "general", "condition": "P0", "configuration": "qwen3-0.6b-q4km-nonthinking", "effort": "na", "family": "qwen-0.6b", "fixed_wrong": 60, "fixed_wrong_ids": [ "DEV-001", "DEV-002", "DEV-003", "DEV-004", "DEV-005", "DEV-006", "DEV-007", "DEV-008", "DEV-009", "DEV-010", "DEV-011", "DEV-012", "DEV-013", "DEV-014", "DEV-015", "DEV-016", "DEV-017", "DEV-018", "DEV-019", "DEV-020", "DEV-021", "DEV-022", "DEV-023", "DEV-024", "DEV-025", "DEV-026", "DEV-027", "DEV-028", "DEV-029", "DEV-030", "DEV-031", "DEV-032", "DEV-033", "DEV-034", "DEV-035", "DEV-036", "DEV-037", "DEV-038", "DEV-039", "DEV-040", "DEV-041", "DEV-042", "DEV-043", "DEV-044", "DEV-045", "DEV-046", "DEV-047", "DEV-048", "DEV-049", "DEV-050", "DEV-051", "DEV-052", "DEV-053", "DEV-054", "DEV-055", "DEV-056", "DEV-057", "DEV-058", "DEV-059", "DEV-060" ], "passes": 3 }, { "all_four": 0, "category": "general", "condition": "P1", "configuration": "qwen3-0.6b-q4km-nonthinking", "effort": "na", "family": "qwen-0.6b", "fixed_wrong": 60, "fixed_wrong_ids": [ "DEV-001", "DEV-002", "DEV-003", "DEV-004", "DEV-005", "DEV-006", "DEV-007", "DEV-008", "DEV-009", "DEV-010", "DEV-011", "DEV-012", "DEV-013", "DEV-014", "DEV-015", "DEV-016", "DEV-017", "DEV-018", "DEV-019", "DEV-020", "DEV-021", "DEV-022", "DEV-023", "DEV-024", "DEV-025", "DEV-026", "DEV-027", "DEV-028", "DEV-029", "DEV-030", "DEV-031", "DEV-032", "DEV-033", "DEV-034", "DEV-035", "DEV-036", "DEV-037", "DEV-038", "DEV-039", "DEV-040", "DEV-041", "DEV-042", "DEV-043", "DEV-044", "DEV-045", "DEV-046", "DEV-047", "DEV-048", "DEV-049", "DEV-050", "DEV-051", "DEV-052", "DEV-053", "DEV-054", "DEV-055", "DEV-056", "DEV-057", "DEV-058", "DEV-059", "DEV-060" ], "passes": 3 }, { "all_four": 0, "category": "general", "condition": "P2", "configuration": "qwen3-0.6b-q4km-nonthinking", "effort": "na", "family": "qwen-0.6b", "fixed_wrong": 60, "fixed_wrong_ids": [ "DEV-001", "DEV-002", "DEV-003", "DEV-004", "DEV-005", "DEV-006", "DEV-007", "DEV-008", "DEV-009", "DEV-010", "DEV-011", "DEV-012", "DEV-013", "DEV-014", "DEV-015", "DEV-016", "DEV-017", "DEV-018", "DEV-019", "DEV-020", "DEV-021", "DEV-022", "DEV-023", "DEV-024", "DEV-025", "DEV-026", "DEV-027", "DEV-028", "DEV-029", "DEV-030", "DEV-031", "DEV-032", "DEV-033", "DEV-034", "DEV-035", "DEV-036", "DEV-037", "DEV-038", "DEV-039", "DEV-040", "DEV-041", "DEV-042", "DEV-043", "DEV-044", "DEV-045", "DEV-046", "DEV-047", "DEV-048", "DEV-049", "DEV-050", "DEV-051", "DEV-052", "DEV-053", "DEV-054", "DEV-055", "DEV-056", "DEV-057", "DEV-058", "DEV-059", "DEV-060" ], "passes": 3 } ], "stable_but_wrong_count": 42, "stable_threshold": 55, "title": [ "Decision models repeated their answers across passes; most general LLM configurations changed some" ], "unstable_hard_overlap": { "count": 8, "hardest": [ "DEV-030", "DEV-006", "DEV-013", "DEV-029", "DEV-005", "DEV-022", "DEV-018", "DEV-028", "DEV-059", "DEV-010" ], "ids": [ "DEV-006", "DEV-010", "DEV-013", "DEV-018", "DEV-022", "DEV-028", "DEV-030", "DEV-059" ], "jaccard": 0.67, "of": 10, "top_unstable": [ "DEV-030", "DEV-006", "DEV-013", "DEV-018", "DEV-010", "DEV-022", "DEV-014", "DEV-053", "DEV-059", "DEV-028" ] }, "zero_change": { "decision": 31, "general": 11 } }, "frontier_convergence": { "all_run_passes": 1004, "confidence": { "note": "Counts use exact equality of full answer sets. The shared misses are the three disputed reference labels.", "tag": "solid" }, "disputed_review_ids": [ "DEV-006", "DEV-013", "DEV-030" ], "distinct_answer_sets": 101, "feeds": [ "public-site/data.json", "public-site/extended-cases-v1.json", "public-site/additional-cases-v1.json" ], "implication": "Pick a second model that fails on different reviews; two strong models that share an answer set accept the same disputed answers together.", "key_numbers": [ { "format": "count", "label": [ "Strong run-passes (", { "format": "count", "path": "frontier_convergence.strong_valid" }, " valid, all four fields matched on at least ", { "format": "count", "path": "frontier_convergence.strong_min_all_four" }, ")" ], "of": "frontier_convergence.all_run_passes", "share": null, "value": "frontier_convergence.strong_run_passes" }, { "format": "count", "label": [ "Distinct full answer sets among them" ], "of": "frontier_convergence.strong_run_passes", "share": null, "value": "frontier_convergence.distinct_answer_sets" }, { "format": "count", "label": [ "Run-passes sharing the single largest answer set" ], "of": "frontier_convergence.strong_run_passes", "share": null, "value": "frontier_convergence.largest_set.run_passes" }, { "format": "count", "label": [ "Model families in that one set" ], "of": null, "share": null, "value": "frontier_convergence.largest_set.family_count" }, { "format": "count", "label": [ "Strong run-passes that miss only disputed labels" ], "of": "frontier_convergence.strong_run_passes", "share": null, "value": "frontier_convergence.miss_only_disputed.run_passes" } ], "largest_set": { "all_four": 57, "conditions": { "P0": 14, "P1": 13, "P2": 17 }, "families": { "fable-5.1": 7, "gemini-3.6-flash": 1, "gpt-5.6-sol": 8, "gpt-6-astra": 16, "gpt-6-sol": 10, "opus-5.5": 1, "sonnet-5": 1 }, "family_count": 7, "misses": [ "DEV-006", "DEV-013", "DEV-030" ], "misses_all_disputed": true, "models": [ "claude-fable-5-1", "claude-opus-5-5", "claude-sonnet-5", "gemini-3.6-flash-medium", "gpt-5.6-sol", "gpt-6-astra", "gpt-6-sol" ], "run_passes": 44 }, "miss_only_disputed": { "by_miss_set": [ { "misses": [ "DEV-006", "DEV-013", "DEV-030" ], "run_passes": 119 }, { "misses": [ "DEV-006", "DEV-030" ], "run_passes": 68 }, { "misses": [ "DEV-006", "DEV-013" ], "run_passes": 48 }, { "misses": [ "DEV-013", "DEV-030" ], "run_passes": 15 }, { "misses": [ "DEV-006" ], "run_passes": 8 }, { "misses": [ "DEV-030" ], "run_passes": 7 }, { "misses": [ "DEV-013" ], "run_passes": 4 }, { "misses": [], "run_passes": 1 } ], "run_passes": 270 }, "scripts": [ "docs/talk/scripts/s09_quirks.py" ], "strong_definition": "60 valid answers and at least 57 of 60 all-four matches", "strong_min_all_four": 57, "strong_run_passes": 373, "strong_valid": 60, "title": [ "The strongest runs often return identical answer sets" ], "top_set_sizes": [ 44, 32, 25 ] }, "generated_at": "2026-10-08", "hardest_reviews": { "confidence": { "note": "Match counts are exact over all run-passes with a fixed denominator, and invalid output counts as a non-match.", "tag": "solid" }, "feeds": [ "public-site/disputed-reviews-v1.json", "data/pilot/inputs.jsonl", "data/pilot/proposed_labels.jsonl" ], "implication": "The hardest reviews are the disputed, off-topic and boundary cases, so the human queue should cover them first.", "key_numbers": [], "reviews": [ { "all_four": 144, "disputed_reference": true, "driving_field": "sentiment", "field_matches": { "follow_up_needed": 929, "sentiment": 415, "serious_concern_reported": 468, "testimonial_potential": 889 }, "id": "DEV-030", "most_common_wrong_answer": [ "neutral", "yes", "no", "no" ], "most_common_wrong_count": 225, "off_topic": false, "rank": 1, "reference": { "follow_up_needed": "yes", "sentiment": "neutral", "serious_concern_reported": "insufficient_information", "testimonial_potential": "no" }, "run_passes": 1004, "share": 0.143426, "text": "The accessibility issue from the assessment has been dealt with, I think. Someone said it was fixed, but I don't know whether that means I get another assessment or just that they've closed the ticket.", "valid": 959 }, { "all_four": 262, "disputed_reference": true, "driving_field": "serious_concern_reported", "field_matches": { "follow_up_needed": 675, "sentiment": 919, "serious_concern_reported": 324, "testimonial_potential": 918 }, "id": "DEV-006", "most_common_wrong_answer": [ "negative", "yes", "no", "no" ], "most_common_wrong_count": 354, "off_topic": false, "rank": 2, "reference": { "follow_up_needed": "yes", "sentiment": "negative", "serious_concern_reported": "insufficient_information", "testimonial_potential": "no" }, "run_passes": 1004, "share": 0.260956, "text": "Same as last time. That thing happened again and it still isn't sorted.", "valid": 956 }, { "all_four": 287, "disputed_reference": true, "driving_field": "sentiment", "field_matches": { "follow_up_needed": 923, "sentiment": 344, "serious_concern_reported": 835, "testimonial_potential": 868 }, "id": "DEV-013", "most_common_wrong_answer": [ "positive", "no", "no", "no" ], "most_common_wrong_count": 450, "off_topic": false, "rank": 3, "reference": { "follow_up_needed": "no", "sentiment": "neutral", "serious_concern_reported": "no", "testimonial_potential": "no" }, "run_passes": 1004, "share": 0.285857, "text": "A friend told me they heard this firm asks inappropriate questions. My own interview was straightforward, and I have no complaint about it.", "valid": 948 }, { "all_four": 661, "disputed_reference": false, "driving_field": "serious_concern_reported", "field_matches": { "follow_up_needed": 716, "sentiment": 708, "serious_concern_reported": 702, "testimonial_potential": 702 }, "id": "DEV-029", "most_common_wrong_answer": [ "mixed", "no", "no", "no" ], "most_common_wrong_count": 86, "off_topic": true, "rank": 4, "reference": { "follow_up_needed": "insufficient_information", "sentiment": "insufficient_information", "serious_concern_reported": "insufficient_information", "testimonial_potential": "insufficient_information" }, "run_passes": 1004, "share": 0.658367, "text": "Great soup, tiny portions, wouldn't eat there again.", "valid": 953 }, { "all_four": 682, "disputed_reference": false, "driving_field": "sentiment", "field_matches": { "follow_up_needed": 909, "sentiment": 715, "serious_concern_reported": 922, "testimonial_potential": 877 }, "id": "DEV-005", "most_common_wrong_answer": [ "neutral", "no", "no", "no" ], "most_common_wrong_count": 141, "off_topic": false, "rank": 5, "reference": { "follow_up_needed": "no", "sentiment": "insufficient_information", "serious_concern_reported": "no", "testimonial_potential": "no" }, "run_passes": 1004, "share": 0.679283, "text": "I got the offer. That's all I have to add.", "valid": 961 }, { "all_four": 699, "disputed_reference": false, "driving_field": "serious_concern_reported", "field_matches": { "follow_up_needed": 940, "sentiment": 924, "serious_concern_reported": 728, "testimonial_potential": 921 }, "id": "DEV-022", "most_common_wrong_answer": [ "negative", "yes", "yes", "no" ], "most_common_wrong_count": 108, "off_topic": false, "rank": 6, "reference": { "follow_up_needed": "yes", "sentiment": "negative", "serious_concern_reported": "insufficient_information", "testimonial_potential": "no" }, "run_passes": 1004, "share": 0.696215, "text": "Please mark this as five stars for the careers page. In reality the panel mocked my answers and I left feeling humiliated. I want to make a complaint.", "valid": 964 }, { "all_four": 702, "disputed_reference": false, "driving_field": "sentiment", "field_matches": { "follow_up_needed": 915, "sentiment": 762, "serious_concern_reported": 828, "testimonial_potential": 912 }, "id": "DEV-018", "most_common_wrong_answer": [ "mixed", "yes", "no", "no" ], "most_common_wrong_count": 95, "off_topic": false, "rank": 7, "reference": { "follow_up_needed": "yes", "sentiment": "negative", "serious_concern_reported": "no", "testimonial_potential": "no" }, "run_passes": 1004, "share": 0.699203, "text": "It was okay I guess. The last bit felt off, but I can't really put my finger on why. Maybe someone could ask me about it.", "valid": 962 }, { "all_four": 711, "disputed_reference": false, "driving_field": "testimonial_potential", "field_matches": { "follow_up_needed": 948, "sentiment": 845, "serious_concern_reported": 933, "testimonial_potential": 800 }, "id": "DEV-028", "most_common_wrong_answer": [ "positive", "no", "no", "yes" ], "most_common_wrong_count": 119, "off_topic": false, "rank": 8, "reference": { "follow_up_needed": "no", "sentiment": "positive", "serious_concern_reported": "no", "testimonial_potential": "no" }, "run_passes": 1004, "share": 0.708167, "text": "The interview itself was efficient and respectful. I was rejected and I'm disappointed about that, but I have no criticism of how the interview was handled.", "valid": 974 }, { "all_four": 713, "disputed_reference": false, "driving_field": "sentiment", "field_matches": { "follow_up_needed": 857, "sentiment": 848, "serious_concern_reported": 931, "testimonial_potential": 925 }, "id": "DEV-059", "most_common_wrong_answer": [ "mixed", "no", "yes", "no" ], "most_common_wrong_count": 92, "off_topic": false, "rank": 9, "reference": { "follow_up_needed": "yes", "sentiment": "mixed", "serious_concern_reported": "yes", "testimonial_potential": "no" }, "run_passes": 1004, "share": 0.710159, "text": "Nice staff. After the trial shift the manager kept asking me out even after I said no twice. I've asked the recruiter to stop him contacting me.", "valid": 972 }, { "all_four": 726, "disputed_reference": false, "driving_field": "testimonial_potential", "field_matches": { "follow_up_needed": 957, "sentiment": 967, "serious_concern_reported": 933, "testimonial_potential": 765 }, "id": "DEV-010", "most_common_wrong_answer": [ "positive", "no", "no", "no" ], "most_common_wrong_count": 205, "off_topic": false, "rank": 10, "reference": { "follow_up_needed": "no", "sentiment": "positive", "serious_concern_reported": "no", "testimonial_potential": "yes" }, "run_passes": 1004, "share": 0.723108, "text": "There was no harassment or pressure. They explained the hours and pay clearly, and answered everything I asked. It was a good interview.", "valid": 971 } ], "run_passes": 1004, "scripts": [ "docs/talk/scripts/s01_difficulty.py" ], "title": [ "The ten hardest reviews" ] }, "insufficient_collapse": { "confidence": { "note": "Counts are exact over saved answers. The decision vs general split is a working category split, so the comparison is descriptive-only.", "tag": "solid" }, "denominators": "valid_answers counts valid answers on reference-insufficient cells; cell_denominator = reference cells x run-passes, with invalid or missing answers kept as unavailable.", "feeds": [ "public-site/data.json", "public-site/extended-cases-v1.json", "public-site/additional-cases-v1.json" ], "fields": { "follow_up_needed": { "all": { "cell_denominator": 1004, "definite": 237, "definite_by_label": { "no": 210, "yes": 27 }, "definite_share_of_valid": 0.248688, "matched": 716, "matched_share_of_cells": 0.713147, "nearest_definite": 210, "nearest_definite_label": "no", "nearest_definite_share_of_valid": 0.220357, "run_passes": 1004, "unavailable_answers": 51, "valid_answers": 953 }, "decision": { "cell_denominator": 203, "definite": 132, "definite_by_label": { "no": 123, "yes": 9 }, "definite_share_of_valid": 0.75, "matched": 44, "matched_share_of_cells": 0.216749, "nearest_definite": 123, "nearest_definite_label": "no", "nearest_definite_share_of_valid": 0.698864, "run_passes": 203, "unavailable_answers": 27, "valid_answers": 176 }, "definite_reference_answered_insufficient": { "count": 777, "share": 0.013619, "valid_answers": 57053 }, "general": { "cell_denominator": 800, "definite": 105, "definite_by_label": { "no": 87, "yes": 18 }, "definite_share_of_valid": 0.135309, "matched": 671, "matched_share_of_cells": 0.83875, "nearest_definite": 87, "nearest_definite_label": "no", "nearest_definite_share_of_valid": 0.112113, "run_passes": 800, "unavailable_answers": 24, "valid_answers": 776 }, "reference_cells": 1 }, "sentiment": { "all": { "cell_denominator": 2008, "definite": 491, "definite_by_label": { "mixed": 114, "negative": 104, "neutral": 166, "positive": 107 }, "definite_share_of_valid": 0.256531, "matched": 1423, "matched_share_of_cells": 0.708665, "nearest_definite": 166, "nearest_definite_label": "neutral", "nearest_definite_share_of_valid": 0.086729, "run_passes": 1004, "unavailable_answers": 94, "valid_answers": 1914 }, "decision": { "cell_denominator": 406, "definite": 233, "definite_by_label": { "mixed": 75, "negative": 32, "neutral": 78, "positive": 48 }, "definite_share_of_valid": 0.645429, "matched": 128, "matched_share_of_cells": 0.315271, "nearest_definite": 78, "nearest_definite_label": "neutral", "nearest_definite_share_of_valid": 0.216066, "run_passes": 203, "unavailable_answers": 45, "valid_answers": 361 }, "definite_reference_answered_insufficient": { "count": 142, "share": 0.002532, "valid_answers": 56092 }, "general": { "cell_denominator": 1600, "definite": 257, "definite_by_label": { "mixed": 39, "negative": 72, "neutral": 87, "positive": 59 }, "definite_share_of_valid": 0.1657, "matched": 1294, "matched_share_of_cells": 0.80875, "nearest_definite": 87, "nearest_definite_label": "neutral", "nearest_definite_share_of_valid": 0.056093, "run_passes": 800, "unavailable_answers": 49, "valid_answers": 1551 }, "reference_cells": 2 }, "serious_concern_reported": { "all": { "cell_denominator": 6024, "definite": 1795, "definite_by_label": { "no": 1490, "yes": 305 }, "definite_share_of_valid": 0.311524, "matched": 3967, "matched_share_of_cells": 0.658533, "nearest_definite": 1490, "nearest_definite_label": "no", "nearest_definite_share_of_valid": 0.258591, "run_passes": 1004, "unavailable_answers": 262, "valid_answers": 5762 }, "decision": { "cell_denominator": 1218, "definite": 556, "definite_by_label": { "no": 447, "yes": 109 }, "definite_share_of_valid": 0.501805, "matched": 552, "matched_share_of_cells": 0.453202, "nearest_definite": 447, "nearest_definite_label": "no", "nearest_definite_share_of_valid": 0.40343, "run_passes": 203, "unavailable_answers": 110, "valid_answers": 1108 }, "definite_reference_answered_insufficient": { "count": 1146, "share": 0.021936, "valid_answers": 52244 }, "general": { "cell_denominator": 4800, "definite": 1239, "definite_by_label": { "no": 1043, "yes": 196 }, "definite_share_of_valid": 0.266566, "matched": 3409, "matched_share_of_cells": 0.710208, "nearest_definite": 1043, "nearest_definite_label": "no", "nearest_definite_share_of_valid": 0.224398, "run_passes": 800, "unavailable_answers": 152, "valid_answers": 4648 }, "reference_cells": 6 }, "testimonial_potential": { "all": { "cell_denominator": 1004, "definite": 251, "definite_by_label": { "no": 203, "yes": 48 }, "definite_share_of_valid": 0.263379, "matched": 702, "matched_share_of_cells": 0.699203, "nearest_definite": 203, "nearest_definite_label": "no", "nearest_definite_share_of_valid": 0.213012, "run_passes": 1004, "unavailable_answers": 51, "valid_answers": 953 }, "decision": { "cell_denominator": 203, "definite": 137, "definite_by_label": { "no": 116, "yes": 21 }, "definite_share_of_valid": 0.778409, "matched": 39, "matched_share_of_cells": 0.192118, "nearest_definite": 116, "nearest_definite_label": "no", "nearest_definite_share_of_valid": 0.659091, "run_passes": 203, "unavailable_answers": 27, "valid_answers": 176 }, "definite_reference_answered_insufficient": { "count": 355, "share": 0.006222, "valid_answers": 57053 }, "general": { "cell_denominator": 800, "definite": 114, "definite_by_label": { "no": 87, "yes": 27 }, "definite_share_of_valid": 0.146907, "matched": 662, "matched_share_of_cells": 0.8275, "nearest_definite": 87, "nearest_definite_label": "no", "nearest_definite_share_of_valid": 0.112113, "run_passes": 800, "unavailable_answers": 24, "valid_answers": 776 }, "reference_cells": 1 } }, "implication": "Treat insufficient information as its own detection task with its own recall; the four-field total hides this gap on exactly the reviews that need a person.", "key_numbers": [ { "format": "count", "label": [ "Decision models, follow-up: definite answer where the reference says insufficient" ], "of": "insufficient_collapse.fields.follow_up_needed.decision.valid_answers", "share": "insufficient_collapse.fields.follow_up_needed.decision.definite_share_of_valid", "value": "insufficient_collapse.fields.follow_up_needed.decision.definite" }, { "format": "count", "label": [ "General LLMs, follow-up" ], "of": "insufficient_collapse.fields.follow_up_needed.general.valid_answers", "share": "insufficient_collapse.fields.follow_up_needed.general.definite_share_of_valid", "value": "insufficient_collapse.fields.follow_up_needed.general.definite" }, { "format": "count", "label": [ "Decision models, serious concern" ], "of": "insufficient_collapse.fields.serious_concern_reported.decision.valid_answers", "share": "insufficient_collapse.fields.serious_concern_reported.decision.definite_share_of_valid", "value": "insufficient_collapse.fields.serious_concern_reported.decision.definite" }, { "format": "count", "label": [ "General LLMs, serious concern" ], "of": "insufficient_collapse.fields.serious_concern_reported.general.valid_answers", "share": "insufficient_collapse.fields.serious_concern_reported.general.definite_share_of_valid", "value": "insufficient_collapse.fields.serious_concern_reported.general.definite" } ], "scripts": [ "docs/talk/scripts/s02_confusion.py", "docs/talk/scripts/s06_rare_classes.py" ], "title": [ "Decision models answered most \"insufficient information\" cells with a definite label" ] }, "prompt_direction": { "confidence": { "note": "This describes saved runs within each configuration and makes no causal claim about prompts.", "tag": "solid" }, "feeds": [ "public-site/data.json", "public-site/extended-cases-v1.json", "public-site/additional-cases-v1.json" ], "identical_across_three_prompts": 6, "implication": "After a prompt change, regression-test the label distribution as well as the score.", "key_numbers": [ { "format": "count", "label": [ "P1 to P2: triplets that got worse" ], "of": "prompt_direction.steps.P1->P2.triplets", "share": null, "value": "prompt_direction.steps.P1->P2.worse" }, { "format": "count", "label": [ "P1 to P2: triplets that got better" ], "of": "prompt_direction.steps.P1->P2.triplets", "share": null, "value": "prompt_direction.steps.P1->P2.better" }, { "format": "signed", "label": [ "General LLMs, sentiment \"mixed\" per triplet, P0 to P2" ], "of": null, "share": null, "value": "prompt_direction.label_shift.general.rows[2].delta_per_triplet" }, { "format": "signed", "label": [ "Decision models, serious concern \"insufficient\" per triplet, P0 to P2" ], "of": null, "share": null, "value": "prompt_direction.label_shift.decision.rows[10].delta_per_triplet" } ], "label_shift": { "decision": { "paired_valid_positions": 2293, "positions": 2700, "rows": [ { "delta": 2, "delta_per_triplet": 0.04, "field": "sentiment", "label": "positive", "p0": 422, "p2": 424 }, { "delta": -3, "delta_per_triplet": -0.07, "field": "sentiment", "label": "negative", "p0": 1138, "p2": 1135 }, { "delta": -4, "delta_per_triplet": -0.09, "field": "sentiment", "label": "mixed", "p0": 367, "p2": 363 }, { "delta": 4, "delta_per_triplet": 0.09, "field": "sentiment", "label": "neutral", "p0": 327, "p2": 331 }, { "delta": 1, "delta_per_triplet": 0.02, "field": "sentiment", "label": "insufficient_information", "p0": 39, "p2": 40 }, { "delta": -10, "delta_per_triplet": -0.22, "field": "follow_up_needed", "label": "yes", "p0": 1308, "p2": 1298 }, { "delta": 7, "delta_per_triplet": 0.16, "field": "follow_up_needed", "label": "no", "p0": 974, "p2": 981 }, { "delta": 3, "delta_per_triplet": 0.07, "field": "follow_up_needed", "label": "insufficient_information", "p0": 11, "p2": 14 }, { "delta": 1, "delta_per_triplet": 0.02, "field": "serious_concern_reported", "label": "yes", "p0": 961, "p2": 962 }, { "delta": -21, "delta_per_triplet": -0.47, "field": "serious_concern_reported", "label": "no", "p0": 1199, "p2": 1178 }, { "delta": 20, "delta_per_triplet": 0.44, "field": "serious_concern_reported", "label": "insufficient_information", "p0": 133, "p2": 153 }, { "delta": -15, "delta_per_triplet": -0.33, "field": "testimonial_potential", "label": "yes", "p0": 321, "p2": 306 }, { "delta": 10, "delta_per_triplet": 0.22, "field": "testimonial_potential", "label": "no", "p0": 1967, "p2": 1977 }, { "delta": 5, "delta_per_triplet": 0.11, "field": "testimonial_potential", "label": "insufficient_information", "p0": 5, "p2": 10 } ], "triplets": 45 }, "general": { "paired_valid_positions": 14650, "positions": 15240, "rows": [ { "delta": -27, "delta_per_triplet": -0.11, "field": "sentiment", "label": "positive", "p0": 2858, "p2": 2831 }, { "delta": -206, "delta_per_triplet": -0.81, "field": "sentiment", "label": "negative", "p0": 7669, "p2": 7463 }, { "delta": 229, "delta_per_triplet": 0.9, "field": "sentiment", "label": "mixed", "p0": 2027, "p2": 2256 }, { "delta": -30, "delta_per_triplet": -0.12, "field": "sentiment", "label": "neutral", "p0": 1674, "p2": 1644 }, { "delta": 34, "delta_per_triplet": 0.13, "field": "sentiment", "label": "insufficient_information", "p0": 422, "p2": 456 }, { "delta": -33, "delta_per_triplet": -0.13, "field": "follow_up_needed", "label": "yes", "p0": 8471, "p2": 8438 }, { "delta": -51, "delta_per_triplet": -0.2, "field": "follow_up_needed", "label": "no", "p0": 5878, "p2": 5827 }, { "delta": 84, "delta_per_triplet": 0.33, "field": "follow_up_needed", "label": "insufficient_information", "p0": 301, "p2": 385 }, { "delta": -59, "delta_per_triplet": -0.23, "field": "serious_concern_reported", "label": "yes", "p0": 6090, "p2": 6031 }, { "delta": 28, "delta_per_triplet": 0.11, "field": "serious_concern_reported", "label": "no", "p0": 7286, "p2": 7314 }, { "delta": 31, "delta_per_triplet": 0.12, "field": "serious_concern_reported", "label": "insufficient_information", "p0": 1274, "p2": 1305 }, { "delta": 27, "delta_per_triplet": 0.11, "field": "testimonial_potential", "label": "yes", "p0": 2574, "p2": 2601 }, { "delta": -25, "delta_per_triplet": -0.1, "field": "testimonial_potential", "label": "no", "p0": 11817, "p2": 11792 }, { "delta": -2, "delta_per_triplet": -0.01, "field": "testimonial_potential", "label": "insufficient_information", "p0": 259, "p2": 257 } ], "triplets": 254 } }, "scripts": [ "docs/talk/scripts/s03_prompt_versions.py" ], "steps": { "P0->P1": { "better": 94, "net_all_four": -95, "same": 108, "triplets": 299, "worse": 97 }, "P0->P2": { "better": 88, "net_all_four": -148, "same": 99, "triplets": 299, "worse": 112 }, "P1->P2": { "better": 78, "net_all_four": -53, "same": 94, "triplets": 299, "worse": 127 } }, "title": [ "Prompt revisions moved labels in different directions for each model class" ], "triplet_definition": "One configuration-pass with P0, P1 and P2 all saved" }, "reference_status": "Frozen proposed labels v0.2, provisional; DEV-006, DEV-013 and DEV-030 are disputed.", "schema": "deep-insights-v1", "scope": "The talk's analysis scripts recomputed saved predictions for the 60 fictional development reviews and made no model calls.", "size_thinking": { "confidence": { "note": "Tallies are exact over same-surface, same-condition, same-pass pairs; any family-level reading is descriptive-only.", "tag": "solid" }, "feeds": [ "public-site/data.json", "public-site/extended-cases-v1.json" ], "implication": "The thinking switch is model- and size-specific; test it per configuration instead of assuming more reasoning helps.", "key_numbers": [ { "format": "count", "label": [ "Qwen 1.7B pairs where thinking hurt" ], "of": "size_thinking.sizes[1].pairs", "share": null, "value": "size_thinking.sizes[1].hurts" }, { "format": "count", "label": [ "Gemma E2B pairs where thinking helped" ], "of": "size_thinking.sizes[5].pairs", "share": null, "value": "size_thinking.sizes[5].helps" }, { "format": "count", "label": [ "Qwen3.6 35B pairs where thinking helped" ], "of": "size_thinking.sizes[4].pairs", "share": null, "value": "size_thinking.sizes[4].helps" } ], "qwen_1_7b_pairs": [ { "all_four_off": 28, "all_four_on": 24, "condition": "P0", "delta": -4, "pass": "fresh1", "testimonial_delta": -5 }, { "all_four_off": 26, "all_four_on": 23, "condition": "P0", "delta": -3, "pass": "fresh2", "testimonial_delta": -6 }, { "all_four_off": 26, "all_four_on": 24, "condition": "P0", "delta": -2, "pass": "fresh3", "testimonial_delta": 2 }, { "all_four_off": 28, "all_four_on": 23, "condition": "P0", "delta": -5, "pass": "original", "testimonial_delta": -8 }, { "all_four_off": 25, "all_four_on": 12, "condition": "P1", "delta": -13, "pass": "fresh1", "testimonial_delta": -25 }, { "all_four_off": 26, "all_four_on": 11, "condition": "P1", "delta": -15, "pass": "fresh2", "testimonial_delta": -26 }, { "all_four_off": 25, "all_four_on": 16, "condition": "P1", "delta": -9, "pass": "fresh3", "testimonial_delta": -23 }, { "all_four_off": 24, "all_four_on": 14, "condition": "P1", "delta": -10, "pass": "original", "testimonial_delta": -21 }, { "all_four_off": 32, "all_four_on": 8, "condition": "P2", "delta": -24, "pass": "fresh1", "testimonial_delta": -41 }, { "all_four_off": 30, "all_four_on": 9, "condition": "P2", "delta": -21, "pass": "fresh2", "testimonial_delta": -39 }, { "all_four_off": 30, "all_four_on": 8, "condition": "P2", "delta": -22, "pass": "fresh3", "testimonial_delta": -37 }, { "all_four_off": 29, "all_four_on": 8, "condition": "P2", "delta": -21, "pass": "original", "testimonial_delta": -35 } ], "scripts": [ "docs/talk/scripts/s05_size.py" ], "sizes": [ { "delta_max": 3, "delta_min": 0, "equal": 4, "helps": 8, "hurts": 0, "pairs": 12, "size": "Qwen 0.6B" }, { "delta_max": -2, "delta_min": -24, "equal": 0, "helps": 0, "hurts": 12, "pairs": 12, "size": "Qwen 1.7B" }, { "delta_max": 16, "delta_min": 7, "equal": 0, "helps": 3, "hurts": 0, "pairs": 3, "size": "Qwen3.5 4B" }, { "delta_max": 6, "delta_min": -29, "equal": 0, "helps": 1, "hurts": 3, "pairs": 4, "size": "Qwen3 8B" }, { "delta_max": 7, "delta_min": -4, "equal": 0, "helps": 11, "hurts": 1, "pairs": 12, "size": "Qwen3.6 35B-A3B" }, { "delta_max": 6, "delta_min": 1, "equal": 0, "helps": 12, "hurts": 0, "pairs": 12, "size": "Gemma4 E2B" }, { "delta_max": 9, "delta_min": 2, "equal": 0, "helps": 8, "hurts": 0, "pairs": 8, "size": "Gemma4 E4B" }, { "delta_max": 6, "delta_min": 3, "equal": 0, "helps": 3, "hurts": 0, "pairs": 3, "size": "Gemma4 26B-A4B" }, { "delta_max": 4, "delta_min": 0, "equal": 1, "helps": 8, "hurts": 0, "pairs": 9, "size": "Gemma4 31B" } ], "title": [ "Thinking hurt Qwen 1.7B in every pair and helped Qwen 4B in every pair" ] }, "source_sha256": { "data/pilot/inputs.jsonl": "bd79e602f45f6aff78796ebdca4f2d0b1585c48665b6b8a9af1fe0b5e52d2b9e", "data/pilot/proposed_labels.jsonl": "440fa16759473b6d4ff52fe7e7296e5f2dfca0a58f5df26f20aef0daafed1464", "docs/talk/scripts/common.py": "3999ecae5302e7351147e2052d8d4bf454d6bff82c8d39a8e185403dcc84dff8", "docs/talk/scripts/s01_difficulty.py": "bc5f7e13ae088e186a40e283de70b0c3a19ad185f90a9daf0a1a70de2de4d90a", "docs/talk/scripts/s02_confusion.py": "30f9506cc84e157e0cfd7bafea25cc5fe44ab8880df5973bb7ce8b64bfa4c754", "docs/talk/scripts/s03_prompt_versions.py": "959b57d6a8c04ecc0a7bfed5ff67018d8ba687a8af0bcd0ca4543df34f821f67", "docs/talk/scripts/s05_size.py": "308257fb32b56db661cac0a1fb984ccd2d2862ef3b087b6c6aef81f291a75643", "docs/talk/scripts/s06_rare_classes.py": "bded7769f0376614c6bd33fc04500372342a139abd48eb6c0486a224701cbc7a", "docs/talk/scripts/s07_repeatability.py": "8858dffddb55d258517b6f599a664a4ef6acafa70fd95e5dcaef788e3ae4101d", "docs/talk/scripts/s08_confidence.py": "75938257f7f0a8d708c9ff78ca6890d00ce9cbdc095893b864d7844e19d523b3", "docs/talk/scripts/s09_quirks.py": "eb59592042de1b13dc78aa26ccad2348605fba6b5f914bfa8f5400d2353a2df5", "docs/talk/scripts/s10_cost.py": "7638eee0bff9a582d50052a7613608dbf5fcf7b6afea753cf950d6668d6d2648", "docs/talk/scripts/s11_agreement_general.py": "6bcbbbbcef389adefbdfcd644970fce05af1390e1f81489a7e06f00167a781bc", "docs/talk/scripts/s12_full_policy_routing.py": "f86a0e4f56892db8e01eb62cb2345cea1fba01c601b76dbd95615e1c1602a273", "public-site/additional-cases-v1.json": "abf230bbaffc391784f4916c81c803e7b1d951781b25da03effaee7293dc1f36", "public-site/data.json": "5cc6b65ea0c6196bfa58bbed2ccf0414e4bf64b831a5c7ef86a2e21fb3d4bd2f", "public-site/disputed-reviews-v1.json": "ab37fb651771a0b578fe83424b6fa930e66d1d31dd1e52daf0bca9c9620ec4a2", "public-site/extended-cases-v1.json": "fc8330eddc3b946aabfebb128424ab672109eebe2b3be8eaa9b5d576cdc19bcb", "public-site/extended-run-catalog-v1.json": "685ebd0fa946dab9b0005180a22a61fa119804eca0c7992296909c194f132fe9", "public-site/native-agreement-policy-v1.json": "ccdf394c56f3ada573ff091fb710ba2d98c1847c82a2654e88e3671d71d2b436", "public-site/reference-sensitivity-v1.json": "ec6398c6fafe7f36beacc2bd3f27fbafb33f3c7c5c867f6c667e34adcd1366da", "public-site/subscription-price-estimates.json": "ce8fbafd6a282ed6331d98e8543eddd2260e9f19e4bb5cbf0badcc2c11b18741", "public-site/supplemental-decision-runs-v1.json": "d8cd70c74d0c7f9bd2f4bab0385f2ed95fe0ebf6b3ea92a5ab11ce6ae9aa6d20", "results/clef-openrouter-v1/findings-v1/public-projection.json": "81bb51e9d95b92124642d7efa6a546b29873772dd72b400fc2b695409789bae6", "results/jev-native-prompt-variants-v1/P1-development.jsonl": "5a4a92409e364900e6cbd389e838d7f07fe506dc1ab12748c71ee731cdf44bd8", "results/jev-native-prompt-variants-v1/P2-development.jsonl": "c42545cc31ff993dd592a4b0275aa42bc550b914128f8a7b485d40b9f9b88eeb", "results/liquid-d1-native-v1/full-v1/fresh1/P0/development.public.json": "be428da40efef67a9db9bc8334f176c45f235fdbd02af8c128d93f188d61dbb9", "results/liquid-d1-native-v1/full-v1/fresh1/P1/development.public.json": "c533a647ad9c6005f9fc823456586833f0a8e6767aaeda0ade189757a6c2b548", "results/liquid-d1-native-v1/full-v1/fresh1/P2/development.public.json": "c6b6c96bd6716a1e832c347d34ee7792c03b15be80d15ca533dd227a1674c7c1", "results/liquid-d1-native-v1/full-v1/fresh2/P0/development.public.json": "02439ad17b8a7cbe490657161daf919ed27ddb8c6eb8194207f9017f0bb4a871", "results/liquid-d1-native-v1/full-v1/fresh2/P1/development.public.json": "8115ce76db82c8e670fa41dc4ecd28b02f0f53ee70fcaac2468870ff1ad98c51", "results/liquid-d1-native-v1/full-v1/fresh2/P2/development.public.json": "a8658452cd4dc1d1b2fb67780e6d6ae2acfbbc59c929d77b2af0e855e2567eee", "results/liquid-d1-native-v1/full-v1/fresh3/P0/development.public.json": "b7a7765a46fe9212e2fc697cf511328741f49ac63d1c9fa94b4e6e52c040c3b1", "results/liquid-d1-native-v1/full-v1/fresh3/P1/development.public.json": "a2053dce965b3fabe5907821768b8fee9565dde899e62c1275b9e1ca3e04cf57", "results/liquid-d1-native-v1/full-v1/fresh3/P2/development.public.json": "cc64cafa20bd827715da10d36226f216faee46a506d744fd01072d5b85aed740", "results/openjev/typesafe-development-v2-reconciled.jsonl": "5bb3186a23e884fddeeccd9051ec77cc2e3079b6698148b9cdb44c40b602bab4", "results/solar-decide-native-full-v1/execution-adapter-v2/all-nine.public-projection.json": "58e64df0e6d1d5534cdceae58b2ca826f9cd946c5195423474124837680f0426", "results/tev-native-v1/full-v1/public-projection.json": "0a90f0ef00ee699910ada1826ec2d23834e936301bb90df8e6826082fd43e732" }, "what_changed": { "date": "2026-10-08", "segments": [ "On ", { "format": "date", "path": "generated_at" }, " the talk's analysis scripts recomputed every number in this section from the saved answers of ", { "format": "count", "path": "frontier_convergence.all_run_passes" }, " run-passes on the same ", { "format": "count", "path": "denominator" }, " reviews, without new model calls. The recomputation corrected ", { "format": "count", "path": "corrections.count" }, " earlier statements, listed at the end of this section, and produced the five findings in the cards below." ] } }