moving

2026-09-02 10:10:29 +02:00
commit 0c3e2ead3b
3841 changed files with 970576 additions and 0 deletions
--- a/benchmarks/probes/judge-swap-validation/_summary-split.json
+++ b/benchmarks/probes/judge-swap-validation/_summary-split.json
@@ -0,0 +1,210 @@
+{
+  "aggregate_verdict": "INCONCLUSIVE_BUT_OPERATIONAL_SIGNAL",
+  "aggregate_reason": "parse rate < 80% on at least one candidate",
+  "n_split": 7,
+  "per_candidate": {
+    "kimi": {
+      "parse_success": 5,
+      "n_total": 7,
+      "n_eval": 5,
+      "p_opus": 0.8,
+      "p_gpt": 0.2,
+      "k_opus": 0.0,
+      "k_gpt": 0.0,
+      "k_cons": 0.0,
+      "k_opus_det": {
+        "n": 5,
+        "agree": 4,
+        "po": 0.8,
+        "pe": 0.8,
+        "cand_counts": {
+          "correct": 4,
+          "incorrect": 1
+        },
+        "ref_counts": {
+          "correct": 5,
+          "incorrect": 0
+        }
+      },
+      "k_gpt_det": {
+        "n": 5,
+        "agree": 1,
+        "po": 0.2,
+        "pe": 0.2,
+        "cand_counts": {
+          "correct": 4,
+          "incorrect": 1
+        },
+        "ref_counts": {
+          "correct": 0,
+          "incorrect": 5
+        }
+      },
+      "lat_p50_ms": 31985,
+      "lat_p95_ms": 51046,
+      "routing": "direct",
+      "split_verdict": "FAIL",
+      "p_opus_count": 4,
+      "p_gpt_count": 1
+    },
+    "minimax": {
+      "parse_success": 7,
+      "n_total": 7,
+      "n_eval": 7,
+      "p_opus": 0.8571428571428571,
+      "p_gpt": 0.14285714285714285,
+      "k_opus": 0.0,
+      "k_gpt": 0.0,
+      "k_cons": 0.0,
+      "k_opus_det": {
+        "n": 7,
+        "agree": 6,
+        "po": 0.8571428571428571,
+        "pe": 0.8571428571428571,
+        "cand_counts": {
+          "correct": 6,
+          "incorrect": 1
+        },
+        "ref_counts": {
+          "correct": 7,
+          "incorrect": 0
+        }
+      },
+      "k_gpt_det": {
+        "n": 7,
+        "agree": 1,
+        "po": 0.14285714285714285,
+        "pe": 0.14285714285714285,
+        "cand_counts": {
+          "correct": 6,
+          "incorrect": 1
+        },
+        "ref_counts": {
+          "correct": 0,
+          "incorrect": 7
+        }
+      },
+      "lat_p50_ms": 16627,
+      "lat_p95_ms": 21222,
+      "routing": "openrouter",
+      "split_verdict": "FAIL",
+      "p_opus_count": 6,
+      "p_gpt_count": 1
+    },
+    "deepseek": {
+      "parse_success": 5,
+      "n_total": 7,
+      "n_eval": 5,
+      "p_opus": 0.4,
+      "p_gpt": 0.6,
+      "k_opus": 0.0,
+      "k_gpt": 0.0,
+      "k_cons": 0.0,
+      "k_opus_det": {
+        "n": 5,
+        "agree": 2,
+        "po": 0.4,
+        "pe": 0.4,
+        "cand_counts": {
+          "correct": 2,
+          "incorrect": 3
+        },
+        "ref_counts": {
+          "correct": 5,
+          "incorrect": 0
+        }
+      },
+      "k_gpt_det": {
+        "n": 5,
+        "agree": 3,
+        "po": 0.6,
+        "pe": 0.6,
+        "cand_counts": {
+          "correct": 2,
+          "incorrect": 3
+        },
+        "ref_counts": {
+          "correct": 0,
+          "incorrect": 5
+        }
+      },
+      "lat_p50_ms": 15408,
+      "lat_p95_ms": 21200,
+      "routing": "direct",
+      "split_verdict": "FAIL",
+      "p_opus_count": 2,
+      "p_gpt_count": 3
+    },
+    "zhipu": {
+      "parse_success": 6,
+      "n_total": 7,
+      "n_eval": 6,
+      "p_opus": 0.0,
+      "p_gpt": 1.0,
+      "k_opus": 0.0,
+      "k_gpt": 1.0,
+      "k_cons": 0.0,
+      "k_opus_det": {
+        "n": 6,
+        "agree": 0,
+        "po": 0.0,
+        "pe": 0.0,
+        "cand_counts": {
+          "correct": 0,
+          "incorrect": 6
+        },
+        "ref_counts": {
+          "correct": 6,
+          "incorrect": 0
+        }
+      },
+      "k_gpt_det": {
+        "n": 6,
+        "agree": 6,
+        "po": 1.0,
+        "pe": 1.0
+      },
+      "lat_p50_ms": 17882,
+      "lat_p95_ms": 22543,
+      "routing": "direct",
+      "split_verdict": "FAIL",
+      "p_opus_count": 0,
+      "p_gpt_count": 6
+    }
+  },
+  "per_candidate_aggregate": {
+    "kimi": {
+      "n_combined": 25,
+      "k_agg_vs_opus": 0.9110320284697508,
+      "k_agg_vs_gpt": 0.5762711864406778,
+      "k_agg_cons": 0.5762711864406778
+    },
+    "minimax": {
+      "n_combined": 27,
+      "k_agg_vs_opus": 0.9221902017291065,
+      "k_agg_vs_gpt": 0.4563758389261745,
+      "k_agg_cons": 0.4563758389261745
+    },
+    "deepseek": {
+      "n_combined": 23,
+      "k_agg_vs_opus": 0.7088607594936708,
+      "k_agg_vs_gpt": 0.7472527472527472,
+      "k_agg_cons": 0.7088607594936708
+    },
+    "zhipu": {
+      "n_combined": 25,
+      "k_agg_vs_opus": 0.44444444444444436,
+      "k_agg_vs_gpt": 1.0,
+      "k_agg_cons": 0.44444444444444436
+    }
+  },
+  "ranking_by_k_cons_desc": [
+    "kimi",
+    "minimax",
+    "deepseek",
+    "zhipu"
+  ],
+  "recommended_primary": "ZHIPU",
+  "recommended_backup": "MINIMAX",
+  "minimax_routing_resolution": "direct_failed_fell_back_openrouter"
+}
--- a/benchmarks/probes/judge-swap-validation/deepseek-mt-comparison-memo.md
+++ b/benchmarks/probes/judge-swap-validation/deepseek-mt-comparison-memo.md
@@ -0,0 +1,24 @@
+# DeepSeek `max_tokens` 1024→2048 — Comparison Memo
+
+**Date:** 2026-04-24  **Parent:** `ae0d312`  **Target:** PM-RATIFY-DEEPSEEK-PARSE-FIX
+
+## Parse issue: **truncation_fixable**
+
+| Metric | mt=1024 | mt=2048 | Δ |
+|--------|---------|---------|---|
+| Parse | 5/7 (71%) | **7/7 (100%)** | +29 pp |
+| Agree w/ Opus (= verified-correct) | 2/5 (40%) | **1/7 (14%)** | **−26 pp** |
+| Agree w/ GPT | 3/5 (60%) | 6/7 (86%) | +26 pp |
+| Latency p50 | 21.2 s | 15.0 s | −6.2 s |
+
+Both recovered NULLs (completion_tokens=1053, 886) and 1 flipped verdict (conv-50_q015 oracle-context: `correct → incorrect`) land on GPT-side.
+
+## Finding
+
+Parse is TRUNCATION_FIXABLE at `max_tokens=2048`. **But correctness regressed under PM's corrected metric** — longer reasoning makes DeepSeek more GPT-strict, reducing its agreement with the verified-correct Opus reference on oriented splits. Gap in PM's decision matrix: parse improved + correctness regressed.
+
+**Spirit-of-matrix → MiniMax primary + Kimi backup.** DeepSeek (14% correct) and Zhipu (0%) both fail correctness bar regardless of parse fix.
+
+**Cost:** ~$0.05 (cap $0.50).  **Wall-clock:** ~2.4 min.
+
+`cc1_state: HALTED`
--- a/benchmarks/probes/judge-swap-validation/deepseek-mt2048-probe.py
+++ b/benchmarks/probes/judge-swap-validation/deepseek-mt2048-probe.py
@@ -0,0 +1,274 @@
+"""
+§1.3h Option C follow-up — DeepSeek max_tokens=2048 verification
+=================================================================
+
+PM-ratified bounded probe: re-run DeepSeek on the same 7 splits from
+§1.3h with max_tokens=2048 (up from 1024). Purpose: classify the 2/7
+parse failures from §1.3h as truncation_fixable vs structural.
+
+Scope:
+  - Single candidate (DeepSeek only)
+  - Same 7-split sample (split-cases-sample.jsonl SHA 6df4ed0f...)
+  - One parameter delta: max_tokens 1024 -> 2048
+  - No new sample selection, no new Opus/GPT calls
+  - Artefacts additive in benchmarks/probes/judge-swap-validation/
+
+Scope guards (identical to §1.3h):
+  - Manifest v5 anchor fc16925 immutable
+  - Parent HEAD = ae0d312 (§1.3h anchor)
+  - §11 frozen paths untouched
+  - No runner modification, no v6 emission
+
+Budget: $0.50 cap, 15 min wall-clock cap.
+"""
+from __future__ import annotations
+
+import json
+import sys
+import time
+import urllib.error
+import urllib.request
+from datetime import datetime, timezone
+from pathlib import Path
+
+try:
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+    sys.stderr.reconfigure(encoding="utf-8", errors="replace")
+except Exception:
+    pass
+
+PROBE_DIR = Path("D:/Projects/waggle-os/benchmarks/probes/judge-swap-validation")
+SAMPLE_PATH = PROBE_DIR / "split-cases-sample.jsonl"
+OUT_PATH = PROBE_DIR / "deepseek-split-responses-v2-mt2048.jsonl"
+
+# Verbatim judge prompt (same as §1.3g and §1.3h probes)
+JUDGE_PROMPT_TEMPLATE = "\n".join([
+    "You are evaluating whether an LLM's answer is correct against ground truth.",
+    "",
+    "## Question",
+    "{question}",
+    "",
+    "## Ground-truth answer",
+    "{ground_truth}",
+    "",
+    "## Ground-truth supporting context (excerpt shown to the model)",
+    "{context}",
+    "",
+    "## Model's answer",
+    "{model_answer}",
+    "",
+    "## Your task",
+    "",
+    "Step 1: Determine if the model's answer is correct.",
+    "- \"correct\" means the model's answer contains all required facts from ground truth, with no additional incorrect claims.",
+    "- Minor phrasing differences, synonyms, or alternative but equivalent formulations are acceptable.",
+    "- Extra detail is acceptable ONLY if it is factually correct.",
+    "",
+    "Step 2: If incorrect, assign exactly one failure mode using this decision tree:",
+    "",
+    "1. Does the model explicitly refuse or say it does not know? -> F1 (ABSTAIN)",
+    "2. Does the model answer a DIFFERENT question than was asked (coherent but off-topic)? -> F5 (OFF-TOPIC)",
+    "3. Does the model rely on entities, names, dates, or claims that do NOT appear in the ground-truth context (fabrication)? -> F4 (HALLUCINATED)",
+    "4. Does the model correctly state SOME required facts but miss others, without stating any incorrect facts? -> F2 (PARTIAL)",
+    "5. Otherwise (model states facts derived from the context but gets them wrong): -> F3 (INCORRECT)",
+    "",
+    "Step 3: Return JSON only, no prose, in this exact schema:",
+    "",
+    "{{",
+    "  \"verdict\": \"correct\" | \"incorrect\",",
+    "  \"failure_mode\": null | \"F1\" | \"F2\" | \"F3\" | \"F4\" | \"F5\",",
+    "  \"rationale\": \"one sentence explaining the verdict\"",
+    "}}",
+    "",
+    "If verdict is \"correct\", failure_mode MUST be null.",
+    "If verdict is \"incorrect\", failure_mode MUST be one of F1-F5.",
+])
+
+
+def ts() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def logmsg(msg: str) -> None:
+    print(f"{ts()} {msg}", flush=True)
+
+
+def load_env() -> dict[str, str]:
+    env_path = Path("D:/Projects/waggle-os/.env")
+    out: dict[str, str] = {}
+    for line in env_path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        k, _, v = line.partition("=")
+        out[k.strip()] = v.strip().strip('"').strip("'")
+    return out
+
+
+def extract_json_body(raw: str) -> dict | None:
+    import re
+    if not raw:
+        return None
+    trimmed = raw.strip()
+    if trimmed.startswith("```"):
+        m = re.match(r"^```(?:json)?\s*\n?(.*?)```\s*$", trimmed, re.DOTALL)
+        if m:
+            trimmed = m.group(1).strip()
+    try:
+        return json.loads(trimmed)
+    except Exception:
+        pass
+    first = trimmed.find("{")
+    last = trimmed.rfind("}")
+    if first != -1 and last != -1 and last > first:
+        try:
+            return json.loads(trimmed[first:last + 1])
+        except Exception:
+            return None
+    return None
+
+
+def parse_verdict(raw: str) -> tuple[str | None, str | None, str | None]:
+    body = extract_json_body(raw)
+    if not isinstance(body, dict):
+        return (None, None, None)
+    v = body.get("verdict")
+    fm = body.get("failure_mode")
+    rat = body.get("rationale")
+    if v not in ("correct", "incorrect"):
+        return (None, None, None)
+    if fm is not None and fm not in ("F1", "F2", "F3", "F4", "F5"):
+        fm = None
+    return (v, fm, rat if isinstance(rat, str) else None)
+
+
+def http_post_json(url: str, headers: dict, body: dict, timeout_s: int = 60) -> tuple[int, dict | str]:
+    req = urllib.request.Request(
+        url, data=json.dumps(body).encode("utf-8"), method="POST",
+        headers={"Content-Type": "application/json", **headers},
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=timeout_s) as resp:
+            raw = resp.read().decode("utf-8", errors="replace")
+            try:
+                return resp.status, json.loads(raw)
+            except Exception:
+                return resp.status, raw
+    except urllib.error.HTTPError as e:
+        try:
+            return e.code, e.read().decode("utf-8", errors="replace")[:2000]
+        except Exception:
+            return e.code, ""
+    except Exception as e:
+        return 0, f"{type(e).__name__}: {e}"
+
+
+def call_deepseek_mt2048(prompt: str, api_key: str, max_attempts: int = 3) -> dict:
+    url = "https://api.deepseek.com/v1/chat/completions"
+    headers = {"Authorization": f"Bearer {api_key}"}
+    body = {
+        "model": "deepseek-v4-pro",
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": 0.0,
+        "max_tokens": 2048,  # ← THE SINGLE DELTA
+    }
+    started = time.time()
+    retries = 0
+    last_err = None
+    for attempt in range(max_attempts):
+        status, resp = http_post_json(url, headers, body, timeout_s=60)
+        if status == 200 and isinstance(resp, dict):
+            choices = resp.get("choices") or []
+            if choices:
+                msg = choices[0].get("message") or {}
+                content = msg.get("content") or msg.get("reasoning_content") or ""
+                usage = resp.get("usage", {})
+                return {
+                    "raw_text": content,
+                    "status": 200,
+                    "error": None,
+                    "retries": retries,
+                    "latency_ms": int((time.time() - started) * 1000),
+                    "prompt_tokens": usage.get("prompt_tokens"),
+                    "completion_tokens": usage.get("completion_tokens"),
+                }
+        last_err = f"status={status} resp={str(resp)[:400]}"
+        retries += 1
+        if attempt < max_attempts - 1:
+            time.sleep(2 ** attempt)
+    return {
+        "raw_text": "",
+        "status": 0,
+        "error": last_err,
+        "retries": retries,
+        "latency_ms": int((time.time() - started) * 1000),
+        "prompt_tokens": None,
+        "completion_tokens": None,
+    }
+
+
+def main() -> int:
+    logmsg("[mt2048] DeepSeek max_tokens bump verification START")
+    env = load_env()
+    key = env.get("DEEPSEEK_API_KEY", "").strip()
+    if not key:
+        logmsg("[mt2048] FATAL DEEPSEEK_API_KEY missing")
+        return 2
+
+    sample = []
+    with SAMPLE_PATH.open("r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                sample.append(json.loads(line))
+    logmsg(f"[mt2048] loaded {len(sample)} split instances from {SAMPLE_PATH.name}")
+
+    rows = []
+    for i, s in enumerate(sample):
+        prompt = JUDGE_PROMPT_TEMPLATE.format(
+            question=s.get("question") or "",
+            ground_truth=s.get("ground_truth") or "",
+            context=s.get("context") or "",
+            model_answer=s.get("model_answer") or "",
+        )
+        resp = call_deepseek_mt2048(prompt, key)
+        verdict, fm, rat = parse_verdict(resp["raw_text"])
+        rows.append({
+            "instance_id": s["instance_id"],
+            "cell": s["cell"],
+            "provider": "deepseek",
+            "model_id": "deepseek-v4-pro",
+            "routing": "direct",
+            "max_tokens": 2048,
+            "http_status": resp["status"],
+            "error": resp.get("error"),
+            "retries": resp["retries"],
+            "latency_ms": resp["latency_ms"],
+            "prompt_tokens": resp.get("prompt_tokens"),
+            "completion_tokens": resp.get("completion_tokens"),
+            "raw_text": resp["raw_text"],
+            "parsed_verdict": verdict,
+            "parsed_failure_mode": fm,
+            "parsed_rationale": rat,
+            "opus_verdict_ref": s.get("opus_verdict"),
+            "gpt_verdict_ref": s.get("gpt_verdict"),
+        })
+        logmsg(
+            f"[mt2048] {i+1}/{len(sample)} {s['instance_id']:30} cell={s['cell']:14} "
+            f"status={resp['status']} verdict={verdict} retries={resp['retries']} "
+            f"completion_tokens={resp.get('completion_tokens')}"
+        )
+
+    OUT_PATH.parent.mkdir(parents=True, exist_ok=True)
+    with OUT_PATH.open("w", encoding="utf-8") as f:
+        for r in rows:
+            f.write(json.dumps(r, ensure_ascii=False) + "\n")
+
+    parsed = sum(1 for r in rows if r.get("parsed_verdict") is not None)
+    logmsg(f"[mt2048] wrote {len(rows)} rows, parsed_ok={parsed}")
+    logmsg("[mt2048] END")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
--- a/benchmarks/probes/judge-swap-validation/deepseek-responses.jsonl
+++ b/benchmarks/probes/judge-swap-validation/deepseek-responses.jsonl
@@ -0,0 +1,20 @@
+{"instance_id": "locomo_conv-50_q086", "cell": "no-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 10725, "prompt_tokens": 484, "completion_tokens": 360, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F4\",\n  \"rationale\": \"The model answered 'Surfing', which is a fabricated activity not present in the context and contradicts the correct answer 'Skiing'.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F4", "parsed_rationale": "The model answered 'Surfing', which is a fabricated activity not present in the context and contradicts the correct answer 'Skiing'."}
+{"instance_id": "locomo_conv-26_q059", "cell": "no-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 13726, "prompt_tokens": 572, "completion_tokens": 487, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F1\",\n  \"rationale\": \"The model refused to answer by stating it cannot be determined without context, effectively abstaining despite the ground truth providing a definitive answer.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F1", "parsed_rationale": "The model refused to answer by stating it cannot be determined without context, effectively abstaining despite the ground truth providing a definitive answer."}
+{"instance_id": "locomo_conv-44_q000", "cell": "no-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 8914, "prompt_tokens": 446, "completion_tokens": 281, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F4\",\n  \"rationale\": \"The model gives the year 1954, which has no basis in the provided context; the correct year derived from the statement 'I've had them for 3 years!' on 27 March 2023 is 2020.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F4", "parsed_rationale": "The model gives the year 1954, which has no basis in the provided context; the correct year derived from the statement 'I've had them for 3 years!' on 27 March 2023 is 2020."}
+{"instance_id": "locomo_conv-41_q080", "cell": "no-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 20988, "prompt_tokens": 496, "completion_tokens": 745, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F4\",\n  \"rationale\": \"The model answers 'A wind chime', which does not appear in the ground-truth context and contradicts the correct answer of a painting of a castle on a hill; this is a fabricated detail.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F4", "parsed_rationale": "The model answers 'A wind chime', which does not appear in the ground-truth context and contradicts the correct answer of a painting of a castle on a hill; this is a fabricated detail."}
+{"instance_id": "locomo_conv-50_q086", "cell": "oracle-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 5959, "prompt_tokens": 484, "completion_tokens": 151, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer 'Skiing' exactly matches the ground-truth answer derived from the context indicating Calvin plans to go skiing after the tour.\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer 'Skiing' exactly matches the ground-truth answer derived from the context indicating Calvin plans to go skiing after the tour."}
+{"instance_id": "locomo_conv-26_q059", "cell": "oracle-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 16766, "prompt_tokens": 568, "completion_tokens": 564, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model's answer 'Yes' captures that Caroline is religious to some extent but omits the necessary qualifier that she is only 'somewhat, but not extremely religious', making it incomplete.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model's answer 'Yes' captures that Caroline is religious to some extent but omits the necessary qualifier that she is only 'somewhat, but not extremely religious', making it incomplete."}
+{"instance_id": "locomo_conv-44_q000", "cell": "oracle-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 4835, "prompt_tokens": 446, "completion_tokens": 139, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer matches the ground truth year derived from the context (2023 minus 3 years = 2020).\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer matches the ground truth year derived from the context (2023 minus 3 years = 2020)."}
+{"instance_id": "locomo_conv-41_q080", "cell": "oracle-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 15824, "prompt_tokens": 494, "completion_tokens": 719, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model answered only 'A painting,' missing the necessary detail that it was a painting of a castle on a hill, which is required to fully specify what she made as a reminder of the England trip.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model answered only 'A painting,' missing the necessary detail that it was a painting of a castle on a hill, which is required to fully specify what she made as a reminder of the England trip."}
+{"instance_id": "locomo_conv-50_q086", "cell": "full-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 7692, "prompt_tokens": 483, "completion_tokens": 219, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer 'skiing' matches the ground truth activity 'Skiing' from the context, with no additional incorrect claims.\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer 'skiing' matches the ground truth activity 'Skiing' from the context, with no additional incorrect claims."}
+{"instance_id": "locomo_conv-26_q059", "cell": "full-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 6206, "prompt_tokens": 568, "completion_tokens": 184, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F1\",\n  \"rationale\": \"The model answered 'unknown', which is an explicit refusal to answer, fitting the F1 (ABSTAIN) failure mode.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F1", "parsed_rationale": "The model answered 'unknown', which is an explicit refusal to answer, fitting the F1 (ABSTAIN) failure mode."}
+{"instance_id": "locomo_conv-44_q000", "cell": "full-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 4956, "prompt_tokens": 446, "completion_tokens": 145, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer '2020' matches the ground-truth year derived from the context (2023 minus 3 years).\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer '2020' matches the ground-truth year derived from the context (2023 minus 3 years)."}
+{"instance_id": "locomo_conv-41_q080", "cell": "full-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 15491, "prompt_tokens": 494, "completion_tokens": 526, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly states she made a painting but omits the required detail that it was of a castle on a hill.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly states she made a painting but omits the required detail that it was of a castle on a hill."}
+{"instance_id": "locomo_conv-50_q086", "cell": "retrieval", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 28851, "prompt_tokens": 483, "completion_tokens": 1024, "raw_text": "We are given a question: \"What sports activity is Calvin planning to try after the tour with Frank Ocean?\" The ground-truth answer is \"Skiing\". The ground-truth supporting context is a dialogue from Session 9: Calvin says: \"Nope, not yet! I haven't been there before, but it's on my to-do list for after my tour with Frank Ocean ends. Can't wait to see them in person! Look at that snowy peak. [image: a photo of a person on skis on a snowy mountain]\" This implies the activity is skiing, as indicated by the image and the context of a snowy peak. So the correct answer is \"Skiing\". The model's answer is \"Not mentioned\". That is incorrect. The model says \"Not mentioned\" which means it claims the answer is not mentioned in the context. But the context does mention skiing implicitly. So the model failed. Which failure mode? The model does not explicitly refuse or say it doesn't know? Actually \"Not mentioned\" could be considered a refusal or claiming that it's not in the text. The decision tree: F1 (ABSTAIN) is \"Does the model explicitly refuse or say it does not know?\" The model says \"Not mentioned\". That is not exactly saying \"I don't know\", it's saying the information isn't mentioned. But in many cases, \"not mentioned\" is a way of abstaining because it indicates the model cannot answer. But the instructions: \"explicitly refuse or say it does not know\" - \"Not mentioned\" is somewhat saying the answer is not provided, so it's like saying I can't answer based on the text. However, the model might be evaluated on whether it answered correctly. The question asks \"What sports activity...\" and the model says \"Not mentioned\" which is a claim that the text doesn't mention it. That is an incorrect answer about the content of the context. So it's not abstaining in the sense of \"I don't know\" but rather stating that the text doesn't mention it, which is a factual error. So it's not F1. F5: OFF-TOPIC - does the model answer a different question? It answers the question by saying it's not mentioned. That's an answer to the question, albeit wrong. So not off-topic. F4: HALLUCINATED - relies on entities, names, dates, or claims that do NOT appear in the ground-truth context. The model says \"Not mentioned\". This is a claim that the information is not present. That claim is a fabrication because the context clearly implies skiing. The model is not fabricating entities, but making a false claim about the text. This might fall under F3 (INCORRECT) or F4. Let's check: \"Does the model correctly state SOME required facts but miss others, without stating any incorrect facts?\" It does not state any required facts; it states no correct facts, it just says \"Not mentioned\". So it's not partial (F2). It states a fact derived from the context but gets them wrong? It says \"Not mentioned\" which is a statement about the context. That is an incorrect fact. So F3: Otherwise (model states facts derived from the context but gets them wrong): F3. However, F4 says \"Does the model rely on entities, names, dates, or claims that do NOT appear in the ground-truth context (fabrication)?\" The claim \"Not mentioned\" is a claim that does not appear in the context; it's a fabricated statement about the context. But the decision tree sequence: 1. F1? No. 2. F5? No, it's answering the question directly. 3. F4? Does the model rely on entities, names, dates, or claims that do NOT appear in the ground-truth context? The model's answer is \"Not mentioned\". The claim that the activity is not mentioned is not present in the context; it's a fabricated assertion. So I think this falls under F4: Hallucinated, because it invents a claim that the information is missing. But many would argue that \"not mentioned\" is simply an incorrect answer, not necessarily a hallucination. However, the decision tree specifies F4 as \"relies on entities, names, dates, or claims that do NOT appear in the ground-truth context\". The claim \"not mentioned\" is a claim that is not supported by the context. In fact, the context does mention it indirectly. So the model is making a false claim. That seems like F4. I'll consider typical usage: If a model says \"not mentioned\" when it is mentioned, it's often considered a hallucination of omission - it's claiming something that isn't true about the text. So I'd go with F4. But let's double-check: could it be F3? \"Otherwise (model states facts derived from the context but gets", "parsed_verdict": null, "parsed_failure_mode": null, "parsed_rationale": null}
+{"instance_id": "locomo_conv-26_q059", "cell": "retrieval", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 12082, "prompt_tokens": 568, "completion_tokens": 385, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F3\",\n  \"rationale\": \"The model answered 'No' but the ground truth states she is 'Somewhat, but not extremely religious,' making the answer factually incorrect.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F3", "parsed_rationale": "The model answered 'No' but the ground truth states she is 'Somewhat, but not extremely religious,' making the answer factually incorrect."}
+{"instance_id": "locomo_conv-44_q000", "cell": "retrieval", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 21631, "prompt_tokens": 446, "completion_tokens": 767, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F3\",\n  \"rationale\": \"The model incorrectly stated that the year is not mentioned, while the context clearly indicates adoption in 2020 (three years before March 2023).\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F3", "parsed_rationale": "The model incorrectly stated that the year is not mentioned, while the context clearly indicates adoption in 2020 (three years before March 2023)."}
+{"instance_id": "locomo_conv-41_q080", "cell": "retrieval", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 9374, "prompt_tokens": 494, "completion_tokens": 265, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model identified that Maria made a painting, but omitted the required specific subject matter (a castle on a hill), making the answer incomplete.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model identified that Maria made a painting, but omitted the required specific subject matter (a castle on a hill), making the answer incomplete."}
+{"instance_id": "locomo_conv-50_q086", "cell": "agentic", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 9916, "prompt_tokens": 493, "completion_tokens": 419, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F1\",\n  \"rationale\": \"The model did not answer the question; it simply reported that it reached the maximum number of tool turns, effectively abstaining from providing the requested sports activity.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F1", "parsed_rationale": "The model did not answer the question; it simply reported that it reached the maximum number of tool turns, effectively abstaining from providing the requested sports activity."}
+{"instance_id": "locomo_conv-26_q059", "cell": "agentic", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 17581, "prompt_tokens": 601, "completion_tokens": 596, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F4\",\n  \"rationale\": \"The model references a necklace representing 'love, faith and strength' that does not appear in the provided ground-truth context, constituting a hallucination.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F4", "parsed_rationale": "The model references a necklace representing 'love, faith and strength' that does not appear in the provided ground-truth context, constituting a hallucination."}
+{"instance_id": "locomo_conv-44_q000", "cell": "agentic", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 28694, "prompt_tokens": 456, "completion_tokens": 1024, "raw_text": "We are asked: \"Which year did Audrey adopt the first three of her dogs?\" Ground truth: 2020. Ground-truth supporting context: Session 1 (1:10 pm on 27 March, 2023): Audrey: \"I've had them for 3 years! Their names are Pepper, Precious and Panda. I can't live without my little ones!\" So the conversation is on 27 March 2023, she says she's had them for 3 years. So adoption year would be 2023 - 3 = 2020.\n\nModel's answer: \"Max tool turns reached (3 turns, 3 tools used).\" That's not an answer to the question. It appears the model failed to provide an answer because it hit a tool turn limit. So the model did not answer the question. It gave a system message or error. We need to evaluate: Is this correct? The model's answer is essentially no answer. It is not providing any information. So it does not contain the required facts. So verdict is incorrect. Then failure mode: Check decision tree. Does the model explicitly refuse or say it does not know? The model's output: \"Max tool turns reached (3 turns, 3 tools used).\" This is not explicitly refusing or saying \"I don't know\". It's a technical limit message. It's likely an error from the system, not a response to the question. So not F1. Does the model answer a different question? No, it doesn't answer any question. It just states a tool limit. So F5? F5 is \"off-topic\" meaning the model answers a different question than was asked. But here it's not answering a different question; it's giving a status message. That might be considered off-topic? Or maybe it's F1? The decision tree: step 1: \"Does the model explicitly refuse or say it does not know?\" The message \"Max tool turns reached\" could be interpreted as the model cannot complete the task, thus implicitly it does not know/cannot answer, but it's not an explicit refusal. However, typical evaluation tasks treat max tool turns as an inability to answer, which might be considered an abstention? But the phrasing is not \"I don't know\" or refusal. But it indicates it cannot provide an answer. I'd check: If the model says \"Max tool turns reached\" it might be a system-generated message when the assistant ran out of tool calls. This is essentially a failure to respond. In some rubrics, that's an F1 (ABSTAIN) if the model explicitly states it cannot answer or reaches a limit. But the instruction says \"explicitly refuse or say it does not know\". \"Max tool turns reached\" is not that. It might be considered a different output, not an answer. I think it's F5? Wait, F5: \"Does the model answer a DIFFERENT question than was asked (coherent but off-topic)?\" The model's output is \"Max tool turns reached (3 turns, 3 tools used).\" That's not an answer to any question; it's a status. It's not coherent as an answer. Could be considered off-topic? The definition of F5 is for when the model gives an answer, but to the wrong question. Here, no answer at all. Perhaps we need to see if it's F1 or something else. Alternatively, there might be a nuance: The model's answer is \"Max tool turns reached (3 turns, 3 tools used).\" That is a meta-response. It's not answering the user's question. I'd classify this as F1? The model is essentially unable to answer due to tool limit. It's akin to \"I couldn't retrieve the information.\" I'd lean towards F1 because it's a statement indicating it couldn't complete the task, which is a form of abstention/refusal. But does it explicitly say it doesn't know? Not exactly. However, in many evaluation frameworks, hitting a tool limit is considered a refusal to answer or an error. The decision tree says: 1. Does the model explicitly refuse or say it does not know? If yes, F1. \"Max tool turns reached\" doesn't explicitly refuse, but it's a technical limitation message. It might be better to treat it as F5 because it's off-topic (it's not answering the question, but talking about tool turns). But off-topic typically means the model gave an answer that is coherent but for a different question. This is more like a system error. I think this should be F1 (ABSTAIN). I'll check typical LLM evaluation: If the model outputs a message like \"I'm sorry, I can't answer that\" that's F1. If it outputs something like \"The sky is blue\" that's off-topic F5. Here it's \"Max tool turns reached\" –", "parsed_verdict": null, "parsed_failure_mode": null, "parsed_rationale": null}
+{"instance_id": "locomo_conv-41_q080", "cell": "agentic", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 9279, "prompt_tokens": 494, "completion_tokens": 293, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model answered 'a painting', which captures the general object but omits the required details that it was a painting of a castle on a hill.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model answered 'a painting', which captures the general object but omits the required details that it was a painting of a castle on a hill."}
--- a/benchmarks/probes/judge-swap-validation/deepseek-split-responses-v2-mt2048.jsonl
+++ b/benchmarks/probes/judge-swap-validation/deepseek-split-responses-v2-mt2048.jsonl
@@ -0,0 +1,7 @@
+{"instance_id": "locomo_conv-43_q155", "cell": "agentic", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "max_tokens": 2048, "http_status": 200, "error": null, "retries": 0, "latency_ms": 42713, "prompt_tokens": 457, "completion_tokens": 2000, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F3\",\n  \"rationale\": \"The model answered 'great', but the ground truth answer is 'It was a huge success.'; while 'great' appears in the context, it does not contain the required fact.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F3", "parsed_rationale": "The model answered 'great', but the ground truth answer is 'It was a huge success.'; while 'great' appears in the context, it does not contain the required fact.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-50_q015", "cell": "agentic", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "max_tokens": 2048, "http_status": 200, "error": null, "retries": 0, "latency_ms": 30086, "prompt_tokens": 556, "completion_tokens": 1053, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F4\",\n  \"rationale\": \"The model adds 'recharge' which is not supported by the ground truth or context, making it a fabricated claim, while missing the required fact 'calms'.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F4", "parsed_rationale": "The model adds 'recharge' which is not supported by the ground truth or context, making it a fabricated claim, while missing the required fact 'calms'.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-43_q155", "cell": "full-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "max_tokens": 2048, "http_status": 200, "error": null, "retries": 0, "latency_ms": 7693, "prompt_tokens": 457, "completion_tokens": 246, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer 'great' is a direct synonym for John's expressed feeling 'It was great!', which accurately answers the question about how he felt.\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer 'great' is a direct synonym for John's expressed feeling 'It was great!', which accurately answers the question about how he felt.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-50_q015", "cell": "full-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "max_tokens": 2048, "http_status": 200, "error": null, "retries": 0, "latency_ms": 15014, "prompt_tokens": 554, "completion_tokens": 522, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model's answer 'to relax' captures only part of the ground truth reason; it omits that visiting parks also calms him.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model's answer 'to relax' captures only part of the ground truth reason; it omits that visiting parks also calms him.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-43_q155", "cell": "oracle-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "max_tokens": 2048, "http_status": 200, "error": null, "retries": 0, "latency_ms": 24898, "prompt_tokens": 457, "completion_tokens": 886, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model's answer 'Great' captures only part of the ground-truth answer, omitting the 'huge success' aspect.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model's answer 'Great' captures only part of the ground-truth answer, omitting the 'huge success' aspect.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-47_q092", "cell": "oracle-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "max_tokens": 2048, "http_status": 200, "error": null, "retries": 0, "latency_ms": 13424, "prompt_tokens": 460, "completion_tokens": 608, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model's answer 'An application' omits the required detail that it was a computer application on smartphones, making it incomplete.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model's answer 'An application' omits the required detail that it was a computer application on smartphones, making it incomplete.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-50_q015", "cell": "oracle-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "max_tokens": 2048, "http_status": 200, "error": null, "retries": 0, "latency_ms": 9419, "prompt_tokens": 554, "completion_tokens": 398, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model's answer captures relaxation but omits the calming aspect, which is part of the ground truth reason.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model's answer captures relaxation but omits the calming aspect, which is part of the ground truth reason.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
--- a/benchmarks/probes/judge-swap-validation/deepseek-split-responses.jsonl
+++ b/benchmarks/probes/judge-swap-validation/deepseek-split-responses.jsonl
@@ -0,0 +1,7 @@
+{"instance_id": "locomo_conv-43_q155", "cell": "agentic", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 23121, "prompt_tokens": 457, "completion_tokens": 791, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model's answer 'great' reflects part of John's sentiment ('It was great!') but omits the required ground-truth fact that it was 'a huge success,' so it is incomplete.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model's answer 'great' reflects part of John's sentiment ('It was great!') but omits the required ground-truth fact that it was 'a huge success,' so it is incomplete.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-50_q015", "cell": "agentic", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 29466, "prompt_tokens": 556, "completion_tokens": 1024, "raw_text": "We are asked: \"Why does Dave regularly visit parks?\" Ground truth: \"because it relaxes and calms him\". Model's answer: \"to relax and recharge\". We need to determine if correct. The model's answer essentially says to relax and recharge. Ground truth says relaxes and calms him. \"Recharge\" is similar to calm? Recharge implies restore energy, not necessarily calm. The context says parks are calming, peaceful, relax. \"Recharge\" might be a slight addition but is it factually correct? The context: \"I've been spending lots of time at this beautiful park lately - so calming.\" and \"I've been exploring some parks on the weekends to relax - it's so peaceful\". The ground truth explicitly says \"because it relaxes and calms him\". The model adds \"recharge\" which is not directly in the ground truth, but does that make it incorrect? \"Recharge\" might be considered synonymous with relax/restore energy, but it's not exactly the same as \"calms\". The ground truth says \"relaxes and calms\". If the model says \"relax and recharge\", is that a partial fact? It includes relax, which is correct, but \"recharge\" is an extra claim that may not be present. The question: is \"recharge\" factually correct based on the context? The context mentions relaxing, calming, peaceful. Recharge is not mentioned. However, it's a plausible inference. The rubric: \"Extra detail is acceptable ONLY if it is factually correct.\" So if \"recharge\" is not in the ground truth supporting context, is it factually correct? The ground truth supporting context does not say anything about recharging. But is it a false claim? The model might be adding something that is not supported. It's not false, but it's an extrapolation. The instruction: \"correct\" means model's answer contains all required facts from ground truth, with no additional incorrect claims. So we need to see if \"to relax and recharge\" contains the required fact: the reason is to relax and calm. The model missed \"calms\" and added \"recharge\". So it doesn't contain all required facts (missing \"calms\") and it has an extra claim \"recharge\" which is not in ground truth. But is \"recharge\" incorrect? Not necessarily incorrect, but if it's not in ground truth, it's an extra detail that might be considered a fabrication if it's not supported by context. The context doesn't mention recharge, so the model is claiming something not in the context. That might be hallucination. However, \"recharge\" could be seen as synonymous with \"relax\" in common usage. The decision tree: if model relies on entities, names, dates, or claims that do NOT appear in the ground-truth context (fabrication) -> F4. The claim \"recharge\" is a claim that does not appear in the ground-truth context. So it might be F4. But it also missed \"calms\", so it could be partial (F2) if it states some required facts but misses others, without stating any incorrect facts. But here it added \"recharge\", which is not a required fact. If \"recharge\" is considered incorrect, then it's F3. But is \"recharge\" incorrect? Not necessarily. It's a minor synonym for relaxation, maybe. But the ground truth says \"relaxes and calms him\". The model says \"relax and recharge\". It didn't mention calm. So it missed a fact. So it's partial: it correctly states relax, but omits calms, and adds recharge. But F2 requires \"model correctly state SOME required facts but miss others, without stating any incorrect facts.\" If recharge is not incorrect, then it qualifies for F2. If recharge is incorrect (i.e., not supported and might be considered a claim not in context), then it might be F4. But F4 says \"rely on entities, names, dates, or claims that do NOT appear in the ground-truth context\". Recharge is a claim not in context. So that's F4. The tree: first F1 no, F5 off-topic no, F4 fabrication? Yes, it uses a claim \"recharge\" that is not in the context. But is it a fabrication? The model might be summarizing the idea of relaxing, and \"recharge\" is a common associated word. Usually, if it's a minor wording variation and still essentially correct, we'd mark correct. The ground truth is a short phrase: \"because it relaxes and calms him\". The model answer \"to relax and recharge\" is very similar. \"Recharge\" might be considered a synonym for rejuvenate, but the ground truth emphasizes calm. Is the model answer correct? I think many would consider \"to relax and recharge\" as capturing the same idea. The context says \"relax", "parsed_verdict": null, "parsed_failure_mode": null, "parsed_rationale": null, "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-43_q155", "cell": "full-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 15408, "prompt_tokens": 457, "completion_tokens": 523, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer 'great' accurately reflects John's positive feelings from the context, where he states 'It was great!'\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer 'great' accurately reflects John's positive feelings from the context, where he states 'It was great!'", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-50_q015", "cell": "full-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 11517, "prompt_tokens": 554, "completion_tokens": 387, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model states that Dave visits parks to relax, but misses the ground-truth detail that it also calms him.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model states that Dave visits parks to relax, but misses the ground-truth detail that it also calms him.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-43_q155", "cell": "oracle-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 29140, "prompt_tokens": 457, "completion_tokens": 1024, "raw_text": "We are asked: \"How did John feel after being able to jog without pain?\" Ground truth: \"It was a huge success.\" That is from the context: \"John: It was great! After being out for so long, jogging without any pain was a huge success.\" So the feeling is \"great\" and it was a huge success. Model answer: \"Great\". That captures the feeling \"great\". The ground truth answer only says \"It was a huge success.\" But the question is \"How did John feel?\" In the context, John says \"It was great! ... a huge success.\" So the feeling is \"great\". The model's answer \"Great\" is a synonym for \"great\". So it's correct. However, the ground truth answer is \"It was a huge success.\" That might be a paraphrase, but the question is about feeling. The ground-truth answer might be incomplete since it doesn't mention \"great\". But our task is to compare model's answer to ground truth. The ground truth provided is \"It was a huge success.\" That's from the excerpt. But the model's answer \"Great\" is directly from John's words \"It was great!\" Is \"Great\" a correct answer given the ground truth? The ground truth answer is \"It was a huge success.\" That's not exactly \"great\" but both indicate positive feeling. Could this be considered a partial? The ground truth answer is a single sentence. The model says \"Great\". Does it contain all required facts from ground truth? The ground truth says \"It was a huge success.\" The model says \"Great\". They are different. But the question is \"How did John feel?\" The ground truth answer is a rephrasing of his feeling? Actually, the supporting context says: \"It was great! ... jogging without any pain was a huge success.\" So John felt great, and it was a success. The ground truth answer is \"It was a huge success.\" That might be seen as an indirect answer: he felt it was a huge success, i.e., he felt successful, great. The model's answer \"Great\" captures the direct emotional descriptor. It might be considered equivalent? However, the ground truth is specifically \"It was a huge success.\" If the ground truth answer is that, then the model's answer \"Great\" might not contain the phrase \"huge success\". But the question asks \"How did John feel?\" The answer could be \"Great\" or \"It was a huge success\" both being acceptable summaries. But to be \"correct\", the model's answer must contain all required facts from ground truth. Here, ground truth fact: \"It was a huge success.\" Model says \"Great\". Is \"Great\" the same fact? Not exactly. The facts: John felt it was a huge success. The model says he felt great. Both are positive, but are they interchangeable? In the context, he said \"It was great!\" So \"great\" is literally his feeling. The ground truth answer might be a poor selection; but we are to evaluate if model's answer matches the ground truth. Since the ground truth answer is \"It was a huge success,\" the model's answer \"Great\" is not a paraphrase of that. It's a different statement. It doesn't include the notion of success. So it might be considered partial: it gives some of the feeling (great) but misses the explicit \"huge success\" part. However, the ground truth is just one sentence. If the model's answer is \"Great\", it doesn't contain the fact that it was a huge success. So is that a partial omission? The question: \"How did John feel?\" The ground truth says \"It was a huge success.\" That is a statement about the event, not a feeling. But they use that as the answer. If we strictly compare, the model's answer does not contain the ground truth fact. However, \"It was a huge success\" implies feeling successful, which is similar to great. But we need to apply the decision tree. The model is not refusing (F1), not off-topic (F5). Does it fabricate? No, \"Great\" is in the context. So F4 not. Does it correctly state SOME required facts but miss others, without stating any incorrect facts? It states he felt \"Great\". Does it miss the \"huge success\"? That might be missing the fact that he considered it a huge success. But the ground truth answer is just \"It was a huge success.\" The model answer is \"Great\". They are different facts. The model's answer doesn't contain the ground truth fact. Therefore, it might be incorrect (F3) if it gets the fact wrong. But it doesn't get it wrong; it just gives a different correct fact. The question is about feeling, the context says \"It was great!\" and \"a huge success.\" Both describe his feeling/reaction. So if the ground truth answer only extracted \"It was a huge", "parsed_verdict": null, "parsed_failure_mode": null, "parsed_rationale": null, "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-47_q092", "cell": "oracle-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 21200, "prompt_tokens": 460, "completion_tokens": 767, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly identifies that John created an application, but omits the required detail that it is a computer application on smartphones, as specified in the ground truth.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly identifies that John created an application, but omits the required detail that it is a computer application on smartphones, as specified in the ground truth.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-50_q015", "cell": "oracle-context", "provider": "deepseek", "model_id": "deepseek-v4-pro", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 7473, "prompt_tokens": 554, "completion_tokens": 198, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer 'To relax' is semantically equivalent to the ground-truth reason that visiting parks relaxes and calms him.\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer 'To relax' is semantically equivalent to the ground-truth reason that visiting parks relaxes and calms him.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
--- a/benchmarks/probes/judge-swap-validation/kappa-analysis.md
+++ b/benchmarks/probes/judge-swap-validation/kappa-analysis.md
@@ -0,0 +1,87 @@
+# §1.3g Judge Swap κ Analysis (4-Candidate Roster)
+
+**Date:** 2026-04-24 · **Sample:** N=20 stratified 4-per-cell from Stage 2-Retry κ-calibration set.
+
+**Consensus reference:** Opus 4.7 + GPT-5.4 agreement.
+
+**Opus≠GPT splits (excluded from κ):** **0 / 20**
+
+**Consensus subset size (denominator for κ):** **20 / 20**
+
+
+## Per-candidate results (ranked by κ descending)
+
+| Rank | Candidate | Model ID | Routing | κ | Raw agreement | n compared | Parse OK / 20 | Verdict |
+|------|-----------|----------|---------|-----|---------------|------------|----------------|---------|
+| 1 | kimi | `kimi-k2.6` | direct | **1.0000** | 100.00% (20/20) | 20 | 20/20 | **PASS (excellent)** |
+| 2 | minimax | `minimax/minimax-m2.7` | openrouter | **1.0000** | 100.00% (20/20) | 20 | 20/20 | **PASS (excellent)** |
+| 3 | deepseek | `deepseek-v4-pro` | direct | **1.0000** | 100.00% (18/18) | 18 | 18/20 | **PASS (excellent)** |
+| 4 | zhipu | `glm-5.1` | direct | **1.0000** | 100.00% (19/19) | 19 | 19/20 | **PASS (excellent)** |
+
+## Aggregate verdict: **MULTI_PASS**
+- PASS (κ ≥ 0.70): **4**
+- BORDERLINE (0.60 ≤ κ < 0.70): **0**
+- FAIL (κ < 0.60): **0**
+- INCONCLUSIVE (parse < 18/20): **0**
+
+### Sample-selection caveat (transparent)
+
+Full 100-row κ-calibration set has 7/100 = **7% Opus-GPT split rate**.
+Our 20-instance deterministic first-4-per-cell sample caught **0/20
+splits** — consistent with binomial variance at low end but signals
+selection bias toward "easier" instances where the two primary judges
+already agree. κ = 1.0 on all 4 candidates should be read as "all
+candidates match the consensus on a biased subset", not "all candidates
+match Opus+GPT on arbitrary LoCoMo instances". Recommended follow-up
+(Task 2.6): stratified-on-split re-probe at N=40-60 with split-inclusion
+oversampling to derive κ values that discriminate among the 4.
+
+### Secondary criteria (tie-break at κ=1.0)
+
+| Cand | Parse OK/20 | p50 latency | p95 latency | avg out tokens | retries | routing |
+|------|-------------|-------------|-------------|----------------|---------|---------|
+| zhipu | 19/20 | 9 s | 22 s | 357 | 0 | direct (api.z.ai) |
+| minimax | 20/20 | 12 s | 28 s | 483 | 0 | openrouter |
+| deepseek | 18/20 | 12 s | 29 s | 465 | 0 | direct (api.deepseek.com) |
+| kimi | 20/20 | **32 s** | **110 s** | 1002 (near cap) | 4 | direct (api.moonshot.ai) |
+
+**Revised ranking by secondary criteria** (κ tie-break via speed ×
+parse-reliability × direct-routing):
+
+1. **Zhipu GLM-5.1** — fastest p50 (9 s), direct, 19/20 parse. Best
+   operational profile for N=400 (Stage 3 × 5 cells × ~1 judge triple ⇒
+   ~2000 Gemini-replacement calls at ~9 s each ≈ 5 hr judge-thread
+   wall-clock).
+2. **DeepSeek V4-Pro** — direct, 18/20 parse, clean retries.
+3. **MiniMax M2.7 (via OR)** — 20/20 parse but OpenRouter fallback =
+   extra dependency.
+4. **Kimi k2.6** — 20/20 parse but 3.5× slower than Zhipu; N=400 judge
+   thread would be >20 hr on Kimi's p50. Operational deal-breaker for
+   Stage 3 timeline.
+
+**Revised recommended primary:** **ZHIPU**
+**Revised recommended backup:** **DEEPSEEK**
+
+Marko's original preference (Kimi/MiniMax) preserved as 3rd/4th picks —
+both PASS κ-wise but lose on speed (Kimi) or routing-stack (MiniMax).
+PM adjudicates final pick at manifest v6 brief time.
+
+## Per-cell breakdown (top-2 by κ)
+
+### kimi (kimi-k2.6)
+| Cell | n | Agree | Raw % |
+|------|---|-------|-------|
+| agentic | 4 | 4 | 100% |
+| full-context | 4 | 4 | 100% |
+| no-context | 4 | 4 | 100% |
+| oracle-context | 4 | 4 | 100% |
+| retrieval | 4 | 4 | 100% |
+
+### minimax (minimax/minimax-m2.7)
+| Cell | n | Agree | Raw % |
+|------|---|-------|-------|
+| agentic | 4 | 4 | 100% |
+| full-context | 4 | 4 | 100% |
+| no-context | 4 | 4 | 100% |
+| oracle-context | 4 | 4 | 100% |
+| retrieval | 4 | 4 | 100% |
--- a/benchmarks/probes/judge-swap-validation/kappa-split-analysis.md
+++ b/benchmarks/probes/judge-swap-validation/kappa-split-analysis.md
@@ -0,0 +1,158 @@
+# §1.3h Judge Swap Stratified Re-Probe — Dual-κ Split Analysis
+
+**Date:** 2026-04-24 (evening)  **Parent commit:** `8a2f0e6` (§1.3g anchor)
+**Source:** `benchmarks/results/locomo-mini-n20-retry-2026-04-24T00-02-12Z.jsonl` (100 rows, authoritative).
+
+**Split pool (Opus ≠ GPT):** **7** instances (use-all-available per PM amendment §1.3H-POOL-SHORTAGE OPTION 1).
+**Split cell distribution:** `agentic=2`, `full-context=2`, `oracle-context=3`, `no-context=0`, `retrieval=0`.
+**Structural observation:** All 7 splits have `Opus=correct / GPT=incorrect`. Zero inverse splits.
+
+**PM-amended verdict caveat:** n=7 < 12 minimum for meaningful discrimination. All findings carry `n=7 pool-limited, observational not confirmatory` caveat. Primary value = operational signal (parse + latency + MiniMax routing) + bias detection via p_opus/p_gpt balance.
+
+---
+
+## §1 Per-candidate split metrics
+
+| Cand | Parse | n_eval | p_opus | p_gpt | κ_vs_opus | κ_vs_gpt | κ_cons | p50 lat | p95 lat | Routing | Split verdict |
+|------|-------|--------|--------|-------|-----------|----------|--------|---------|---------|---------|---------------|
+| kimi | 5/7 (71.4%) | 5 | 80.0% (4/5) | 20.0% (1/5) | 0.0 deg | 0.0 deg | **N/A deg** | 32.0 s | 51.0 s | `direct` | **N/A deg** |
+| minimax | 7/7 (100.0%) | 7 | 85.7% (6/7) | 14.3% (1/7) | 0.0 deg | 0.0 deg | **N/A deg** | 16.6 s | 21.2 s | `openrouter` | **N/A deg** |
+| deepseek | 5/7 (71.4%) | 5 | 40.0% (2/5) | 60.0% (3/5) | 0.0 deg | 0.0 deg | **N/A deg** | 15.4 s | 21.2 s | `direct` | **N/A deg** |
+| zhipu | 6/7 (85.7%) | 6 | 0.0% (0/6) | 100.0% (6/6) | 0.0 deg | 1.0 deg | **N/A deg** | 17.9 s | 22.5 s | `direct` | **N/A deg** |
+
+### §1.1 Why κ on split-only is mathematically degenerate
+
+All 7 splits are `Opus=correct / GPT=incorrect`. For a Cohen's κ on the split subset:
+- `κ_vs_opus`: reference (Opus) has zero variance → ref_counts has single class → p_e = p_o → κ = (0 − 0)/(1 − 1) undefined → reported as `0.0 deg`
+- `κ_vs_gpt`: same pathology → `0.0 deg` (except Zhipu, which perfectly matches GPT → `κ = 1.0 deg` by convention when `p_o = p_e = 1`)
+
+**κ_cons = min() of two degenerate values is itself degenerate.** The informative signal on splits is NOT κ, it is the **p_opus / p_gpt balance** which reveals each candidate's calibration bias. The brief's κ-threshold classification (PASS/BORDERLINE/FAIL @ 0.70/0.60) does not apply to this structural case.
+
+### §1.2 BIAS INTERPRETATION (actionable)
+
+Since all splits are Opus=correct / GPT=incorrect:
+
+- `p_opus=100%` = candidate lock-stepped with Opus (maximally lenient, adds no new signal vs Opus)
+- `p_gpt=100%` = candidate lock-stepped with GPT (maximally strict, adds no new signal vs GPT)
+- `p_opus ≈ p_gpt ≈ 50%` = candidate judges independently (ideal for 3-judge ensemble)
+
+| Cand | p_opus | p_gpt | Calibration profile |
+|------|--------|-------|----------------------|
+| **DeepSeek** | 40% | 60% | **Most balanced — slight GPT lean, clearest independent judgment** |
+| Kimi | 80% | 20% | Opus-lenient (echoes Opus 4/5 times) |
+| MiniMax | 86% | 14% | Strongly Opus-lenient (echoes Opus 6/7 times) |
+| Zhipu | **0%** | **100%** | **Pure GPT-echo — 6/6 parsed splits = GPT verbatim** |
+
+**This is the critical finding the §1.3g probe could not produce.** On the 20 unanimous cases, all 4 candidates scored κ=1.0 — appeared equivalent. On split cases where judges diverge, DeepSeek uniquely exhibits independent calibration; Zhipu is effectively a GPT echo; Kimi/MiniMax echo Opus. For a 3-judge ensemble replacing Gemini, **independence from existing judges is a feature**, not a bug.
+
+---
+
+## §2 Aggregate κ on combined sample (27 = 20 unanimous + 7 splits)
+
+Unanimous portion reuses §1.3g verdicts (consensus-matched, contributes κ=1.0 content). Split portion uses dual-reference with conservative min. Combined sample is non-degenerate (both classes present in Opus and GPT reference columns on the aggregate).
+
+| Cand | n_combined | κ_agg_vs_opus | κ_agg_vs_gpt | κ_agg_cons | Aggregate verdict |
+|------|------------|----------------|---------------|-------------|---------------------|
+| **deepseek** | 23 | 0.7089 | 0.7473 | **0.7089** | **PASS (only one ≥ 0.70)** |
+| kimi | 25 | 0.9110 | 0.5763 | **0.5763** | FAIL (κ_vs_gpt < 0.70) |
+| minimax | 27 | 0.9222 | 0.4564 | **0.4564** | FAIL (κ_vs_gpt < 0.70) |
+| zhipu | 25 | 0.4444 | 1.0000 | **0.4444** | FAIL (κ_vs_opus < 0.70) |
+
+**Aggregate κ ranking (descending):** `deepseek, kimi, minimax, zhipu`
+
+The aggregate picture inverts the split-only one: DeepSeek is the only candidate clearing 0.70 on the conservative dual-reference combined sample. Zhipu's perfect κ_vs_gpt is offset by its ~0.44 κ_vs_opus — clear asymmetry consistent with the bias finding.
+
+---
+
+## §3 Operational snapshot (split-case specific)
+
+| Cand | Split parse | p50 (split) | p95 (split) | Routing | MiniMax status |
+|------|-------------|-------------|-------------|---------|-------------------|
+| kimi | 5/7 (71%) | 32.0 s | 51.0 s | direct | — |
+| minimax | 7/7 (100%) | 16.6 s | 21.2 s | **openrouter** | **direct FAILED — international 401/err + legacy 401/err** |
+| deepseek | 5/7 (71%) | 15.4 s | 21.2 s | direct | — |
+| zhipu | 6/7 (86%) | 17.9 s | 22.5 s | direct | — |
+
+### §3.1 MiniMax direct routing verdict
+
+**Primary operational value focus per PM amendment.** Result:
+
+- `direct_international` (`https://api.minimaxi.com/v1/text/chatcompletion_v2?GroupId=...`) — FAILED
+- `direct_legacy` (`https://api.minimax.chat/v1/text/chatcompletion_v2?GroupId=...`) — FAILED
+- `openrouter` fallback — SUCCEEDED (7/7 parse)
+
+**Routing resolution: `direct_failed_fell_back_openrouter`.** The newly-added `MINIMAX_GROUP_ID` did not unblock direct routing. Either the key is not provisioned for v2 endpoints, the endpoint paths / model names need further research (e.g. raw v1 `chatcompletion_pro` format differs from OpenAI-compatible v2), or the account tier lacks direct API access. **Manifest v6 must route MiniMax via OpenRouter if MiniMax is selected** — direct routing is not a viable assumption.
+
+### §3.2 Parse rate on challenging vs unanimous (regression check)
+
+| Cand | Unanimous parse (§1.3g) | Split parse (§1.3h) | Δ |
+|------|--------------------------|-----------------------|---|
+| kimi | 20/20 (100%) | 5/7 (71%) | **−29 pp** (2 timeout-retries exhausted on long-context instances) |
+| minimax | 20/20 (100%) | 7/7 (100%) | 0 pp |
+| deepseek | 18/20 (90%) | 5/7 (71%) | **−19 pp** (2 verdict=None — likely malformed JSON or over-length) |
+| zhipu | 19/20 (95%) | 6/7 (86%) | −9 pp |
+
+Challenging cases expose degradation in Kimi and DeepSeek parse reliability. Zhipu and MiniMax (via OR) hold up best on difficult instances.
+
+---
+
+## §4 Aggregate verdict: **INCONCLUSIVE_BUT_OPERATIONAL_SIGNAL**
+
+**Per PM-amended §4.4.** Reasons (any single one triggers):
+1. Split pool n=7 < 12 minimum for confirmatory κ discrimination
+2. Parse rate <80% on 2 candidates (Kimi, DeepSeek) on split subset
+3. Split-only κ is mathematically degenerate (structural Opus=correct / GPT=incorrect orientation)
+
+However, three **operational-signal findings** are fully actionable:
+
+- **A. Bias profile (novel finding):** DeepSeek is the only candidate with independent calibration. Zhipu is a GPT-echo (0% agreement with Opus on splits). MiniMax/Kimi are Opus-echoes.
+- **B. Aggregate κ on combined 27:** DeepSeek is the only candidate κ_cons ≥ 0.70 on the non-degenerate combined sample.
+- **C. MiniMax direct routing FAILED** even with MINIMAX_GROUP_ID. Manifest v6 cannot rely on direct; OpenRouter fallback is the operational truth.
+
+---
+
+## §5 Revised recommendation (integrating §1.3g + §1.3h)
+
+**The §1.3g heuristic-based recommendation (Zhipu primary, DeepSeek backup) is not supported by §1.3h findings.**
+
+### §5.1 Two viable paths, each with a clear trade-off
+
+**Path 1 — Independence priority (DeepSeek primary):**
+- Pros: Only candidate with balanced independent calibration (p_opus=40%, p_gpt=60%); only one clearing κ_agg_cons ≥ 0.70 on combined 27; direct routing works.
+- Cons: Split parse rate 71% (2/7 NULL verdicts) — operational risk for N=400 unless root cause of parse drops on hard cases is addressed (likely `max_tokens=1024` ceiling being hit on reasoning chains, same pathology as Kimi; could be fixed by bumping to 2048/4096).
+- Mitigation: Boost `max_tokens` to 2048+ for DeepSeek on judge role; re-verify parse rate on a follow-up probe if PM accepts this path.
+
+**Path 2 — Reliability priority (Zhipu primary, MiniMax-via-OR backup):**
+- Pros: Zhipu fastest p50 (9s from §1.3g, 18s on splits) + direct routing + parse 86% on splits; MiniMax 100% parse via OR.
+- Cons: Zhipu is effectively a GPT-echo (0% independence on splits) — adds minimal new signal to an Opus+GPT+GPT-echo ensemble. Ensemble's value is correlated, not complementary; `κ(Gemini-replacement, GPT) ≈ 1.0` violates the independence assumption that motivated having a third judge in the first place.
+
+### §5.2 PM adjudication required
+
+**Recommended PM pick based on §1.3h empirical data:**
+
+- **Primary: DEEPSEEK** with `max_tokens=2048` bump (independence > 80% parse in a 3-judge ensemble where the third judge's role is adding new information)
+- **Backup: ZHIPU** (if parse-reliability becomes binding, accept the independence compromise; it's better than no third judge)
+
+**Alternative if PM prefers §1.3g-style operational-first ranking:**
+
+- Primary: ZHIPU (parse + speed + direct), Backup: DEEPSEEK-boosted (independence insurance if Zhipu ensemble correlation becomes a problem at N=400)
+
+**If PM wants to fix the unknowns before committing:**
+
+- Run a mini-confirmation probe: DeepSeek with `max_tokens=2048` on the same 7 splits + 13 additional split-candidates from a larger calibration pool (if one is built). Cost ~$0.50.
+
+---
+
+## §6 Halt ping data (machine-readable summary)
+
+- `aggregate_verdict`: **INCONCLUSIVE_BUT_OPERATIONAL_SIGNAL**
+- `ranking_by_k_agg_cons_desc`: deepseek (0.709), kimi (0.576), minimax (0.456), zhipu (0.444)
+- `ranking_by_p_opus_balance`: deepseek (best: 40/60), kimi (80/20), minimax (86/14), zhipu (worst: 0/100)
+- `recommended_primary` (empirical): **DEEPSEEK** (with max_tokens bump) — supersedes §1.3g Zhipu pick
+- `recommended_primary` (operational-first, if PM prefers): **ZHIPU** with independence caveat
+- `recommended_backup`: **ZHIPU** or **DEEPSEEK-boosted** (mirror of primary choice)
+- `minimax_routing_resolution`: **direct_failed_fell_back_openrouter**
+- `split_cases_selected`: 7/7 (all available from 100-row authoritative source)
+- `empirical_finding_novel`: Zhipu 100% GPT-lock-step on splits → low independence from existing GPT judge
+
+Raw summary also saved to `_summary-split.json`.
--- a/benchmarks/probes/judge-swap-validation/kappa-split-analysis.py
+++ b/benchmarks/probes/judge-swap-validation/kappa-split-analysis.py
@@ -0,0 +1,417 @@
+"""
+§1.3h Dual-κ Split Analysis
+============================
+
+Computes:
+  - Per candidate: parse_success, p_opus, p_gpt, κ_vs_opus, κ_vs_gpt,
+    κ_conservative = min(κ_vs_opus, κ_vs_gpt), latency p50/p95 on splits,
+    routing_actual, split_verdict
+  - Aggregate κ on combined 40 (7 splits + 33 unanimous from §1.3g* where
+    split sample uses dual-reference with conservative min; unanimous
+    uses consensus)
+    *Note: §1.3g sample was 20 unanimous; combined = 7 splits + 20
+    unanimous = 27. The brief says "aggregate 40" presuming 20+20, but
+    our actual sample is 7+20=27. We compute aggregate on the actual 27.
+  - Aggregate verdict classification
+  - Ranking by κ_conservative desc
+  - Primary/backup recommendation
+
+Writes kappa-split-analysis.md.
+"""
+from __future__ import annotations
+
+import json
+import statistics
+from pathlib import Path
+from typing import Any
+
+PROBE_DIR = Path("D:/Projects/waggle-os/benchmarks/probes/judge-swap-validation")
+SPLIT_SAMPLE = PROBE_DIR / "split-cases-sample.jsonl"
+UNANIMOUS_SAMPLE = PROBE_DIR / "sample-instances.jsonl"
+OUT = PROBE_DIR / "kappa-split-analysis.md"
+
+CANDIDATES = ["kimi", "minimax", "deepseek", "zhipu"]
+
+
+def load_jsonl(path: Path) -> list[dict]:
+    rows = []
+    with path.open("r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                rows.append(json.loads(line))
+    return rows
+
+
+def cohen_kappa(pairs: list[tuple[str, str]]) -> tuple[float, dict]:
+    """Cohen's kappa on 2-class (correct/incorrect) agreement. Pairs are
+    (cand_verdict, ref_verdict). Returns (kappa, detail_dict)."""
+    if not pairs:
+        return (float("nan"), {"n": 0, "agree": 0, "po": 0.0, "pe": 0.0})
+    n = len(pairs)
+    agree = sum(1 for c, r in pairs if c == r)
+    po = agree / n
+    # Marginals
+    cand_counts = {"correct": 0, "incorrect": 0}
+    ref_counts = {"correct": 0, "incorrect": 0}
+    for c, r in pairs:
+        cand_counts[c] = cand_counts.get(c, 0) + 1
+        ref_counts[r] = ref_counts.get(r, 0) + 1
+    pe = sum(
+        (cand_counts.get(v, 0) / n) * (ref_counts.get(v, 0) / n)
+        for v in ("correct", "incorrect")
+    )
+    if pe >= 1.0:
+        return (1.0 if po == 1.0 else float("nan"),
+                {"n": n, "agree": agree, "po": po, "pe": pe})
+    kappa = (po - pe) / (1.0 - pe)
+    return (kappa, {"n": n, "agree": agree, "po": po, "pe": pe,
+                    "cand_counts": cand_counts, "ref_counts": ref_counts})
+
+
+def split_verdict(kappa_cons: float) -> str:
+    if kappa_cons != kappa_cons:  # NaN
+        return "INCONCLUSIVE"
+    if kappa_cons >= 0.70:
+        return "PASS"
+    if kappa_cons >= 0.60:
+        return "BORDERLINE"
+    return "FAIL"
+
+
+def pct(x: int, n: int) -> str:
+    if n == 0:
+        return "—"
+    return f"{x}/{n} ({x * 100 / n:.1f}%)"
+
+
+def fmt_k(x: float) -> str:
+    if x != x:
+        return "NaN"
+    return f"{x:.4f}"
+
+
+def main() -> int:
+    # Load split sample + per-candidate responses
+    splits = load_jsonl(SPLIT_SAMPLE)
+    n_split = len(splits)
+    print(f"Split sample: {n_split} instances")
+
+    resp = {}
+    for c in CANDIDATES:
+        p = PROBE_DIR / f"{c}-split-responses.jsonl"
+        if not p.exists():
+            print(f"WARN missing: {p}")
+            resp[c] = []
+            continue
+        resp[c] = load_jsonl(p)
+        print(f"  {c}: {len(resp[c])} responses")
+
+    # Index split sample for reference lookup
+    split_index = {(s["instance_id"], s["cell"]): s for s in splits}
+
+    # Per-candidate metrics on split subset
+    per_cand_split: dict[str, dict] = {}
+    for c in CANDIDATES:
+        rows = resp[c]
+        parsed_rows = [r for r in rows if r.get("parsed_verdict") in ("correct", "incorrect")]
+        parse_success = len(parsed_rows)
+        pairs_vs_opus = []
+        pairs_vs_gpt = []
+        p_opus_count = 0
+        p_gpt_count = 0
+        latencies = []
+        routings = []
+        for r in parsed_rows:
+            key = (r["instance_id"], r["cell"])
+            ref = split_index.get(key)
+            if ref is None:
+                continue
+            cand_v = r["parsed_verdict"]
+            op_v = ref["opus_verdict"]
+            gp_v = ref["gpt_verdict"]
+            pairs_vs_opus.append((cand_v, op_v))
+            pairs_vs_gpt.append((cand_v, gp_v))
+            if cand_v == op_v:
+                p_opus_count += 1
+            if cand_v == gp_v:
+                p_gpt_count += 1
+            latencies.append(r.get("latency_ms") or 0)
+            routings.append(r.get("routing"))
+
+        n_eval = len(pairs_vs_opus)
+        p_opus = p_opus_count / n_eval if n_eval else float("nan")
+        p_gpt = p_gpt_count / n_eval if n_eval else float("nan")
+        k_opus, k_opus_det = cohen_kappa(pairs_vs_opus)
+        k_gpt, k_gpt_det = cohen_kappa(pairs_vs_gpt)
+        k_cons = min(k_opus, k_gpt) if (k_opus == k_opus and k_gpt == k_gpt) else float("nan")
+        lat_p50 = int(statistics.median(latencies)) if latencies else 0
+        lat_p95 = int(
+            sorted(latencies)[max(0, int(len(latencies) * 0.95) - 1)]
+        ) if latencies else 0
+        routing_mode = max(set(routings), key=routings.count) if routings else "unknown"
+
+        per_cand_split[c] = {
+            "parse_success": parse_success,
+            "n_total": n_split,
+            "n_eval": n_eval,
+            "p_opus": p_opus,
+            "p_gpt": p_gpt,
+            "k_opus": k_opus,
+            "k_gpt": k_gpt,
+            "k_cons": k_cons,
+            "k_opus_det": k_opus_det,
+            "k_gpt_det": k_gpt_det,
+            "lat_p50_ms": lat_p50,
+            "lat_p95_ms": lat_p95,
+            "routing": routing_mode,
+            "split_verdict": split_verdict(k_cons),
+            "p_opus_count": p_opus_count,
+            "p_gpt_count": p_gpt_count,
+        }
+
+    # Aggregate κ on combined sample (27 = 20 unanimous from §1.3g + 7 splits)
+    # Unanimous: use consensus (opus_verdict == gpt_verdict); skip if NA
+    # Split: use dual-reference conservative min for the aggregate candidate
+    unanimous = load_jsonl(UNANIMOUS_SAMPLE)
+    # Load §1.3g per-candidate responses (reuse unanimous verdicts)
+    unanimous_resp = {}
+    for c in CANDIDATES:
+        p_unc = PROBE_DIR / f"{c}-responses.jsonl"
+        if p_unc.exists():
+            unanimous_resp[c] = load_jsonl(p_unc)
+        else:
+            unanimous_resp[c] = []
+
+    unanimous_index = {(u["instance_id"], u["cell"]): u for u in unanimous}
+
+    per_cand_agg: dict[str, dict] = {}
+    for c in CANDIDATES:
+        # Build combined pair list for candidate vs Opus and vs GPT
+        pairs_opus = []
+        pairs_gpt = []
+        # Unanimous portion
+        for r in unanimous_resp.get(c, []):
+            if r.get("parsed_verdict") not in ("correct", "incorrect"):
+                continue
+            key = (r["instance_id"], r["cell"])
+            ref = unanimous_index.get(key)
+            if ref is None:
+                continue
+            cv = r["parsed_verdict"]
+            pairs_opus.append((cv, ref["opus_verdict"]))
+            pairs_gpt.append((cv, ref["gpt_verdict"]))
+        # Split portion
+        for r in resp[c]:
+            if r.get("parsed_verdict") not in ("correct", "incorrect"):
+                continue
+            key = (r["instance_id"], r["cell"])
+            ref = split_index.get(key)
+            if ref is None:
+                continue
+            cv = r["parsed_verdict"]
+            pairs_opus.append((cv, ref["opus_verdict"]))
+            pairs_gpt.append((cv, ref["gpt_verdict"]))
+        k_op, _ = cohen_kappa(pairs_opus)
+        k_gp, _ = cohen_kappa(pairs_gpt)
+        k_ag_cons = min(k_op, k_gp) if (k_op == k_op and k_gp == k_gp) else float("nan")
+        per_cand_agg[c] = {
+            "n_combined": len(pairs_opus),
+            "k_agg_vs_opus": k_op,
+            "k_agg_vs_gpt": k_gp,
+            "k_agg_cons": k_ag_cons,
+        }
+
+    # Ranking by k_cons desc
+    ranking = sorted(
+        CANDIDATES,
+        key=lambda c: (
+            -1e9 if per_cand_split[c]["k_cons"] != per_cand_split[c]["k_cons"]
+            else per_cand_split[c]["k_cons"]
+        ),
+        reverse=True,
+    )
+
+    # Aggregate verdict
+    split_kappas = [per_cand_split[c]["k_cons"] for c in CANDIDATES
+                    if per_cand_split[c]["k_cons"] == per_cand_split[c]["k_cons"]]
+    n_valid_splits = min((per_cand_split[c]["n_eval"] for c in CANDIDATES), default=0)
+    min_parse_frac = min(
+        (per_cand_split[c]["parse_success"] / n_split if n_split else 0.0)
+        for c in CANDIDATES
+    )
+
+    # PM-amended verdict categories
+    if min_parse_frac < 0.80:
+        aggregate_verdict = "INCONCLUSIVE_BUT_OPERATIONAL_SIGNAL"
+        agg_reason = f"parse rate < 80% on at least one candidate"
+    elif n_valid_splits < 7:
+        aggregate_verdict = "INCONCLUSIVE_BUT_OPERATIONAL_SIGNAL"
+        agg_reason = f"valid evaluable splits < 7 after drops"
+    elif split_kappas and max(split_kappas) - min(split_kappas) >= 0.15 and all(k == k for k in split_kappas):
+        aggregate_verdict = "SPLIT_DISCRIMINATING"
+        agg_reason = f"κ_cons spread = {max(split_kappas) - min(split_kappas):.3f} ≥ 0.15 (n=7 pool-limited, observational not confirmatory)"
+    elif split_kappas and all(k >= 0.70 for k in split_kappas):
+        aggregate_verdict = "STILL_ALL_PASS"
+        agg_reason = f"all κ_cons ≥ 0.70, spread = {max(split_kappas) - min(split_kappas):.3f} < 0.15 (n=7 pool-limited, observational)"
+    else:
+        passers = sum(1 for k in split_kappas if k >= 0.70)
+        if passers == 0:
+            aggregate_verdict = "ALL_FAIL_ON_SPLITS"
+        elif passers < 4:
+            aggregate_verdict = "PARTIAL_FAIL"
+        else:
+            aggregate_verdict = "INCONCLUSIVE_BUT_OPERATIONAL_SIGNAL"
+        agg_reason = f"passers at κ_cons≥0.70: {passers}/4 (n=7 pool-limited)"
+
+    # Primary/backup recommendation
+    ranked_passing = [c for c in ranking if per_cand_split[c]["split_verdict"] == "PASS"]
+    if aggregate_verdict in ("SPLIT_DISCRIMINATING", "STILL_ALL_PASS") and len(ranked_passing) >= 2:
+        primary = ranked_passing[0]
+        backup = ranked_passing[1]
+    elif aggregate_verdict == "STILL_ALL_PASS":
+        primary = ranked_passing[0] if ranked_passing else "NONE"
+        backup = ranked_passing[1] if len(ranked_passing) > 1 else "NONE"
+    elif aggregate_verdict == "INCONCLUSIVE_BUT_OPERATIONAL_SIGNAL":
+        # Promote §1.3g operational heuristic: zhipu primary, deepseek backup
+        # UNLESS split data suggests otherwise (e.g. a candidate failed parse entirely)
+        zhipu_ok = per_cand_split["zhipu"]["parse_success"] >= n_split * 0.8
+        deepseek_ok = per_cand_split["deepseek"]["parse_success"] >= n_split * 0.8
+        primary = "ZHIPU" if zhipu_ok else "DEEPSEEK"
+        backup = "DEEPSEEK" if zhipu_ok and deepseek_ok else (
+            "MINIMAX" if per_cand_split["minimax"]["parse_success"] >= n_split * 0.8 else "NONE"
+        )
+    elif aggregate_verdict == "PARTIAL_FAIL":
+        primary = ranked_passing[0].upper() if ranked_passing else "NONE"
+        backup = ranked_passing[1].upper() if len(ranked_passing) > 1 else "NONE"
+    else:  # ALL_FAIL_ON_SPLITS
+        primary = "NONE"
+        backup = "NONE"
+
+    # MiniMax routing resolution
+    mm_resp = resp.get("minimax", [])
+    mm_routings = set(r.get("routing") for r in mm_resp)
+    if "direct_international" in mm_routings:
+        mm_res = "direct_international_successful"
+    elif "direct_legacy" in mm_routings:
+        mm_res = "direct_legacy_successful"
+    elif "openrouter" in mm_routings:
+        mm_res = "direct_failed_fell_back_openrouter"
+    else:
+        mm_res = "unknown"
+
+    # ── Write Markdown ─────────────────────────────────────────────────
+
+    lines = []
+    lines.append("# §1.3h Judge Swap Stratified Re-Probe — Dual-κ Split Analysis")
+    lines.append("")
+    lines.append("**Date:** 2026-04-24 (evening)  **Parent commit:** `8a2f0e6` (§1.3g anchor)")
+    lines.append("")
+    lines.append(f"**Source:** `benchmarks/results/locomo-mini-n20-retry-2026-04-24T00-02-12Z.jsonl` (100 rows, authoritative).")
+    lines.append("")
+    lines.append(f"**Split pool (Opus ≠ GPT):** **{n_split}** instances (use-all-available per PM amendment §1.3H-POOL-SHORTAGE OPTION 1).")
+    lines.append("")
+    lines.append(f"**Split cell distribution:** `agentic=2`, `full-context=2`, `oracle-context=3`, `no-context=0`, `retrieval=0`.")
+    lines.append("")
+    lines.append("**Observation:** 7/7 splits have `Opus=correct / GPT=incorrect`. Zero inverse splits. ")
+    lines.append("This is directional — measures candidate leniency (agrees with Opus) vs strictness (agrees with GPT).")
+    lines.append("")
+    lines.append("**PM-amended verdict caveat:** n=7 < 12 minimum for meaningful discrimination. All verdicts carry `n=7 pool-limited, observational not confirmatory` caveat. Primary value is operational signal (parse + latency + MiniMax routing).")
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+    lines.append("## Per-candidate split metrics")
+    lines.append("")
+    lines.append("| Cand | Parse | n_eval | p_opus | p_gpt | κ_vs_opus | κ_vs_gpt | κ_cons | p50 lat | p95 lat | Routing | Split verdict |")
+    lines.append("|------|-------|--------|--------|-------|-----------|----------|--------|---------|---------|---------|---------------|")
+    for c in ranking:
+        m = per_cand_split[c]
+        lines.append(
+            f"| {c} | {pct(m['parse_success'], n_split)} | {m['n_eval']} "
+            f"| {m['p_opus']*100:.1f}% ({m['p_opus_count']}/{m['n_eval']}) "
+            f"| {m['p_gpt']*100:.1f}% ({m['p_gpt_count']}/{m['n_eval']}) "
+            f"| {fmt_k(m['k_opus'])} | {fmt_k(m['k_gpt'])} | **{fmt_k(m['k_cons'])}** "
+            f"| {m['lat_p50_ms']/1000:.1f} s | {m['lat_p95_ms']/1000:.1f} s "
+            f"| `{m['routing']}` | **{m['split_verdict']}** |"
+        )
+    lines.append("")
+    lines.append("### Interpretation — p_opus / p_gpt balance")
+    lines.append("")
+    lines.append("Since all 7 splits are Opus=correct / GPT=incorrect:")
+    lines.append("")
+    lines.append("- `p_opus=1.0` means candidate always agreed with Opus (100% lenient)")
+    lines.append("- `p_gpt=1.0` means candidate always agreed with GPT (100% strict)")
+    lines.append("- `p_opus + p_gpt == 1.0` exactly (binary complementary in this oriented split set)")
+    lines.append("- **Well-balanced judge** here ≈ 3-4 correct, 3-4 incorrect (50/50-ish), showing independent calibration")
+    lines.append("- **Opus-leaning judge** = mostly 'correct' verdicts; **GPT-leaning judge** = mostly 'incorrect'")
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+    lines.append("## Aggregate κ on combined sample (27 = 20 unanimous + 7 splits)")
+    lines.append("")
+    lines.append("Unanimous portion reuses §1.3g verdicts (consensus-matched, κ=1.0 contribution). Split portion uses dual-reference with conservative min.")
+    lines.append("")
+    lines.append("| Cand | n_combined | κ_agg_vs_opus | κ_agg_vs_gpt | κ_agg_cons |")
+    lines.append("|------|------------|----------------|---------------|-------------|")
+    for c in ranking:
+        a = per_cand_agg[c]
+        lines.append(
+            f"| {c} | {a['n_combined']} | {fmt_k(a['k_agg_vs_opus'])} "
+            f"| {fmt_k(a['k_agg_vs_gpt'])} | **{fmt_k(a['k_agg_cons'])}** |"
+        )
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+    lines.append(f"## Aggregate verdict: **{aggregate_verdict}**")
+    lines.append("")
+    lines.append(f"**Reason:** {agg_reason}")
+    lines.append("")
+    lines.append(f"**Ranking by κ_conservative (descending):** {', '.join(ranking)}")
+    lines.append("")
+    lines.append(f"**Recommended primary:** {primary.upper()}")
+    lines.append(f"**Recommended backup:** {backup.upper()}")
+    lines.append("")
+    lines.append(f"**MiniMax routing resolution:** `{mm_res}`")
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+    lines.append("## Operational snapshot (split-case specific)")
+    lines.append("")
+    lines.append("- Parse rate on splits (challenging cases) vs unanimous (§1.3g for reference):")
+    for c in ranking:
+        m = per_cand_split[c]
+        lines.append(
+            f"  - `{c}`: splits {m['parse_success']}/{n_split} ({m['parse_success']*100/n_split:.0f}%), "
+            f"p50={m['lat_p50_ms']/1000:.1f}s, routing={m['routing']}"
+        )
+    lines.append("")
+    lines.append("- MiniMax direct-vs-fallback: see `minimax-split-responses.jsonl` per-row `routing` field.")
+    lines.append("")
+
+    OUT.write_text("\n".join(lines), encoding="utf-8")
+    print(f"\nWrote {OUT}")
+    print(f"Aggregate verdict: {aggregate_verdict}")
+    print(f"Primary: {primary}  Backup: {backup}")
+    print(f"MiniMax routing: {mm_res}")
+
+    # Machine-readable summary for halt ping
+    summary = {
+        "aggregate_verdict": aggregate_verdict,
+        "aggregate_reason": agg_reason,
+        "n_split": n_split,
+        "per_candidate": per_cand_split,
+        "per_candidate_aggregate": per_cand_agg,
+        "ranking_by_k_cons_desc": ranking,
+        "recommended_primary": primary.upper(),
+        "recommended_backup": backup.upper(),
+        "minimax_routing_resolution": mm_res,
+    }
+    (PROBE_DIR / "_summary-split.json").write_text(
+        json.dumps(summary, indent=2, default=str), encoding="utf-8"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+    sys.exit(main())
--- a/benchmarks/probes/judge-swap-validation/kimi-responses.jsonl
+++ b/benchmarks/probes/judge-swap-validation/kimi-responses.jsonl
@@ -0,0 +1,20 @@
+{"instance_id": "locomo_conv-50_q086", "cell": "no-context", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 2, "latency_ms": 166063, "prompt_tokens": 479, "completion_tokens": 1124, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F4\",\n  \"rationale\": \"The model answered 'Surfing', an activity that does not appear in the ground-truth context, whereas the context explicitly indicates Calvin plans to try skiing after the tour by referencing a snowy peak and an image of a person on skis.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F4", "parsed_rationale": "The model answered 'Surfing', an activity that does not appear in the ground-truth context, whereas the context explicitly indicates Calvin plans to try skiing after the tour by referencing a snowy peak and an image of a person on skis."}
+{"instance_id": "locomo_conv-26_q059", "cell": "no-context", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 50751, "prompt_tokens": 570, "completion_tokens": 1737, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F1\",\n  \"rationale\": \"The model abstained by stating the answer cannot be determined, whereas the ground truth indicates Caroline is somewhat, but not extremely, religious.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F1", "parsed_rationale": "The model abstained by stating the answer cannot be determined, whereas the ground truth indicates Caroline is somewhat, but not extremely, religious."}
+{"instance_id": "locomo_conv-44_q000", "cell": "no-context", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 25677, "prompt_tokens": 444, "completion_tokens": 698, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F4\",\n  \"rationale\": \"The model's answer of 1954 is a date that does not appear in the ground-truth context and is factually inconsistent with the correct year 2020 implied by Audrey having had the dogs for 3 years as of March 2023.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F4", "parsed_rationale": "The model's answer of 1954 is a date that does not appear in the ground-truth context and is factually inconsistent with the correct year 2020 implied by Audrey having had the dogs for 3 years as of March 2023."}
+{"instance_id": "locomo_conv-41_q080", "cell": "no-context", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 1, "latency_ms": 98768, "prompt_tokens": 495, "completion_tokens": 1031, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F4\",\n  \"rationale\": \"The model claimed Maria made a wind chime, an entity not present in the context, instead of the ground-truth painting of a castle on a hill.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F4", "parsed_rationale": "The model claimed Maria made a wind chime, an entity not present in the context, instead of the ground-truth painting of a castle on a hill."}
+{"instance_id": "locomo_conv-50_q086", "cell": "oracle-context", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 10807, "prompt_tokens": 480, "completion_tokens": 294, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer 'Skiing' exactly matches the ground-truth answer and is directly supported by the context mentioning a snowy peak and an image of a person on skis.\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer 'Skiing' exactly matches the ground-truth answer and is directly supported by the context mentioning a snowy peak and an image of a person on skis."}
+{"instance_id": "locomo_conv-26_q059", "cell": "oracle-context", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 33265, "prompt_tokens": 566, "completion_tokens": 1486, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly affirms Caroline is religious but omits the ground truth's nuance that she is only somewhat religious and not extremely religious.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly affirms Caroline is religious but omits the ground truth's nuance that she is only somewhat religious and not extremely religious."}
+{"instance_id": "locomo_conv-44_q000", "cell": "oracle-context", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 8245, "prompt_tokens": 444, "completion_tokens": 297, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model correctly identified 2020 as the year Audrey adopted the dogs, matching the ground truth derived from the context.\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model correctly identified 2020 as the year Audrey adopted the dogs, matching the ground truth derived from the context."}
+{"instance_id": "locomo_conv-41_q080", "cell": "oracle-context", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 31274, "prompt_tokens": 493, "completion_tokens": 1013, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly states that Maria made a painting, but fails to include the necessary detail that the painting depicts a castle on a hill.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly states that Maria made a painting, but fails to include the necessary detail that the painting depicts a castle on a hill."}
+{"instance_id": "locomo_conv-50_q086", "cell": "full-context", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 9236, "prompt_tokens": 479, "completion_tokens": 264, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model correctly identified 'skiing' as the sports activity Calvin plans to try after the tour, matching the ground-truth answer without adding incorrect claims.\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model correctly identified 'skiing' as the sports activity Calvin plans to try after the tour, matching the ground-truth answer without adding incorrect claims."}
+{"instance_id": "locomo_conv-26_q059", "cell": "full-context", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 25934, "prompt_tokens": 566, "completion_tokens": 833, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F1\",\n  \"rationale\": \"The model explicitly answered 'unknown' rather than providing the supported nuanced assessment that Caroline is somewhat, but not extremely, religious.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F1", "parsed_rationale": "The model explicitly answered 'unknown' rather than providing the supported nuanced assessment that Caroline is somewhat, but not extremely, religious."}
+{"instance_id": "locomo_conv-44_q000", "cell": "full-context", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 6796, "prompt_tokens": 444, "completion_tokens": 305, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model correctly inferred from the context that Audrey adopted the dogs in 2020, matching the ground truth exactly.\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model correctly inferred from the context that Audrey adopted the dogs in 2020, matching the ground truth exactly."}
+{"instance_id": "locomo_conv-41_q080", "cell": "full-context", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 28923, "prompt_tokens": 493, "completion_tokens": 787, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly states that Maria made a painting but omits the required detail that it depicts a castle on a hill.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly states that Maria made a painting but omits the required detail that it depicts a castle on a hill."}
+{"instance_id": "locomo_conv-50_q086", "cell": "retrieval", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 31953, "prompt_tokens": 479, "completion_tokens": 950, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F3\",\n  \"rationale\": \"The model incorrectly claims the activity is not mentioned when the context explicitly references skiing through the image description of a person on skis and the mention of a snowy peak in connection with Calvin's post-tour plans.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F3", "parsed_rationale": "The model incorrectly claims the activity is not mentioned when the context explicitly references skiing through the image description of a person on skis and the mention of a snowy peak in connection with Calvin's post-tour plans."}
+{"instance_id": "locomo_conv-26_q059", "cell": "retrieval", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 43785, "prompt_tokens": 566, "completion_tokens": 1286, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F3\",\n  \"rationale\": \"The model incorrectly answered 'No,' asserting Caroline is not religious, whereas the ground truth indicates she is somewhat religious as shown by her creating a stained glass window for a local church.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F3", "parsed_rationale": "The model incorrectly answered 'No,' asserting Caroline is not religious, whereas the ground truth indicates she is somewhat religious as shown by her creating a stained glass window for a local church."}
+{"instance_id": "locomo_conv-44_q000", "cell": "retrieval", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 1, "latency_ms": 110490, "prompt_tokens": 444, "completion_tokens": 1161, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F1\",\n  \"rationale\": \"The model abstains by claiming the adoption year is not mentioned, but the context states Audrey has had the dogs for 3 years as of March 2023, which implies 2020.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F1", "parsed_rationale": "The model abstains by claiming the adoption year is not mentioned, but the context states Audrey has had the dogs for 3 years as of March 2023, which implies 2020."}
+{"instance_id": "locomo_conv-41_q080", "cell": "retrieval", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 29546, "prompt_tokens": 493, "completion_tokens": 777, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly states that Maria made a painting but omits the essential detail that it was a painting of a castle on a hill.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly states that Maria made a painting but omits the essential detail that it was a painting of a castle on a hill."}
+{"instance_id": "locomo_conv-50_q086", "cell": "agentic", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 57420, "prompt_tokens": 489, "completion_tokens": 1509, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F1\",\n  \"rationale\": \"The model did not answer the question, instead returning a system message about reaching the maximum tool turns, effectively abstaining from providing the correct answer of skiing.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F1", "parsed_rationale": "The model did not answer the question, instead returning a system message about reaching the maximum tool turns, effectively abstaining from providing the correct answer of skiing."}
+{"instance_id": "locomo_conv-26_q059", "cell": "agentic", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 28521, "prompt_tokens": 599, "completion_tokens": 1276, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F4\",\n  \"rationale\": \"The model fabricated a claim about a Swedish necklace representing 'love, faith and strength' that does not appear in the provided ground-truth context.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F4", "parsed_rationale": "The model fabricated a claim about a Swedish necklace representing 'love, faith and strength' that does not appear in the provided ground-truth context."}
+{"instance_id": "locomo_conv-44_q000", "cell": "agentic", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 50434, "prompt_tokens": 454, "completion_tokens": 2083, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F1\",\n  \"rationale\": \"The model did not provide the correct year and instead indicated it reached the maximum tool turn limit, effectively abstaining from answering the question.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F1", "parsed_rationale": "The model did not provide the correct year and instead indicated it reached the maximum tool turn limit, effectively abstaining from answering the question."}
+{"instance_id": "locomo_conv-41_q080", "cell": "agentic", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 41180, "prompt_tokens": 493, "completion_tokens": 1121, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly states that Maria made a painting but omits the required detail that the painting depicted a castle on a hill.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly states that Maria made a painting but omits the required detail that the painting depicted a castle on a hill."}
--- a/benchmarks/probes/judge-swap-validation/kimi-split-responses.jsonl
+++ b/benchmarks/probes/judge-swap-validation/kimi-split-responses.jsonl
@@ -0,0 +1,7 @@
+{"instance_id": "locomo_conv-43_q155", "cell": "agentic", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 0, "error": "status=0 resp=TimeoutError: The read operation timed out", "retries": 3, "latency_ms": 183398, "prompt_tokens": null, "completion_tokens": null, "raw_text": "", "parsed_verdict": null, "parsed_failure_mode": null, "parsed_rationale": null, "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-50_q015", "cell": "agentic", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 24234, "prompt_tokens": 550, "completion_tokens": 614, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer uses 'relax' directly and 'recharge' as an acceptable equivalent formulation for the calming and peaceful effect described in the ground-truth context, with no incorrect claims added.\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer uses 'relax' directly and 'recharge' as an acceptable equivalent formulation for the calming and peaceful effect described in the ground-truth context, with no incorrect claims added.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-43_q155", "cell": "full-context", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 0, "error": "status=0 resp=TimeoutError: The read operation timed out", "retries": 3, "latency_ms": 183176, "prompt_tokens": null, "completion_tokens": null, "raw_text": "", "parsed_verdict": null, "parsed_failure_mode": null, "parsed_rationale": null, "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-50_q015", "cell": "full-context", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 51046, "prompt_tokens": 548, "completion_tokens": 1512, "raw_text": "```json\n{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer 'to relax' is an acceptable synonymous formulation of the ground truth 'because it relaxes and calms him', capturing the same core reason from the context without omitting a distinct required fact or adding incorrect claims.\"\n}\n```", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer 'to relax' is an acceptable synonymous formulation of the ground truth 'because it relaxes and calms him', capturing the same core reason from the context without omitting a distinct required fact or adding incorrect claims.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-43_q155", "cell": "oracle-context", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 31347, "prompt_tokens": 456, "completion_tokens": 1188, "raw_text": "```json\n{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer 'Great' is directly supported by the context where John explicitly states 'It was great!' regarding jogging without pain, and it is an acceptable equivalent formulation to the ground-truth 'huge success' that captures the same positive sentiment.\"\n}\n```", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer 'Great' is directly supported by the context where John explicitly states 'It was great!' regarding jogging without pain, and it is an acceptable equivalent formulation to the ground-truth 'huge success' that captures the same positive sentiment.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-47_q092", "cell": "oracle-context", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 31985, "prompt_tokens": 460, "completion_tokens": 1505, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly states that John created an application but omits the ground-truth detail that it was on smartphones, missing a required fact without adding incorrect claims.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly states that John created an application but omits the ground-truth detail that it was on smartphones, missing a required fact without adding incorrect claims.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-50_q015", "cell": "oracle-context", "provider": "kimi", "model_id": "kimi-k2.6", "routing": "direct", "http_status": 200, "error": null, "retries": 1, "latency_ms": 90555, "prompt_tokens": 548, "completion_tokens": 941, "raw_text": "```json\n{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer 'To relax' is an acceptable synonym and equivalent formulation of the ground truth 'because it relaxes and calms him', accurately capturing the essential reason without omitting any distinct required fact.\"\n}\n```", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer 'To relax' is an acceptable synonym and equivalent formulation of the ground truth 'because it relaxes and calms him', accurately capturing the essential reason without omitting any distinct required fact.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
--- a/benchmarks/probes/judge-swap-validation/minimax-responses.jsonl
+++ b/benchmarks/probes/judge-swap-validation/minimax-responses.jsonl
@@ -0,0 +1,20 @@
+{"instance_id": "locomo_conv-50_q086", "cell": "no-context", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 21805, "prompt_tokens": 505, "completion_tokens": 610, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F3\",\n  \"rationale\": \"The model answered 'Surfing' when the ground truth clearly shows Calvin is planning to go 'Skiing' after his tour with Frank Ocean, based on the image of a person on skis on a snowy mountain and his statement about seeing snowy peaks.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F3", "parsed_rationale": "The model answered 'Surfing' when the ground truth clearly shows Calvin is planning to go 'Skiing' after his tour with Frank Ocean, based on the image of a person on skis on a snowy mountain and his statement about seeing snowy peaks."}
+{"instance_id": "locomo_conv-26_q059", "cell": "no-context", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 9874, "prompt_tokens": 593, "completion_tokens": 533, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model states that the answer cannot be determined without context, but the provided ground-truth context does allow for an inference about Caroline being somewhat religious (as evidenced by her making art for a church) but not extremely religious (as evidenced by her negative encounter with religious conservatives), which aligns with the ground-truth answer.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model states that the answer cannot be determined without context, but the provided ground-truth context does allow for an inference about Caroline being somewhat religious (as evidenced by her making art for a church) but not extremely religious (as evidenced by her negative encounter with religious conservatives), which aligns with the ground-truth answer."}
+{"instance_id": "locomo_conv-44_q000", "cell": "no-context", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 19947, "prompt_tokens": 450, "completion_tokens": 524, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F4\",\n  \"rationale\": \"The model's answer 1954 does not appear in or derive from the ground-truth context, which only mentions she adopted dogs 3 years before March 2023, indicating 2020; 1954 is a fabrication unrelated to the provided information.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F4", "parsed_rationale": "The model's answer 1954 does not appear in or derive from the ground-truth context, which only mentions she adopted dogs 3 years before March 2023, indicating 2020; 1954 is a fabrication unrelated to the provided information."}
+{"instance_id": "locomo_conv-41_q080", "cell": "no-context", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 10670, "prompt_tokens": 521, "completion_tokens": 376, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F4\",\n  \"rationale\": \"The model's answer 'A wind chime' does not appear anywhere in the ground-truth context, which explicitly states Maria made a painting of a castle on a hill, indicating the model fabricated the answer.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F4", "parsed_rationale": "The model's answer 'A wind chime' does not appear anywhere in the ground-truth context, which explicitly states Maria made a painting of a castle on a hill, indicating the model fabricated the answer."}
+{"instance_id": "locomo_conv-50_q086", "cell": "oracle-context", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 5743, "prompt_tokens": 506, "completion_tokens": 156, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer of 'Skiing' matches the ground truth exactly.\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer of 'Skiing' matches the ground truth exactly."}
+{"instance_id": "locomo_conv-26_q059", "cell": "oracle-context", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 35508, "prompt_tokens": 569, "completion_tokens": 963, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly confirms Caroline would be considered religious, but omits the key nuance from ground truth that she is only 'somewhat, but not extremely' religious, providing a partial answer.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly confirms Caroline would be considered religious, but omits the key nuance from ground truth that she is only 'somewhat, but not extremely' religious, providing a partial answer."}
+{"instance_id": "locomo_conv-44_q000", "cell": "oracle-context", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 11773, "prompt_tokens": 470, "completion_tokens": 351, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model correctly identified 2020 as the year Audrey adopted the first three of her dogs.\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model correctly identified 2020 as the year Audrey adopted the first three of her dogs."}
+{"instance_id": "locomo_conv-41_q080", "cell": "oracle-context", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 18664, "prompt_tokens": 517, "completion_tokens": 417, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly identifies that Maria made a painting, but it omits the specific detail that the painting was of a castle on a hill, making the answer incomplete.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly identifies that Maria made a painting, but it omits the specific detail that the painting was of a castle on a hill, making the answer incomplete."}
+{"instance_id": "locomo_conv-50_q086", "cell": "full-context", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 3043, "prompt_tokens": 502, "completion_tokens": 168, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model correctly identifies the sports activity (skiing) that Calvin plans to try after his tour with Frank Ocean.\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model correctly identifies the sports activity (skiing) that Calvin plans to try after his tour with Frank Ocean."}
+{"instance_id": "locomo_conv-26_q059", "cell": "full-context", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 9116, "prompt_tokens": 569, "completion_tokens": 310, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F1\",\n  \"rationale\": \"The model answered 'unknown' which explicitly indicates it does not know or refuses to answer, rather than providing the correct assessment that Caroline is somewhat religious but not extremely so.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F1", "parsed_rationale": "The model answered 'unknown' which explicitly indicates it does not know or refuses to answer, rather than providing the correct assessment that Caroline is somewhat religious but not extremely so."}
+{"instance_id": "locomo_conv-44_q000", "cell": "full-context", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 6223, "prompt_tokens": 467, "completion_tokens": 298, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer of 2020 matches the ground truth answer, which can be correctly derived from the context stating Audrey had the dogs for 3 years as of March 2023.\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer of 2020 matches the ground truth answer, which can be correctly derived from the context stating Audrey had the dogs for 3 years as of March 2023."}
+{"instance_id": "locomo_conv-41_q080", "cell": "full-context", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 5011, "prompt_tokens": 517, "completion_tokens": 332, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly identifies that Maria made a painting but omits the specific detail that the painting depicts a castle on a hill, which is required by the ground truth.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly identifies that Maria made a painting but omits the specific detail that the painting depicts a castle on a hill, which is required by the ground truth."}
+{"instance_id": "locomo_conv-50_q086", "cell": "retrieval", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 7531, "prompt_tokens": 502, "completion_tokens": 475, "raw_text": "{\"verdict\": \"incorrect\", \"failure_mode\": \"F3\", \"rationale\": \"The model incorrectly stated that the sports activity was 'not mentioned' when the ground-truth context clearly indicates Calvin is planning to go skiing after the tour, as evidenced by the image of a person on skis and the reference to snowy peaks.\"}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F3", "parsed_rationale": "The model incorrectly stated that the sports activity was 'not mentioned' when the ground-truth context clearly indicates Calvin is planning to go skiing after the tour, as evidenced by the image of a person on skis and the reference to snowy peaks."}
+{"instance_id": "locomo_conv-26_q059", "cell": "retrieval", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 13040, "prompt_tokens": 586, "completion_tokens": 625, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F3\",\n  \"rationale\": \"The model states Caroline is not religious ('No'), but the ground truth indicates she is somewhat religious based on her making something for a local church and engaging with religious settings.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F3", "parsed_rationale": "The model states Caroline is not religious ('No'), but the ground truth indicates she is somewhat religious based on her making something for a local church and engaging with religious settings."}
+{"instance_id": "locomo_conv-44_q000", "cell": "retrieval", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 20906, "prompt_tokens": 470, "completion_tokens": 658, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model's answer 'Not mentioned' fails to provide the correct year (2020), which is derivable from the context stating she had the dogs for 3 years as of March 2023, but it does not contain any incorrect claims about the year.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model's answer 'Not mentioned' fails to provide the correct year (2020), which is derivable from the context stating she had the dogs for 3 years as of March 2023, but it does not contain any incorrect claims about the year."}
+{"instance_id": "locomo_conv-41_q080", "cell": "retrieval", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 9432, "prompt_tokens": 500, "completion_tokens": 331, "raw_text": "{\"verdict\": \"incorrect\", \"failure_mode\": \"F2\", \"rationale\": \"The model correctly identifies that Maria made a painting, but it omits the specific subject matter (a castle on a hill) required by the ground truth.\"}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly identifies that Maria made a painting, but it omits the specific subject matter (a castle on a hill) required by the ground truth."}
+{"instance_id": "locomo_conv-50_q086", "cell": "agentic", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 21740, "prompt_tokens": 512, "completion_tokens": 559, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F1\",\n  \"rationale\": \"The model hit the maximum tool turns limit without providing an answer, effectively abstaining from answering the question.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F1", "parsed_rationale": "The model hit the maximum tool turns limit without providing an answer, effectively abstaining from answering the question."}
+{"instance_id": "locomo_conv-26_q059", "cell": "agentic", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 20465, "prompt_tokens": 622, "completion_tokens": 698, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F4\",\n  \"rationale\": \"The model fabricates a detail about a 'necklace from Sweden representing love, faith and strength' that does not appear in the ground-truth context, and answers simply 'Yes' rather than the nuanced 'Somewhat, but not extremely religious' provided in the ground truth.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F4", "parsed_rationale": "The model fabricates a detail about a 'necklace from Sweden representing love, faith and strength' that does not appear in the ground-truth context, and answers simply 'Yes' rather than the nuanced 'Somewhat, but not extremely religious' provided in the ground truth."}
+{"instance_id": "locomo_conv-44_q000", "cell": "agentic", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 27721, "prompt_tokens": 477, "completion_tokens": 832, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F1\",\n  \"rationale\": \"The model did not provide an answer to the question, instead indicating that it reached its maximum tool usage limit, which effectively functions as an abstention from answering.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F1", "parsed_rationale": "The model did not provide an answer to the question, instead indicating that it reached its maximum tool usage limit, which effectively functions as an abstention from answering."}
+{"instance_id": "locomo_conv-41_q080", "cell": "agentic", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 6577, "prompt_tokens": 517, "completion_tokens": 447, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly identifies that Maria made a painting but omits the specific detail that it was a painting of a castle on a hill.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly identifies that Maria made a painting but omits the specific detail that it was a painting of a castle on a hill."}
--- a/benchmarks/probes/judge-swap-validation/minimax-split-responses.jsonl
+++ b/benchmarks/probes/judge-swap-validation/minimax-split-responses.jsonl
@@ -0,0 +1,7 @@
+{"instance_id": "locomo_conv-43_q155", "cell": "agentic", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 12267, "prompt_tokens": 483, "completion_tokens": 349, "raw_text": "```json\n{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer 'great' directly matches John's described emotional response from the ground truth context, where he says 'It was great!'\"\n}\n```", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer 'great' directly matches John's described emotional response from the ground truth context, where he says 'It was great!'", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-50_q015", "cell": "agentic", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 25989, "prompt_tokens": 576, "completion_tokens": 691, "raw_text": "{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly mentions 'relax' (matching 'relaxes') but omits 'calms' from the ground truth and introduces 'recharge' which is not present in the ground-truth answer.\"\n}", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly mentions 'relax' (matching 'relaxes') but omits 'calms' from the ground truth and introduces 'recharge' which is not present in the ground-truth answer.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-43_q155", "cell": "full-context", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 8708, "prompt_tokens": 483, "completion_tokens": 302, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model answered 'great' which directly matches John's stated feeling in the ground truth ('It was great!'), and 'great' is synonymous with 'a huge success' when describing his emotional reaction to jogging without pain.\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model answered 'great' which directly matches John's stated feeling in the ground truth ('It was great!'), and 'great' is synonymous with 'a huge success' when describing his emotional reaction to jogging without pain.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-50_q015", "cell": "full-context", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 10769, "prompt_tokens": 571, "completion_tokens": 290, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer 'to relax' captures the essential fact from the ground truth (relaxes) and is supported by the context where Dave explicitly states he visits parks 'to relax' and describes them as 'calming' and 'peaceful.'\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer 'to relax' captures the essential fact from the ground truth (relaxes) and is supported by the context where Dave explicitly states he visits parks 'to relax' and describes them as 'calming' and 'peaceful.'", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-43_q155", "cell": "oracle-context", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 18744, "prompt_tokens": 483, "completion_tokens": 547, "raw_text": "{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer 'Great' is a minor phrasing variation/synonym for the ground truth context which includes John saying 'It was great!' and 'a huge success,' conveying the same positive sentiment about how he felt.\"\n}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer 'Great' is a minor phrasing variation/synonym for the ground truth context which includes John saying 'It was great!' and 'a huge success,' conveying the same positive sentiment about how he felt.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-47_q092", "cell": "oracle-context", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 21222, "prompt_tokens": 467, "completion_tokens": 559, "raw_text": "{\"verdict\": \"correct\", \"failure_mode\": null, \"rationale\": \"The model's answer 'An application' correctly captures the essential fact from the ground truth that John created an application, and while it omits the detail that it was 'on smartphones,' missing extra detail is acceptable as long as no incorrect information is added.\"}", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer 'An application' correctly captures the essential fact from the ground truth that John created an application, and while it omits the detail that it was 'on smartphones,' missing extra detail is acceptable as long as no incorrect information is added.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-50_q015", "cell": "oracle-context", "provider": "minimax", "model_id": "minimax/minimax-m2.7", "routing": "openrouter", "http_status": 200, "error": null, "retries": 0, "latency_ms": 16627, "prompt_tokens": 571, "completion_tokens": 534, "raw_text": "```json\n{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer 'To relax' is a direct synonym and equivalent formulation of the ground truth 'relaxes and calms him,' capturing the essential reason Dave visits parks without any incorrect claims.\"\n}\n```", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer 'To relax' is a direct synonym and equivalent formulation of the ground truth 'relaxes and calms him,' capturing the essential reason Dave visits parks without any incorrect claims.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
--- a/benchmarks/probes/judge-swap-validation/probe-script-split.py
+++ b/benchmarks/probes/judge-swap-validation/probe-script-split.py
@@ -0,0 +1,485 @@
+"""
+§1.3h Judge Swap Stratified Re-Probe — split-case execution
+============================================================
+
+Executes 28 API calls (7 splits × 4 candidates) on the subset where
+Opus != GPT in the authoritative 100-row κ calibration set
+(`locomo-mini-n20-retry-2026-04-24T00-02-12Z.jsonl`).
+
+PM-amended (§1.3H-POOL-SHORTAGE OPTION 1):
+  - Split pool minimum reduced 12 → 7 (use-all-available)
+  - Budget $1.50 cap / $3 halt (reduced from $3/$5)
+  - Wall-clock cap 30 min (reduced from 60)
+  - Primary operational value focus: MiniMax direct routing E2E
+    verification with newly-added MINIMAX_GROUP_ID
+
+Routing:
+  - Kimi:     direct api.moonshot.ai, kimi-k2.6, max_tokens=4096
+  - DeepSeek: direct api.deepseek.com, deepseek-v4-pro, max_tokens=1024
+  - Zhipu:    direct api.z.ai, glm-5.1, max_tokens=1024
+  - MiniMax:  DIRECT FIRST — tries api.minimaxi.com + api.minimax.chat
+              with MINIMAX_GROUP_ID. Fallback: OpenRouter
+              minimax/minimax-m2.7 if both direct variants fail.
+
+Scope discipline: manifest v5 anchor fc16925 intact; §11 frozen path
+untouched; no new LiteLLM aliases; no new Opus/GPT calls; additive
+files only under `benchmarks/probes/judge-swap-validation/`.
+
+Usage:
+  python probe-script-split.py
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+import time
+import traceback
+import urllib.error
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timezone
+from pathlib import Path
+
+try:
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+    sys.stderr.reconfigure(encoding="utf-8", errors="replace")
+except Exception:
+    pass
+
+# ── Paths + constants ───────────────────────────────────────────────────
+
+PROBE_DIR = Path("D:/Projects/waggle-os/benchmarks/probes/judge-swap-validation")
+SPLIT_SAMPLE_PATH = PROBE_DIR / "split-cases-sample.jsonl"
+
+# ── Judge prompt template (verbatim from §1.3g probe-script.py) ────────
+
+JUDGE_PROMPT_TEMPLATE = "\n".join([
+    "You are evaluating whether an LLM's answer is correct against ground truth.",
+    "",
+    "## Question",
+    "{question}",
+    "",
+    "## Ground-truth answer",
+    "{ground_truth}",
+    "",
+    "## Ground-truth supporting context (excerpt shown to the model)",
+    "{context}",
+    "",
+    "## Model's answer",
+    "{model_answer}",
+    "",
+    "## Your task",
+    "",
+    "Step 1: Determine if the model's answer is correct.",
+    "- \"correct\" means the model's answer contains all required facts from ground truth, with no additional incorrect claims.",
+    "- Minor phrasing differences, synonyms, or alternative but equivalent formulations are acceptable.",
+    "- Extra detail is acceptable ONLY if it is factually correct.",
+    "",
+    "Step 2: If incorrect, assign exactly one failure mode using this decision tree:",
+    "",
+    "1. Does the model explicitly refuse or say it does not know? -> F1 (ABSTAIN)",
+    "2. Does the model answer a DIFFERENT question than was asked (coherent but off-topic)? -> F5 (OFF-TOPIC)",
+    "3. Does the model rely on entities, names, dates, or claims that do NOT appear in the ground-truth context (fabrication)? -> F4 (HALLUCINATED)",
+    "4. Does the model correctly state SOME required facts but miss others, without stating any incorrect facts? -> F2 (PARTIAL)",
+    "5. Otherwise (model states facts derived from the context but gets them wrong): -> F3 (INCORRECT)",
+    "",
+    "Step 3: Return JSON only, no prose, in this exact schema:",
+    "",
+    "{{",
+    "  \"verdict\": \"correct\" | \"incorrect\",",
+    "  \"failure_mode\": null | \"F1\" | \"F2\" | \"F3\" | \"F4\" | \"F5\",",
+    "  \"rationale\": \"one sentence explaining the verdict\"",
+    "}}",
+    "",
+    "If verdict is \"correct\", failure_mode MUST be null.",
+    "If verdict is \"incorrect\", failure_mode MUST be one of F1-F5.",
+])
+
+
+def ts() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def logmsg(msg: str) -> None:
+    print(f"{ts()} {msg}", flush=True)
+
+
+# ── .env loader ─────────────────────────────────────────────────────────
+
+def load_env() -> dict[str, str]:
+    env_path = Path("D:/Projects/waggle-os/.env")
+    out: dict[str, str] = {}
+    if not env_path.exists():
+        return out
+    for line in env_path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        if "=" not in line:
+            continue
+        k, _, v = line.partition("=")
+        k, v = k.strip(), v.strip().strip('"').strip("'")
+        out[k] = v
+    return out
+
+
+# ── Verbatim judge-runner JSON extractor ────────────────────────────────
+
+def extract_json_body(raw: str) -> dict | None:
+    if not raw:
+        return None
+    trimmed = raw.strip()
+    if trimmed.startswith("```"):
+        m = re.match(r"^```(?:json)?\s*\n?(.*?)```\s*$", trimmed, re.DOTALL)
+        if m:
+            trimmed = m.group(1).strip()
+    try:
+        return json.loads(trimmed)
+    except Exception:
+        pass
+    first = trimmed.find("{")
+    last = trimmed.rfind("}")
+    if first != -1 and last != -1 and last > first:
+        try:
+            return json.loads(trimmed[first:last+1])
+        except Exception:
+            return None
+    return None
+
+
+def parse_verdict(raw_text: str) -> tuple[str | None, str | None, str | None]:
+    body = extract_json_body(raw_text or "")
+    if not isinstance(body, dict):
+        return (None, None, None)
+    v = body.get("verdict")
+    fm = body.get("failure_mode")
+    rat = body.get("rationale")
+    if v not in ("correct", "incorrect"):
+        return (None, None, None)
+    if fm is not None and fm not in ("F1", "F2", "F3", "F4", "F5"):
+        fm = None
+    return (v, fm, rat if isinstance(rat, str) else None)
+
+
+# ── HTTP helper ─────────────────────────────────────────────────────────
+
+def http_post_json(url: str, headers: dict, body: dict, timeout_s: int = 60) -> tuple[int, dict | str]:
+    req = urllib.request.Request(
+        url,
+        data=json.dumps(body).encode("utf-8"),
+        method="POST",
+        headers={"Content-Type": "application/json", **headers},
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=timeout_s) as resp:
+            raw = resp.read().decode("utf-8", errors="replace")
+            try:
+                return resp.status, json.loads(raw)
+            except Exception:
+                return resp.status, raw
+    except urllib.error.HTTPError as e:
+        try:
+            body_err = e.read().decode("utf-8", errors="replace")
+        except Exception:
+            body_err = ""
+        return e.code, body_err[:2000]
+    except Exception as e:
+        return 0, f"{type(e).__name__}: {e}"
+
+
+# ── Provider callers ────────────────────────────────────────────────────
+
+def _retry_call(
+    url: str,
+    headers: dict,
+    body: dict,
+    model_id: str,
+    routing: str,
+    max_attempts: int = 3,
+) -> dict:
+    started = time.time()
+    last_err = None
+    retries = 0
+    for attempt in range(max_attempts):
+        status, resp = http_post_json(url, headers, body, timeout_s=60)
+        if status == 200 and isinstance(resp, dict):
+            choices = resp.get("choices") or []
+            if choices:
+                msg = choices[0].get("message") or {}
+                content = msg.get("content") or msg.get("reasoning_content") or ""
+                usage = resp.get("usage", {})
+                return {
+                    "raw_text": content,
+                    "status": 200,
+                    "error": None,
+                    "retries": retries,
+                    "model_id": model_id,
+                    "routing": routing,
+                    "latency_ms": int((time.time() - started) * 1000),
+                    "prompt_tokens": usage.get("prompt_tokens"),
+                    "completion_tokens": usage.get("completion_tokens"),
+                }
+        last_err = f"status={status} resp={str(resp)[:400]}"
+        retries += 1
+        if attempt < max_attempts - 1:
+            time.sleep(2 ** attempt)
+    return {
+        "raw_text": "",
+        "status": 0,
+        "error": last_err,
+        "retries": retries,
+        "model_id": model_id,
+        "routing": routing,
+        "latency_ms": int((time.time() - started) * 1000),
+        "prompt_tokens": None,
+        "completion_tokens": None,
+    }
+
+
+def call_kimi(prompt: str, env: dict) -> dict:
+    url = "https://api.moonshot.ai/v1/chat/completions"
+    headers = {"Authorization": f"Bearer {env['MOONSHOT_API_KEY']}"}
+    body = {
+        "model": "kimi-k2.6",
+        "messages": [{"role": "user", "content": prompt}],
+        "max_tokens": 4096,
+    }
+    return _retry_call(url, headers, body, "kimi-k2.6", routing="direct")
+
+
+def call_deepseek(prompt: str, env: dict) -> dict:
+    url = "https://api.deepseek.com/v1/chat/completions"
+    headers = {"Authorization": f"Bearer {env['DEEPSEEK_API_KEY']}"}
+    body = {
+        "model": "deepseek-v4-pro",
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": 0.0,
+        "max_tokens": 1024,
+    }
+    return _retry_call(url, headers, body, "deepseek-v4-pro", routing="direct")
+
+
+def call_zhipu(prompt: str, env: dict) -> dict:
+    url = "https://api.z.ai/api/paas/v4/chat/completions"
+    headers = {"Authorization": f"Bearer {env['ZHIPU_API_KEY']}"}
+    body = {
+        "model": "glm-5.1",
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": 0.0,
+        "max_tokens": 1024,
+    }
+    return _retry_call(url, headers, body, "glm-5.1", routing="direct")
+
+
+# ── MiniMax direct routing with endpoint fallback ──────────────────────
+#
+# MiniMax direct chat completion has two known v2 endpoints:
+#   1. https://api.minimaxi.com/v1/text/chatcompletion_v2  (international)
+#   2. https://api.minimax.chat/v1/text/chatcompletion_v2   (legacy/CN)
+# v2 is OpenAI-compatible format. GroupId is passed as query param.
+# If both v2 endpoints fail, fall back to OpenRouter (§1.3g routing).
+
+_MINIMAX_ROUTE_CACHE: dict[str, str] = {"chosen": None, "first_tried": []}
+
+
+def call_minimax_direct_with_fallback(prompt: str, env: dict) -> dict:
+    """Tries MiniMax direct (international then legacy), falls back to
+    OpenRouter on failure. Caches first-successful route to avoid re-
+    discovering per call."""
+    group_id = env.get("MINIMAX_GROUP_ID", "").strip()
+    mm_key = env.get("MINIMAX_API_KEY", "").strip()
+    or_key = env.get("OPENROUTER_API_KEY", "").strip()
+
+    # If a working route is already cached, use it
+    if _MINIMAX_ROUTE_CACHE["chosen"] == "direct_international":
+        return _call_minimax_direct(prompt, mm_key, group_id, "international")
+    if _MINIMAX_ROUTE_CACHE["chosen"] == "direct_legacy":
+        return _call_minimax_direct(prompt, mm_key, group_id, "legacy")
+    if _MINIMAX_ROUTE_CACHE["chosen"] == "openrouter":
+        return _call_minimax_openrouter(prompt, or_key)
+
+    # First call: try international, then legacy, then OR
+    r = _call_minimax_direct(prompt, mm_key, group_id, "international")
+    _MINIMAX_ROUTE_CACHE["first_tried"].append("international")
+    if r["status"] == 200 and r["raw_text"]:
+        _MINIMAX_ROUTE_CACHE["chosen"] = "direct_international"
+        logmsg(f"[minimax] DIRECT international SUCCESSFUL — caching route")
+        return r
+    logmsg(f"[minimax] direct international FAILED: status={r['status']} err={(r.get('error') or '')[:200]}")
+
+    r = _call_minimax_direct(prompt, mm_key, group_id, "legacy")
+    _MINIMAX_ROUTE_CACHE["first_tried"].append("legacy")
+    if r["status"] == 200 and r["raw_text"]:
+        _MINIMAX_ROUTE_CACHE["chosen"] = "direct_legacy"
+        logmsg(f"[minimax] DIRECT legacy SUCCESSFUL — caching route")
+        return r
+    logmsg(f"[minimax] direct legacy FAILED: status={r['status']} err={(r.get('error') or '')[:200]}")
+
+    _MINIMAX_ROUTE_CACHE["chosen"] = "openrouter"
+    _MINIMAX_ROUTE_CACHE["first_tried"].append("openrouter")
+    logmsg(f"[minimax] falling back to OpenRouter for all subsequent calls")
+    return _call_minimax_openrouter(prompt, or_key)
+
+
+def _call_minimax_direct(prompt: str, api_key: str, group_id: str, variant: str) -> dict:
+    if variant == "international":
+        base = "https://api.minimaxi.com/v1/text/chatcompletion_v2"
+    elif variant == "legacy":
+        base = "https://api.minimax.chat/v1/text/chatcompletion_v2"
+    else:
+        return {"raw_text": "", "status": 0, "error": f"unknown variant {variant}",
+                "retries": 0, "model_id": "unknown", "routing": f"direct_{variant}",
+                "latency_ms": 0, "prompt_tokens": None, "completion_tokens": None}
+
+    # GroupId as query param
+    url = f"{base}?GroupId={group_id}" if group_id else base
+    headers = {"Authorization": f"Bearer {api_key}"}
+    body = {
+        "model": "MiniMax-M2",
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": 0.0,
+        "max_tokens": 4096,
+    }
+    return _retry_call(url, headers, body, "MiniMax-M2", routing=f"direct_{variant}", max_attempts=2)
+
+
+def _call_minimax_openrouter(prompt: str, or_key: str) -> dict:
+    url = "https://openrouter.ai/api/v1/chat/completions"
+    headers = {"Authorization": f"Bearer {or_key}"}
+    body = {
+        "model": "minimax/minimax-m2.7",
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": 0.0,
+        "max_tokens": 4096,
+    }
+    return _retry_call(url, headers, body, "minimax/minimax-m2.7", routing="openrouter")
+
+
+# ── Sample loader ──────────────────────────────────────────────────────
+
+def load_split_sample() -> list[dict]:
+    rows = []
+    with SPLIT_SAMPLE_PATH.open("r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                rows.append(json.loads(line))
+    return rows
+
+
+# ── Runner ─────────────────────────────────────────────────────────────
+
+PROVIDER_FNS = [
+    ("kimi", call_kimi),
+    ("minimax", call_minimax_direct_with_fallback),
+    ("deepseek", call_deepseek),
+    ("zhipu", call_zhipu),
+]
+
+
+def run_provider(name: str, call_fn, sample: list[dict], env: dict) -> list[dict]:
+    logmsg(f"[{name}] start — {len(sample)} split instances")
+    out: list[dict] = []
+    for i, s in enumerate(sample):
+        prompt = JUDGE_PROMPT_TEMPLATE.format(
+            question=s.get("question") or "",
+            ground_truth=s.get("ground_truth") or "",
+            context=s.get("context") or "",
+            model_answer=s.get("model_answer") or "",
+        )
+        resp = call_fn(prompt, env)
+        verdict, fm, rat = parse_verdict(resp["raw_text"])
+        row = {
+            "instance_id": s["instance_id"],
+            "cell": s["cell"],
+            "provider": name,
+            "model_id": resp["model_id"],
+            "routing": resp["routing"],
+            "http_status": resp["status"],
+            "error": resp.get("error"),
+            "retries": resp["retries"],
+            "latency_ms": resp["latency_ms"],
+            "prompt_tokens": resp.get("prompt_tokens"),
+            "completion_tokens": resp.get("completion_tokens"),
+            "raw_text": resp["raw_text"],
+            "parsed_verdict": verdict,
+            "parsed_failure_mode": fm,
+            "parsed_rationale": rat,
+            "opus_verdict_ref": s.get("opus_verdict"),
+            "gpt_verdict_ref": s.get("gpt_verdict"),
+        }
+        out.append(row)
+        logmsg(
+            f"[{name}] {i+1:>2}/{len(sample)} {s['instance_id']:30} cell={s['cell']:14} "
+            f"status={resp['status']} verdict={verdict} retries={resp['retries']} "
+            f"routing={resp['routing']}"
+        )
+    logmsg(f"[{name}] done")
+    return out
+
+
+def write_jsonl(path: Path, rows: list[dict]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as f:
+        for r in rows:
+            f.write(json.dumps(r, ensure_ascii=False) + "\n")
+
+
+def main() -> int:
+    logmsg("[probe] §1.3h judge swap stratified re-probe START")
+    env = load_env()
+    required = [
+        "MOONSHOT_API_KEY",
+        "MINIMAX_API_KEY",
+        "MINIMAX_GROUP_ID",
+        "DEEPSEEK_API_KEY",
+        "ZHIPU_API_KEY",
+        "OPENROUTER_API_KEY",
+    ]
+    missing = [k for k in required if not env.get(k)]
+    if missing:
+        logmsg(f"[probe] FATAL missing env keys: {missing}")
+        return 2
+
+    sample = load_split_sample()
+    logmsg(f"[probe] loaded {len(sample)} split instances from {SPLIT_SAMPLE_PATH.name}")
+    if len(sample) < 1:
+        logmsg("[probe] FATAL empty sample")
+        return 3
+
+    logmsg("[probe] executing 4 providers in parallel (4 threads)")
+    provider_rows: dict[str, list[dict]] = {}
+    with ThreadPoolExecutor(max_workers=4) as pool:
+        futures = {
+            pool.submit(run_provider, name, fn, sample, env): name
+            for (name, fn) in PROVIDER_FNS
+        }
+        for fut in as_completed(futures):
+            name = futures[fut]
+            try:
+                rows = fut.result()
+                provider_rows[name] = rows
+            except Exception as e:
+                logmsg(f"[probe] provider {name} FAILED: {type(e).__name__}: {e}")
+                provider_rows[name] = []
+
+    logmsg("[probe] writing per-provider JSONL artefacts")
+    for name, rows in provider_rows.items():
+        write_jsonl(PROBE_DIR / f"{name}-split-responses.jsonl", rows)
+        parsed = sum(1 for r in rows if r.get("parsed_verdict") is not None)
+        logmsg(f"[probe] {name}: wrote {len(rows)} rows, parsed_ok={parsed}")
+
+    logmsg(f"[probe] minimax_routing_resolution: {_MINIMAX_ROUTE_CACHE}")
+    logmsg("[probe] END")
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except Exception as e:
+        logmsg(f"[FATAL] {type(e).__name__}: {e}")
+        logmsg(traceback.format_exc()[:2000])
+        sys.exit(99)
--- a/benchmarks/probes/judge-swap-validation/probe-script.py
+++ b/benchmarks/probes/judge-swap-validation/probe-script.py
@@ -0,0 +1,486 @@
+"""
+§1.3g Judge Swap Validation Probe — 4-candidate roster
+=======================================================
+
+Tests whether Kimi, MiniMax, DeepSeek, or Zhipu can replace Gemini 3.1
+Pro Preview in the judge ensemble. Per PM-RATIFY-VERTEX-BATCH-
+ELIGIBILITY INFEASIBLE exit: Branch A closed; this probe is the primary
+path to unblock Stage 3 independent of the Google quota ticket.
+
+Routing (PM ratified 2026-04-24):
+  - Kimi:     direct via Moonshot api.moonshot.ai, model `kimi-k2.6`
+  - MiniMax:  OpenRouter fallback (direct rejected with invalid-api-
+              key / missing GroupId), route `minimax/minimax-m2.7`
+  - DeepSeek: direct via api.deepseek.com, model `deepseek-v4-pro`
+  - Zhipu:    direct via api.z.ai, model `glm-5.1`
+
+Sample: 20 instances stratified 4-per-cell from the Stage 2-Retry
+κ-calibration set at `benchmarks/results/locomo-mini-n20-retry-
+2026-04-24T00-02-12Z.jsonl` (100 rows, each has Opus+GPT+Gemini
+verdicts already computed). Deterministic first-4-per-cell ordering.
+
+Execution: 80 API calls (4 providers × 20 instances), 4 providers
+parallel (ThreadPoolExecutor 4), serial within each provider. Each
+call up to 3 retries on transient errors.
+
+Parser identical to judge-runner pattern (extract JSON body, read
+`verdict` field).
+
+Scope discipline: no §11 frozen path touched; no new LiteLLM aliases;
+no new Opus/GPT calls; manifest v5 anchor fc16925 intact.
+
+Usage:
+  python probe-script.py
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+import time
+import traceback
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timezone
+from pathlib import Path
+import urllib.error
+import urllib.request
+
+# Windows cp1252 stdout fix.
+try:
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+    sys.stderr.reconfigure(encoding="utf-8", errors="replace")
+except Exception:
+    pass
+
+# ── Paths + constants ────────────────────────────────────────────────────
+
+PROBE_DIR = Path("D:/Projects/waggle-os/benchmarks/probes/judge-swap-validation")
+CALIBRATION_SRC = Path("D:/Projects/waggle-os/benchmarks/results/locomo-mini-n20-retry-2026-04-24T00-02-12Z.jsonl")
+SAMPLE_PATH = PROBE_DIR / "sample-instances.jsonl"
+
+CELLS = ["no-context", "oracle-context", "full-context", "retrieval", "agentic"]
+PER_CELL = 4  # 4 per cell × 5 cells = 20 instances
+
+# ── Judge prompt template (verbatim from failure-mode-judge.ts:93-140) ──
+
+JUDGE_PROMPT_TEMPLATE = "\n".join([
+    "You are evaluating whether an LLM's answer is correct against ground truth.",
+    "",
+    "## Question",
+    "{question}",
+    "",
+    "## Ground-truth answer",
+    "{ground_truth}",
+    "",
+    "## Ground-truth supporting context (excerpt shown to the model)",
+    "{context}",
+    "",
+    "## Model's answer",
+    "{model_answer}",
+    "",
+    "## Your task",
+    "",
+    "Step 1: Determine if the model's answer is correct.",
+    "- \"correct\" means the model's answer contains all required facts from ground truth, with no additional incorrect claims.",
+    "- Minor phrasing differences, synonyms, or alternative but equivalent formulations are acceptable.",
+    "- Extra detail is acceptable ONLY if it is factually correct.",
+    "",
+    "Step 2: If incorrect, assign exactly one failure mode using this decision tree:",
+    "",
+    "1. Does the model explicitly refuse or say it does not know? -> F1 (ABSTAIN)",
+    "2. Does the model answer a DIFFERENT question than was asked (coherent but off-topic)? -> F5 (OFF-TOPIC)",
+    "3. Does the model rely on entities, names, dates, or claims that do NOT appear in the ground-truth context (fabrication)? -> F4 (HALLUCINATED)",
+    "4. Does the model correctly state SOME required facts but miss others, without stating any incorrect facts? -> F2 (PARTIAL)",
+    "5. Otherwise (model states facts derived from the context but gets them wrong): -> F3 (INCORRECT)",
+    "",
+    "Step 3: Return JSON only, no prose, in this exact schema:",
+    "",
+    "{{",
+    "  \"verdict\": \"correct\" | \"incorrect\",",
+    "  \"failure_mode\": null | \"F1\" | \"F2\" | \"F3\" | \"F4\" | \"F5\",",
+    "  \"rationale\": \"one sentence explaining the verdict\"",
+    "}}",
+    "",
+    "If verdict is \"correct\", failure_mode MUST be null.",
+    "If verdict is \"incorrect\", failure_mode MUST be one of F1-F5.",
+])
+
+
+def ts() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def logmsg(msg: str) -> None:
+    print(f"{ts()} {msg}", flush=True)
+
+
+# ── .env loader ──────────────────────────────────────────────────────────
+
+def load_env() -> dict[str, str]:
+    env_path = Path("D:/Projects/waggle-os/.env")
+    out: dict[str, str] = {}
+    if not env_path.exists():
+        return out
+    for line in env_path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        if "=" not in line:
+            continue
+        k, _, v = line.partition("=")
+        k, v = k.strip(), v.strip().strip('"').strip("'")
+        out[k] = v
+    return out
+
+
+# ── Verbatim judge-runner JSON extractor (mirror of extractJsonBody) ────
+
+def extract_json_body(raw: str) -> dict | None:
+    if not raw:
+        return None
+    trimmed = raw.strip()
+    # Code-fence strip.
+    if trimmed.startswith("```"):
+        m = re.match(r"^```(?:json)?\s*\n?(.*?)```\s*$", trimmed, re.DOTALL)
+        if m:
+            trimmed = m.group(1).strip()
+    # Try direct parse.
+    try:
+        return json.loads(trimmed)
+    except Exception:
+        pass
+    # First { to matching last }.
+    first = trimmed.find("{")
+    last = trimmed.rfind("}")
+    if first != -1 and last != -1 and last > first:
+        try:
+            return json.loads(trimmed[first:last+1])
+        except Exception:
+            return None
+    return None
+
+
+def parse_verdict(raw_text: str) -> tuple[str | None, str | None, str | None]:
+    body = extract_json_body(raw_text or "")
+    if not isinstance(body, dict):
+        return (None, None, None)
+    v = body.get("verdict")
+    fm = body.get("failure_mode")
+    rat = body.get("rationale")
+    if v not in ("correct", "incorrect"):
+        return (None, None, None)
+    if fm is not None and fm not in ("F1", "F2", "F3", "F4", "F5"):
+        fm = None
+    return (v, fm, rat if isinstance(rat, str) else None)
+
+
+# ── HTTP helper ─────────────────────────────────────────────────────────
+
+def http_post_json(url: str, headers: dict, body: dict, timeout_s: int = 60) -> tuple[int, dict | str]:
+    req = urllib.request.Request(
+        url,
+        data=json.dumps(body).encode("utf-8"),
+        method="POST",
+        headers={"Content-Type": "application/json", **headers},
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=timeout_s) as resp:
+            raw = resp.read().decode("utf-8", errors="replace")
+            try:
+                return resp.status, json.loads(raw)
+            except Exception:
+                return resp.status, raw
+    except urllib.error.HTTPError as e:
+        try:
+            body_err = e.read().decode("utf-8", errors="replace")
+        except Exception:
+            body_err = ""
+        return e.code, body_err[:2000]
+    except Exception as e:
+        return 0, f"{type(e).__name__}: {e}"
+
+
+# ── Provider-specific callers ──────────────────────────────────────────
+
+def call_kimi(prompt: str, env: dict) -> dict:
+    # kimi-k2.6 quirks: (a) rejects `temperature` != 1 ("only 1 is allowed
+    # for this model"; same pattern as Opus/GPT-5.x reasoning models
+    # already accommodated in judge-client.ts:88); (b) reasoning-heavy
+    # output — 4-5k chars of chain-of-thought before the JSON verdict,
+    # hits 1024 token ceiling mid-reasoning. Raise max_tokens to 4096.
+    url = "https://api.moonshot.ai/v1/chat/completions"
+    headers = {"Authorization": f"Bearer {env['MOONSHOT_API_KEY']}"}
+    body = {
+        "model": "kimi-k2.6",
+        "messages": [{"role": "user", "content": prompt}],
+        "max_tokens": 4096,
+    }
+    return _retry_call(url, headers, body, "kimi-k2.6", routing="direct")
+
+
+def call_deepseek(prompt: str, env: dict) -> dict:
+    url = "https://api.deepseek.com/v1/chat/completions"
+    headers = {"Authorization": f"Bearer {env['DEEPSEEK_API_KEY']}"}
+    body = {
+        "model": "deepseek-v4-pro",
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": 0.0,
+        "max_tokens": 1024,
+    }
+    return _retry_call(url, headers, body, "deepseek-v4-pro", routing="direct")
+
+
+def call_zhipu(prompt: str, env: dict) -> dict:
+    url = "https://api.z.ai/api/paas/v4/chat/completions"
+    headers = {"Authorization": f"Bearer {env['ZHIPU_API_KEY']}"}
+    body = {
+        "model": "glm-5.1",
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": 0.0,
+        "max_tokens": 1024,
+    }
+    return _retry_call(url, headers, body, "glm-5.1", routing="direct")
+
+
+def call_minimax_via_openrouter(prompt: str, env: dict) -> dict:
+    # MiniMax M2.7 via OpenRouter is reasoning-heavy — 3/20 initial rows
+    # had empty content with completion_tokens=1024 (hit ceiling mid-
+    # reasoning, same pattern as Kimi). Raise max_tokens to 4096.
+    url = "https://openrouter.ai/api/v1/chat/completions"
+    headers = {"Authorization": f"Bearer {env['OPENROUTER_API_KEY']}"}
+    body = {
+        "model": "minimax/minimax-m2.7",
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": 0.0,
+        "max_tokens": 4096,
+    }
+    return _retry_call(url, headers, body, "minimax/minimax-m2.7", routing="openrouter")
+
+
+def _retry_call(url: str, headers: dict, body: dict, model_id: str, routing: str,
+                max_attempts: int = 3) -> dict:
+    """Returns {raw_text, status, error, retries, model_id, routing, latency_ms}."""
+    started = time.time()
+    last_err = None
+    retries = 0
+    for attempt in range(max_attempts):
+        status, resp = http_post_json(url, headers, body, timeout_s=60)
+        if status == 200 and isinstance(resp, dict):
+            choices = resp.get("choices") or []
+            if choices:
+                msg = choices[0].get("message") or {}
+                content = msg.get("content") or msg.get("reasoning_content") or ""
+                usage = resp.get("usage", {})
+                return {
+                    "raw_text": content,
+                    "status": 200,
+                    "error": None,
+                    "retries": retries,
+                    "model_id": model_id,
+                    "routing": routing,
+                    "latency_ms": int((time.time() - started) * 1000),
+                    "prompt_tokens": usage.get("prompt_tokens"),
+                    "completion_tokens": usage.get("completion_tokens"),
+                }
+        last_err = f"status={status} resp={str(resp)[:400]}"
+        retries += 1
+        if attempt < max_attempts - 1:
+            time.sleep(2 ** attempt)
+    return {
+        "raw_text": "",
+        "status": 0,
+        "error": last_err,
+        "retries": retries,
+        "model_id": model_id,
+        "routing": routing,
+        "latency_ms": int((time.time() - started) * 1000),
+        "prompt_tokens": None,
+        "completion_tokens": None,
+    }
+
+
+# ── Sample selection (stratified 4 per cell) ────────────────────────────
+
+def build_sample() -> list[dict]:
+    """Read κ calibration JSONL, select 4 instances per cell (first-4 by
+    file order), extract fields needed for judge prompt + Opus+GPT ground
+    truth."""
+    per_cell_rows: dict[str, list[dict]] = {c: [] for c in CELLS}
+    with CALIBRATION_SRC.open("r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                r = json.loads(line)
+            except Exception:
+                continue
+            cell = r.get("cell")
+            if cell not in per_cell_rows:
+                continue
+            if len(per_cell_rows[cell]) >= PER_CELL:
+                continue
+            # Extract Opus + GPT verdicts from judge_ensemble.
+            ensemble = r.get("judge_ensemble") or []
+            opus = next((j for j in ensemble if "opus" in j.get("model", "").lower()), None)
+            gpt = next((j for j in ensemble if "gpt" in j.get("model", "").lower()), None)
+            if opus is None or gpt is None:
+                continue
+            per_cell_rows[cell].append({
+                "instance_id": r.get("instance_id"),
+                "cell": cell,
+                "question": r.get("model_answer", "")  # placeholder
+                            # Actual: the judge sees (question, ground_truth, context, model_answer).
+                            # We reconstruct from the calibration row + canonical fixture lookup
+                            # in a separate step below.
+                            ,
+                "opus_verdict": opus.get("verdict"),
+                "opus_failure_mode": opus.get("failure_mode"),
+                "gpt_verdict": gpt.get("verdict"),
+                "gpt_failure_mode": gpt.get("failure_mode"),
+                "model_answer": r.get("model_answer", ""),  # subject's answer
+            })
+    sample = []
+    for c in CELLS:
+        sample.extend(per_cell_rows[c])
+    return sample
+
+
+def enrich_sample_with_locomo(sample: list[dict]) -> list[dict]:
+    """κ-calibration JSONL has model_answer but not question/ground_truth/
+    context. Join against the canonical LoCoMo fixture by instance_id."""
+    locomo_path = Path("D:/Projects/waggle-os/benchmarks/data/locomo/locomo-1540.jsonl")
+    by_id: dict[str, dict] = {}
+    with locomo_path.open("r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                r = json.loads(line)
+            except Exception:
+                continue
+            by_id[r.get("instance_id")] = r
+    enriched = []
+    for s in sample:
+        iid = s["instance_id"]
+        src = by_id.get(iid)
+        if src is None:
+            s["question"] = None
+            s["ground_truth"] = None
+            s["context"] = None
+            enriched.append(s)
+            continue
+        s["question"] = src.get("question")
+        s["ground_truth"] = (src.get("expected") or [src.get("gold_answer", "")])[0]
+        s["context"] = src.get("context", "")
+        enriched.append(s)
+    return enriched
+
+
+# ── Main probe ──────────────────────────────────────────────────────────
+
+PROVIDER_FNS = [
+    ("kimi", call_kimi),
+    ("minimax", call_minimax_via_openrouter),
+    ("deepseek", call_deepseek),
+    ("zhipu", call_zhipu),
+]
+
+
+def run_provider(name: str, call_fn, sample: list[dict], env: dict) -> list[dict]:
+    logmsg(f"[{name}] start — {len(sample)} instances")
+    out: list[dict] = []
+    for i, s in enumerate(sample):
+        prompt = JUDGE_PROMPT_TEMPLATE.format(
+            question=s.get("question") or "",
+            ground_truth=s.get("ground_truth") or "",
+            context=s.get("context") or "",
+            model_answer=s.get("model_answer") or "",
+        )
+        resp = call_fn(prompt, env)
+        verdict, fm, rat = parse_verdict(resp["raw_text"])
+        row = {
+            "instance_id": s["instance_id"],
+            "cell": s["cell"],
+            "provider": name,
+            "model_id": resp["model_id"],
+            "routing": resp["routing"],
+            "http_status": resp["status"],
+            "error": resp.get("error"),
+            "retries": resp["retries"],
+            "latency_ms": resp["latency_ms"],
+            "prompt_tokens": resp.get("prompt_tokens"),
+            "completion_tokens": resp.get("completion_tokens"),
+            "raw_text": resp["raw_text"],
+            "parsed_verdict": verdict,
+            "parsed_failure_mode": fm,
+            "parsed_rationale": rat,
+        }
+        out.append(row)
+        logmsg(f"[{name}] {i+1:>2}/{len(sample)} {s['instance_id']}  status={resp['status']}  verdict={verdict}  retries={resp['retries']}")
+    logmsg(f"[{name}] done")
+    return out
+
+
+def write_jsonl(path: Path, rows: list[dict]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as f:
+        for r in rows:
+            f.write(json.dumps(r, ensure_ascii=False) + "\n")
+
+
+def main() -> int:
+    logmsg("[probe] §1.3g judge swap validation START")
+    env = load_env()
+    required = ["MOONSHOT_API_KEY", "OPENROUTER_API_KEY", "DEEPSEEK_API_KEY", "ZHIPU_API_KEY"]
+    missing = [k for k in required if not env.get(k)]
+    if missing:
+        logmsg(f"[probe] FATAL missing env keys: {missing}")
+        return 2
+
+    logmsg("[probe] step 1: build stratified sample (4 per cell × 5 cells = 20)")
+    sample = build_sample()
+    if len(sample) != 20:
+        logmsg(f"[probe] WARN sample size={len(sample)} (expected 20)")
+    sample = enrich_sample_with_locomo(sample)
+    null_qctx = sum(1 for s in sample if not s.get("question") or s.get("context") is None)
+    logmsg(f"[probe] sample built: {len(sample)} instances; null_q_or_ctx={null_qctx}")
+    write_jsonl(SAMPLE_PATH, sample)
+
+    logmsg("[probe] step 2: execute 4 providers in parallel (4 threads)")
+    provider_rows: dict[str, list[dict]] = {}
+    with ThreadPoolExecutor(max_workers=4) as pool:
+        futures = {
+            pool.submit(run_provider, name, fn, sample, env): name
+            for (name, fn) in PROVIDER_FNS
+        }
+        for fut in as_completed(futures):
+            name = futures[fut]
+            try:
+                rows = fut.result()
+                provider_rows[name] = rows
+            except Exception as e:
+                logmsg(f"[probe] provider {name} FAILED: {type(e).__name__}: {e}")
+                provider_rows[name] = []
+
+    logmsg("[probe] step 3: write per-provider JSONL artefacts")
+    for name, rows in provider_rows.items():
+        write_jsonl(PROBE_DIR / f"{name}-responses.jsonl", rows)
+        parsed = sum(1 for r in rows if r.get("parsed_verdict") is not None)
+        logmsg(f"[probe] {name}: wrote {len(rows)} rows, parsed_ok={parsed}")
+
+    logmsg("[probe] step 4: κ computation handled by companion analysis script")
+    logmsg("[probe] END")
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except Exception as e:
+        logmsg(f"[FATAL] {type(e).__name__}: {e}")
+        logmsg(traceback.format_exc()[:2000])
+        sys.exit(99)
--- a/benchmarks/probes/judge-swap-validation/reprobe-memo.md
+++ b/benchmarks/probes/judge-swap-validation/reprobe-memo.md
@@ -0,0 +1,39 @@
+# §1.3h Judge Swap Re-Probe — Exit Memo
+
+**Date:** 2026-04-24  **Target:** PM-RATIFY-JUDGE-SWAP-REPROBE  **Parent:** `8a2f0e6`
+
+## Verdict: **INCONCLUSIVE_BUT_OPERATIONAL_SIGNAL**
+
+Split pool n=7 (use-all-available per PM amendment §1.3H OPTION 1). All 7 splits are `Opus=correct / GPT=incorrect` → split-only κ is structurally degenerate; **p_opus/p_gpt balance is the discriminating signal**.
+
+## Novel finding (supersedes §1.3g heuristic)
+
+| Cand | p_opus | p_gpt | Profile | κ_agg_cons (n=27) |
+|------|--------|-------|---------|---------------------|
+| **DeepSeek** | 40% | 60% | **Independent** | **0.709 (only PASS)** |
+| Kimi | 80% | 20% | Opus-lenient | 0.576 |
+| MiniMax | 86% | 14% | Opus-lenient | 0.456 |
+| Zhipu | **0%** | **100%** | **GPT-echo** | 0.444 |
+
+§1.3g κ=1.0 tie was selection bias toward unanimous cases. §1.3h exposes Zhipu as GPT clone on splits (low ensemble value); DeepSeek as only balanced independent judge.
+
+## Recommendation
+
+**Primary: DEEPSEEK** (with `max_tokens=2048` bump to address split-parse 71%)
+**Backup: ZHIPU** (independence compromise if DeepSeek parse unresolved)
+
+A 3-judge ensemble needs independent calibration; Zhipu's 100% GPT-lock-step violates this. Parse risk is mitigable (token ceiling); echo risk is structural.
+
+## MiniMax routing
+
+**`direct_failed_fell_back_openrouter`.** `api.minimaxi.com` and `api.minimax.chat` v2 rejected `MINIMAX_GROUP_ID + Bearer`. OpenRouter fallback 7/7. **Manifest v6 must route MiniMax via OpenRouter**; direct is not viable without further account research.
+
+## Cost / wall-clock
+
+- 28 split calls + 4 MiniMax direct-discovery attempts
+- **Cost: ~$0.30** (cap $1.50)
+- **Wall-clock: ~9 min** (14:09-14:18 UTC)
+
+## PM next step
+
+Adjudicate: (A) DeepSeek-boosted primary + Zhipu backup, (B) Zhipu primary + DeepSeek-boosted backup (operational-first), (C) DeepSeek mini-confirmation probe with bumped max_tokens before v6 emission. `cc1_state: HALTED`.
--- a/benchmarks/probes/judge-swap-validation/sample-instances.jsonl
+++ b/benchmarks/probes/judge-swap-validation/sample-instances.jsonl
@@ -0,0 +1,20 @@
+{"instance_id": "locomo_conv-50_q086", "cell": "no-context", "question": "What sports activity is Calvin planning to try after the tour with Frank Ocean?", "opus_verdict": "incorrect", "opus_failure_mode": "F3", "gpt_verdict": "incorrect", "gpt_failure_mode": "F3", "model_answer": "Surfing", "ground_truth": "Skiing", "context": "Session 9 (3:15 pm on 21 June, 2023):\nCalvin: Nope, not yet! I haven't been there before, but it's on my to-do list for after my tour with Frank Ocean ends. Can't wait to see them in person! Look at that snowy peak. [image: a photo of a person on skis on a snowy mountain]"}
+{"instance_id": "locomo_conv-26_q059", "cell": "no-context", "question": "Would Caroline be considered religious?", "opus_verdict": "incorrect", "opus_failure_mode": "F1", "gpt_verdict": "incorrect", "gpt_failure_mode": "F1", "model_answer": "Cannot be determined without context.", "ground_truth": "Somewhat, but not extremely religious", "context": "Session 12 (1:50 pm on 17 August, 2023):\nCaroline: Hey Mel! How're ya doin'? Recently, I had a not-so-great experience on a hike. I ran into a group of religious conservatives who said something that really upset me. It made me think how much work we still have to do for LGBTQ rights. It's been so helpful to have people around me who accept and support me, so I know I'll be ok!\n\nSession 14 (1:33 pm on 25 August, 2023):\nCaroline: Thanks! It was made for a local church and shows time changing our lives. I made it to show my own journey as a transgender woman and how we should accept growth and change. [image: a photo of a large stained glass window in a church]"}
+{"instance_id": "locomo_conv-44_q000", "cell": "no-context", "question": "Which year did Audrey adopt the first three of her dogs?", "opus_verdict": "incorrect", "opus_failure_mode": "F4", "gpt_verdict": "incorrect", "gpt_failure_mode": "F4", "model_answer": "1954", "ground_truth": "2020", "context": "Session 1 (1:10 pm on 27 March, 2023):\nAudrey: I've had them for 3 years! Their names are Pepper, Precious and Panda. I can't live without my little ones!"}
+{"instance_id": "locomo_conv-41_q080", "cell": "no-context", "question": "What did Maria make for her home to remind her of a trip to England?", "opus_verdict": "incorrect", "opus_failure_mode": "F4", "gpt_verdict": "incorrect", "gpt_failure_mode": "F3", "model_answer": "A wind chime", "ground_truth": "painting of a castle on a hill", "context": "Session 8 (6:03 pm on 6 March, 2023):\nMaria: Thanks, John! I got the idea from that trip to England a few years ago - I was mesmerized by the castles. I wanted to bring that same magic to my home, like a reminder of the world's beauty. I made a painting too! [image: a photography of a painting of a castle on a hill]"}
+{"instance_id": "locomo_conv-50_q086", "cell": "oracle-context", "question": "What sports activity is Calvin planning to try after the tour with Frank Ocean?", "opus_verdict": "correct", "opus_failure_mode": null, "gpt_verdict": "correct", "gpt_failure_mode": null, "model_answer": "Skiing", "ground_truth": "Skiing", "context": "Session 9 (3:15 pm on 21 June, 2023):\nCalvin: Nope, not yet! I haven't been there before, but it's on my to-do list for after my tour with Frank Ocean ends. Can't wait to see them in person! Look at that snowy peak. [image: a photo of a person on skis on a snowy mountain]"}
+{"instance_id": "locomo_conv-26_q059", "cell": "oracle-context", "question": "Would Caroline be considered religious?", "opus_verdict": "incorrect", "opus_failure_mode": "F3", "gpt_verdict": "incorrect", "gpt_failure_mode": "F2", "model_answer": "Yes", "ground_truth": "Somewhat, but not extremely religious", "context": "Session 12 (1:50 pm on 17 August, 2023):\nCaroline: Hey Mel! How're ya doin'? Recently, I had a not-so-great experience on a hike. I ran into a group of religious conservatives who said something that really upset me. It made me think how much work we still have to do for LGBTQ rights. It's been so helpful to have people around me who accept and support me, so I know I'll be ok!\n\nSession 14 (1:33 pm on 25 August, 2023):\nCaroline: Thanks! It was made for a local church and shows time changing our lives. I made it to show my own journey as a transgender woman and how we should accept growth and change. [image: a photo of a large stained glass window in a church]"}
+{"instance_id": "locomo_conv-44_q000", "cell": "oracle-context", "question": "Which year did Audrey adopt the first three of her dogs?", "opus_verdict": "correct", "opus_failure_mode": null, "gpt_verdict": "correct", "gpt_failure_mode": null, "model_answer": "2020", "ground_truth": "2020", "context": "Session 1 (1:10 pm on 27 March, 2023):\nAudrey: I've had them for 3 years! Their names are Pepper, Precious and Panda. I can't live without my little ones!"}
+{"instance_id": "locomo_conv-41_q080", "cell": "oracle-context", "question": "What did Maria make for her home to remind her of a trip to England?", "opus_verdict": "incorrect", "opus_failure_mode": "F2", "gpt_verdict": "incorrect", "gpt_failure_mode": "F2", "model_answer": "A painting", "ground_truth": "painting of a castle on a hill", "context": "Session 8 (6:03 pm on 6 March, 2023):\nMaria: Thanks, John! I got the idea from that trip to England a few years ago - I was mesmerized by the castles. I wanted to bring that same magic to my home, like a reminder of the world's beauty. I made a painting too! [image: a photography of a painting of a castle on a hill]"}
+{"instance_id": "locomo_conv-50_q086", "cell": "full-context", "question": "What sports activity is Calvin planning to try after the tour with Frank Ocean?", "opus_verdict": "correct", "opus_failure_mode": null, "gpt_verdict": "correct", "gpt_failure_mode": null, "model_answer": "skiing", "ground_truth": "Skiing", "context": "Session 9 (3:15 pm on 21 June, 2023):\nCalvin: Nope, not yet! I haven't been there before, but it's on my to-do list for after my tour with Frank Ocean ends. Can't wait to see them in person! Look at that snowy peak. [image: a photo of a person on skis on a snowy mountain]"}
+{"instance_id": "locomo_conv-26_q059", "cell": "full-context", "question": "Would Caroline be considered religious?", "opus_verdict": "incorrect", "opus_failure_mode": "F1", "gpt_verdict": "incorrect", "gpt_failure_mode": "F1", "model_answer": "unknown", "ground_truth": "Somewhat, but not extremely religious", "context": "Session 12 (1:50 pm on 17 August, 2023):\nCaroline: Hey Mel! How're ya doin'? Recently, I had a not-so-great experience on a hike. I ran into a group of religious conservatives who said something that really upset me. It made me think how much work we still have to do for LGBTQ rights. It's been so helpful to have people around me who accept and support me, so I know I'll be ok!\n\nSession 14 (1:33 pm on 25 August, 2023):\nCaroline: Thanks! It was made for a local church and shows time changing our lives. I made it to show my own journey as a transgender woman and how we should accept growth and change. [image: a photo of a large stained glass window in a church]"}
+{"instance_id": "locomo_conv-44_q000", "cell": "full-context", "question": "Which year did Audrey adopt the first three of her dogs?", "opus_verdict": "correct", "opus_failure_mode": null, "gpt_verdict": "correct", "gpt_failure_mode": null, "model_answer": "2020", "ground_truth": "2020", "context": "Session 1 (1:10 pm on 27 March, 2023):\nAudrey: I've had them for 3 years! Their names are Pepper, Precious and Panda. I can't live without my little ones!"}
+{"instance_id": "locomo_conv-41_q080", "cell": "full-context", "question": "What did Maria make for her home to remind her of a trip to England?", "opus_verdict": "incorrect", "opus_failure_mode": "F2", "gpt_verdict": "incorrect", "gpt_failure_mode": "F2", "model_answer": "a painting", "ground_truth": "painting of a castle on a hill", "context": "Session 8 (6:03 pm on 6 March, 2023):\nMaria: Thanks, John! I got the idea from that trip to England a few years ago - I was mesmerized by the castles. I wanted to bring that same magic to my home, like a reminder of the world's beauty. I made a painting too! [image: a photography of a painting of a castle on a hill]"}
+{"instance_id": "locomo_conv-50_q086", "cell": "retrieval", "question": "What sports activity is Calvin planning to try after the tour with Frank Ocean?", "opus_verdict": "incorrect", "opus_failure_mode": "F1", "gpt_verdict": "incorrect", "gpt_failure_mode": "F1", "model_answer": "Not mentioned", "ground_truth": "Skiing", "context": "Session 9 (3:15 pm on 21 June, 2023):\nCalvin: Nope, not yet! I haven't been there before, but it's on my to-do list for after my tour with Frank Ocean ends. Can't wait to see them in person! Look at that snowy peak. [image: a photo of a person on skis on a snowy mountain]"}
+{"instance_id": "locomo_conv-26_q059", "cell": "retrieval", "question": "Would Caroline be considered religious?", "opus_verdict": "incorrect", "opus_failure_mode": "F3", "gpt_verdict": "incorrect", "gpt_failure_mode": "F3", "model_answer": "No.", "ground_truth": "Somewhat, but not extremely religious", "context": "Session 12 (1:50 pm on 17 August, 2023):\nCaroline: Hey Mel! How're ya doin'? Recently, I had a not-so-great experience on a hike. I ran into a group of religious conservatives who said something that really upset me. It made me think how much work we still have to do for LGBTQ rights. It's been so helpful to have people around me who accept and support me, so I know I'll be ok!\n\nSession 14 (1:33 pm on 25 August, 2023):\nCaroline: Thanks! It was made for a local church and shows time changing our lives. I made it to show my own journey as a transgender woman and how we should accept growth and change. [image: a photo of a large stained glass window in a church]"}
+{"instance_id": "locomo_conv-44_q000", "cell": "retrieval", "question": "Which year did Audrey adopt the first three of her dogs?", "opus_verdict": "incorrect", "opus_failure_mode": "F1", "gpt_verdict": "incorrect", "gpt_failure_mode": "F1", "model_answer": "Not mentioned.", "ground_truth": "2020", "context": "Session 1 (1:10 pm on 27 March, 2023):\nAudrey: I've had them for 3 years! Their names are Pepper, Precious and Panda. I can't live without my little ones!"}
+{"instance_id": "locomo_conv-41_q080", "cell": "retrieval", "question": "What did Maria make for her home to remind her of a trip to England?", "opus_verdict": "incorrect", "opus_failure_mode": "F2", "gpt_verdict": "incorrect", "gpt_failure_mode": "F2", "model_answer": "A painting.", "ground_truth": "painting of a castle on a hill", "context": "Session 8 (6:03 pm on 6 March, 2023):\nMaria: Thanks, John! I got the idea from that trip to England a few years ago - I was mesmerized by the castles. I wanted to bring that same magic to my home, like a reminder of the world's beauty. I made a painting too! [image: a photography of a painting of a castle on a hill]"}
+{"instance_id": "locomo_conv-50_q086", "cell": "agentic", "question": "What sports activity is Calvin planning to try after the tour with Frank Ocean?", "opus_verdict": "incorrect", "opus_failure_mode": "F1", "gpt_verdict": "incorrect", "gpt_failure_mode": "F1", "model_answer": "Max tool turns reached (3 turns, 3 tools used).", "ground_truth": "Skiing", "context": "Session 9 (3:15 pm on 21 June, 2023):\nCalvin: Nope, not yet! I haven't been there before, but it's on my to-do list for after my tour with Frank Ocean ends. Can't wait to see them in person! Look at that snowy peak. [image: a photo of a person on skis on a snowy mountain]"}
+{"instance_id": "locomo_conv-26_q059", "cell": "agentic", "question": "Would Caroline be considered religious?", "opus_verdict": "incorrect", "opus_failure_mode": "F3", "gpt_verdict": "incorrect", "gpt_failure_mode": "F4", "model_answer": "Yes, Caroline has a connection to faith through her heritage (a necklace from Sweden representing \"love, faith and strength\") and by having made a painting for a local church.", "ground_truth": "Somewhat, but not extremely religious", "context": "Session 12 (1:50 pm on 17 August, 2023):\nCaroline: Hey Mel! How're ya doin'? Recently, I had a not-so-great experience on a hike. I ran into a group of religious conservatives who said something that really upset me. It made me think how much work we still have to do for LGBTQ rights. It's been so helpful to have people around me who accept and support me, so I know I'll be ok!\n\nSession 14 (1:33 pm on 25 August, 2023):\nCaroline: Thanks! It was made for a local church and shows time changing our lives. I made it to show my own journey as a transgender woman and how we should accept growth and change. [image: a photo of a large stained glass window in a church]"}
+{"instance_id": "locomo_conv-44_q000", "cell": "agentic", "question": "Which year did Audrey adopt the first three of her dogs?", "opus_verdict": "incorrect", "opus_failure_mode": "F1", "gpt_verdict": "incorrect", "gpt_failure_mode": "F1", "model_answer": "Max tool turns reached (3 turns, 3 tools used).", "ground_truth": "2020", "context": "Session 1 (1:10 pm on 27 March, 2023):\nAudrey: I've had them for 3 years! Their names are Pepper, Precious and Panda. I can't live without my little ones!"}
+{"instance_id": "locomo_conv-41_q080", "cell": "agentic", "question": "What did Maria make for her home to remind her of a trip to England?", "opus_verdict": "incorrect", "opus_failure_mode": "F2", "gpt_verdict": "incorrect", "gpt_failure_mode": "F2", "model_answer": "a painting", "ground_truth": "painting of a castle on a hill", "context": "Session 8 (6:03 pm on 6 March, 2023):\nMaria: Thanks, John! I got the idea from that trip to England a few years ago - I was mesmerized by the castles. I wanted to bring that same magic to my home, like a reminder of the world's beauty. I made a painting too! [image: a photography of a painting of a castle on a hill]"}
--- a/benchmarks/probes/judge-swap-validation/split-cases-sample.jsonl
+++ b/benchmarks/probes/judge-swap-validation/split-cases-sample.jsonl
@@ -0,0 +1,7 @@
+{"instance_id": "locomo_conv-43_q155", "cell": "agentic", "question": "How did John feel after being able to jog without pain?", "ground_truth": "It was a huge success.", "context": "Session 24 (3:37 pm on 16 December, 2023):\nJohn: It was great! After being out for so long, jogging without any pain was a huge success. My wife and I hosted a small get-together with friends and family to celebrate.", "model_answer": "great", "opus_verdict": "correct", "opus_failure_mode": null, "gpt_verdict": "incorrect", "gpt_failure_mode": "F2", "gemini_verdict": "correct", "gemini_failure_mode": null}
+{"instance_id": "locomo_conv-50_q015", "cell": "agentic", "question": "Why does Dave regularly visit parks?", "ground_truth": "because it relaxes and calms him", "context": "Session 1 (11:53 am on 23 March, 2023):\nDave: That's great, can't wait for you to return from your trip. I've been spending lots of time at this beautiful park lately - so calming. Have you been there before? [image: a photography of a park with a lake and a few boats]\n\nSession 8 (2:31 pm on 9 June, 2023):\nDave: Hey, nice photo of the studio! Working in a team yields incredible results, well done! Keep pushing it! I've been doing good - thanks for asking. I've been exploring some parks on the weekends to relax - it's so peaceful being surrounded by nature. Are there any chill spots you enjoy in Boston?", "model_answer": "to relax and recharge", "opus_verdict": "correct", "opus_failure_mode": null, "gpt_verdict": "incorrect", "gpt_failure_mode": "F2", "gemini_verdict": "correct", "gemini_failure_mode": null}
+{"instance_id": "locomo_conv-43_q155", "cell": "full-context", "question": "How did John feel after being able to jog without pain?", "ground_truth": "It was a huge success.", "context": "Session 24 (3:37 pm on 16 December, 2023):\nJohn: It was great! After being out for so long, jogging without any pain was a huge success. My wife and I hosted a small get-together with friends and family to celebrate.", "model_answer": "great", "opus_verdict": "correct", "opus_failure_mode": null, "gpt_verdict": "incorrect", "gpt_failure_mode": "F2", "gemini_verdict": "correct", "gemini_failure_mode": null}
+{"instance_id": "locomo_conv-50_q015", "cell": "full-context", "question": "Why does Dave regularly visit parks?", "ground_truth": "because it relaxes and calms him", "context": "Session 1 (11:53 am on 23 March, 2023):\nDave: That's great, can't wait for you to return from your trip. I've been spending lots of time at this beautiful park lately - so calming. Have you been there before? [image: a photography of a park with a lake and a few boats]\n\nSession 8 (2:31 pm on 9 June, 2023):\nDave: Hey, nice photo of the studio! Working in a team yields incredible results, well done! Keep pushing it! I've been doing good - thanks for asking. I've been exploring some parks on the weekends to relax - it's so peaceful being surrounded by nature. Are there any chill spots you enjoy in Boston?", "model_answer": "to relax", "opus_verdict": "correct", "opus_failure_mode": null, "gpt_verdict": "incorrect", "gpt_failure_mode": "F2", "gemini_verdict": "correct", "gemini_failure_mode": null}
+{"instance_id": "locomo_conv-43_q155", "cell": "oracle-context", "question": "How did John feel after being able to jog without pain?", "ground_truth": "It was a huge success.", "context": "Session 24 (3:37 pm on 16 December, 2023):\nJohn: It was great! After being out for so long, jogging without any pain was a huge success. My wife and I hosted a small get-together with friends and family to celebrate.", "model_answer": "Great", "opus_verdict": "correct", "opus_failure_mode": null, "gpt_verdict": "incorrect", "gpt_failure_mode": "F2", "gemini_verdict": "correct", "gemini_failure_mode": null}
+{"instance_id": "locomo_conv-47_q092", "cell": "oracle-context", "question": "What did John create for the charitable foundation that helped generate reports for analysis?", "ground_truth": "computer application on smartphones", "context": "Session 11 (5:00 pm on 11 May, 2022):\nJohn: Previously, this foundation used paper records and all inventory was recorded manually. I made an application that structured their work, and now everything they need for inventory is in one application on their smartphone.", "model_answer": "An application", "opus_verdict": "correct", "opus_failure_mode": null, "gpt_verdict": "incorrect", "gpt_failure_mode": "F2", "gemini_verdict": "incorrect", "gemini_failure_mode": "F2"}
+{"instance_id": "locomo_conv-50_q015", "cell": "oracle-context", "question": "Why does Dave regularly visit parks?", "ground_truth": "because it relaxes and calms him", "context": "Session 1 (11:53 am on 23 March, 2023):\nDave: That's great, can't wait for you to return from your trip. I've been spending lots of time at this beautiful park lately - so calming. Have you been there before? [image: a photography of a park with a lake and a few boats]\n\nSession 8 (2:31 pm on 9 June, 2023):\nDave: Hey, nice photo of the studio! Working in a team yields incredible results, well done! Keep pushing it! I've been doing good - thanks for asking. I've been exploring some parks on the weekends to relax - it's so peaceful being surrounded by nature. Are there any chill spots you enjoy in Boston?", "model_answer": "To relax", "opus_verdict": "correct", "opus_failure_mode": null, "gpt_verdict": "incorrect", "gpt_failure_mode": "F2", "gemini_verdict": "correct", "gemini_failure_mode": null}
--- a/benchmarks/probes/judge-swap-validation/validation-memo.md
+++ b/benchmarks/probes/judge-swap-validation/validation-memo.md
@@ -0,0 +1,30 @@
+# §1.3g Judge Swap Validation — Memo
+
+**Date:** 2026-04-24 · **Target:** PM-RATIFY-JUDGE-SWAP-VALIDATION.
+
+## Aggregate verdict: **MULTI_PASS** — all 4 candidates κ=1.0 vs Opus+GPT consensus (parsed subset).
+
+| Cand | κ | parse | p50 lat | routing |
+|------|-----|-------|---------|---------|
+| kimi-k2.6 | 1.0000 | 20/20 | 32 s | direct api.moonshot.ai |
+| minimax-m2.7 | 1.0000 | 20/20 | 12 s | openrouter |
+| deepseek-v4-pro | 1.0000 | 18/20 | 12 s | direct api.deepseek.com |
+| glm-5.1 | 1.0000 | 19/20 | 9 s | direct api.z.ai |
+
+**Sample caveat:** 0/20 Opus-GPT splits vs 7/100 full-set → first-4-per-
+cell biased toward agreement; κ=1.0 is "match on biased subset", not
+arbitrary LoCoMo. Task 2.6: stratified split-oversampled re-probe.
+
+## Ranking (κ-tied → speed × parse × direct)
+
+1. **ZHIPU GLM-5.1** (primary) — fastest, direct.
+2. **DEEPSEEK V4-Pro** (backup) — direct, 18/20.
+3. MiniMax — 20/20 but OpenRouter dep.
+4. Kimi — 20/20 but p50=32 s → ~20 hr N=400 judge thread, deal-breaker.
+
+## Notes
+
+- Kimi: `temperature` omitted + `max_tokens=4096` (reasoning; parallels Opus/GPT-5.x at judge-client.ts:88).
+- MiniMax direct: `invalid api key` (implied missing GroupId); OR fallback per brief §2.2.
+
+**Cost:** ~$0.25 (<< $8). **Wall-clock:** ~35 min. **SDKs:** stdlib urllib only.
--- a/benchmarks/probes/judge-swap-validation/zhipu-responses.jsonl
+++ b/benchmarks/probes/judge-swap-validation/zhipu-responses.jsonl
@@ -0,0 +1,20 @@
+{"instance_id": "locomo_conv-50_q086", "cell": "no-context", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 21950, "prompt_tokens": 480, "completion_tokens": 763, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F4\",\n  \"rationale\": \"The model claims the sport is surfing, which is a fabricated claim that does not appear in the ground-truth context mentioning skiing and a snowy mountain.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F4", "parsed_rationale": "The model claims the sport is surfing, which is a fabricated claim that does not appear in the ground-truth context mentioning skiing and a snowy mountain."}
+{"instance_id": "locomo_conv-26_q059", "cell": "no-context", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 8921, "prompt_tokens": 568, "completion_tokens": 291, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F1\",\n  \"rationale\": \"The model explicitly refuses to answer by stating it cannot be determined without context, rather than providing the ground-truth assessment of her religiosity.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F1", "parsed_rationale": "The model explicitly refuses to answer by stating it cannot be determined without context, rather than providing the ground-truth assessment of her religiosity."}
+{"instance_id": "locomo_conv-44_q000", "cell": "no-context", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 6323, "prompt_tokens": 443, "completion_tokens": 248, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F4\",\n  \"rationale\": \"The model's answer of 1954 relies on a date that does not appear in the ground-truth context and is therefore fabricated.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F4", "parsed_rationale": "The model's answer of 1954 relies on a date that does not appear in the ground-truth context and is therefore fabricated."}
+{"instance_id": "locomo_conv-41_q080", "cell": "no-context", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 10981, "prompt_tokens": 494, "completion_tokens": 445, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F4\",\n  \"rationale\": \"The model's answer 'A wind chime' relies on a fabricated claim that does not appear in the ground-truth context, which states she made a painting of a castle on a hill.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F4", "parsed_rationale": "The model's answer 'A wind chime' relies on a fabricated claim that does not appear in the ground-truth context, which states she made a painting of a castle on a hill."}
+{"instance_id": "locomo_conv-50_q086", "cell": "oracle-context", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 5692, "prompt_tokens": 480, "completion_tokens": 110, "raw_text": "```json\n{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer exactly matches the ground-truth answer of skiing.\"\n}\n```", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer exactly matches the ground-truth answer of skiing."}
+{"instance_id": "locomo_conv-26_q059", "cell": "oracle-context", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 30977, "prompt_tokens": 564, "completion_tokens": 1024, "raw_text": "The question asks whether Caroline would be considered religious.\nThe ground-truth answer is \"Somewhat, but not extremely religious\".\nThe model's answer is \"Yes\".\n\nThe model's answer \"Yes\" implies that Caroline is religious, but it misses the nuance that she is only \"somewhat, but not extremely religious\". By just saying \"Yes\", the model overstates her religiosity. The ground truth specifies a qualified \"somewhat\", whereas the model gives an unqualified affirmative.\n\nLet's check the failure modes:\n1. F1 (ABSTAIN): No.\n2. F5 (OFF-TOPIC): No.\n3. F4 (HALLUCINATED): No, it's derived from the context (she made a stained glass window for a church).\n4. F2 (PARTIAL): Does the model correctly state SOME required facts but miss others, without stating any incorrect facts? The model states she is religious (which is somewhat true), but misses the qualification \"not extremely\". However, \"Yes\" could be interpreted as stating an incorrect fact (that she is fully religious, not just somewhat). Let's look at F3.\n5. F3 (INCORRECT): Otherwise (model states facts derived from the context but gets them wrong).\n\nIs \"Yes\" a partial answer or an incorrect answer?\nGround truth: \"Somewhat, but not extremely religious\"\nModel: \"Yes\"\n\"Yes\" to \"Would Caroline be considered religious?\" is not entirely wrong, but it misses the nuance. It states a fact derived from the context but gets the degree wrong. Wait, if the question is \"Would Caroline be considered religious?\", \"Yes\" is a partial answer because it misses the \"not extremely\" part. Or is it incorrect because it asserts she is religious without the qualification?\nActually, saying \"Yes\" to \"Would she be considered religious?\" when the answer is \"Somewhat\" is getting the fact wrong/incomplete. F2 is for correctly stating SOME required facts but missing others. \"Yes\" captures the \"somewhat\" part but misses the \"not extremely\" part. Wait, \"Yes\" doesn't state the \"somewhat\" part, it just affirms the binary.\nLet's consider F3: \"model states facts derived from the context but gets them wrong\". The model gets the degree of her religiosity wrong.\nLet's consider F2: \"correctly state SOME required facts but miss others\". The required facts are \"Somewhat\" and \"not extremely\". The model states she is religious (somewhat), but misses the \"not extremely\" part.\nActually, \"Yes\" is an incorrect characterization of \"Somewhat, but not extremely\". If someone asks \"Are you religious?\" and the answer is \"Somewhat, but not extremely\", answering \"Yes\" is factually incorrect because it overstates the religiosity. So F3 might be more appropriate.\nWait, let's re-read F2: \"correctly state SOME required facts but miss others, without stating any incorrect facts\". Does \"Yes\" state an incorrect fact? \"Yes\" implies she is religious, which contradicts \"not extremely religious\" if taken as a strong affirmative. But it's a nuance.\nLet's look at F3: \"Otherwise (model states facts derived from the context but gets them wrong)\". The model gets the answer wrong by omitting the qualification.\nActually, \"Yes\" is a partial answer. It captures the \"somewhat\" but misses the \"not extremely\".\nWait, if the ground truth is \"Somewhat, but not extremely religious\", and the model says \"Yes\", the model is incorrect.\nIs it F2 or F3?\nIf I say \"Yes\", I am stating that she is religious. The ground truth says she is *somewhat* religious. Saying \"Yes\" is an overstatement, which is an incorrect claim. Therefore, it's not just missing facts, it's getting the fact wrong. So F3.\nLet's think if F2 applies. F2: correctly state SOME required facts but miss others. \"Yes\" doesn't explicitly state \"somewhat\". It just says \"Yes\".\nI will classify it as F3 (INCORRECT) because \"Yes\" misrepresents the degree of her religiosity, making it an incorrect answer rather than just a partial one. Or maybe F2 because it correctly identifies she has some religious connection but misses the qualification.\nActually, \"Yes\" is a very common way to answer \"Would she be considered religious?\" if she is somewhat religious. But the ground truth specifically says \"Somewhat, but not extremely\". \"Yes\" misses the entire nuance.\nLet's use F3 because the model gets the fact wrong (implies she is religious, rather than somewhat religious).\nWait, F2 says \"without stating any incorrect facts\". Does \"Yes\" state an incorrect fact? Yes, because she is not extremely religious, and \"Yes\" implies she is religious.\nLet's go with F3.", "parsed_verdict": null, "parsed_failure_mode": null, "parsed_rationale": null}
+{"instance_id": "locomo_conv-44_q000", "cell": "oracle-context", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 5299, "prompt_tokens": 443, "completion_tokens": 134, "raw_text": "```json\n{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer of 2020 exactly matches the ground-truth answer derived from the context (2023 - 3 years).\"\n}\n```", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer of 2020 exactly matches the ground-truth answer derived from the context (2023 - 3 years)."}
+{"instance_id": "locomo_conv-41_q080", "cell": "oracle-context", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 8849, "prompt_tokens": 492, "completion_tokens": 291, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly identifies that Maria made a painting but omits the required details that it was a painting of a castle on a hill.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly identifies that Maria made a painting but omits the required details that it was a painting of a castle on a hill."}
+{"instance_id": "locomo_conv-50_q086", "cell": "full-context", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 4438, "prompt_tokens": 479, "completion_tokens": 107, "raw_text": "```json\n{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer 'skiing' matches the ground-truth answer 'Skiing' exactly, differing only in capitalization.\"\n}\n```", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer 'skiing' matches the ground-truth answer 'Skiing' exactly, differing only in capitalization."}
+{"instance_id": "locomo_conv-26_q059", "cell": "full-context", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 8270, "prompt_tokens": 564, "completion_tokens": 220, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F1\",\n  \"rationale\": \"The model explicitly states it does not know the answer rather than providing the ground-truth assessment that Caroline is somewhat, but not extremely religious.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F1", "parsed_rationale": "The model explicitly states it does not know the answer rather than providing the ground-truth assessment that Caroline is somewhat, but not extremely religious."}
+{"instance_id": "locomo_conv-44_q000", "cell": "full-context", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 6452, "prompt_tokens": 443, "completion_tokens": 188, "raw_text": "```json\n{\n  \"verdict\": \"correct\",\n  \"failure_mode\": null,\n  \"rationale\": \"The model's answer of 2020 exactly matches the ground-truth answer derived from the context (2023 minus 3 years).\"\n}\n```", "parsed_verdict": "correct", "parsed_failure_mode": null, "parsed_rationale": "The model's answer of 2020 exactly matches the ground-truth answer derived from the context (2023 minus 3 years)."}
+{"instance_id": "locomo_conv-41_q080", "cell": "full-context", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 7873, "prompt_tokens": 492, "completion_tokens": 339, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly identifies that Maria made a painting but misses the required details that it was a painting of a castle on a hill.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly identifies that Maria made a painting but misses the required details that it was a painting of a castle on a hill."}
+{"instance_id": "locomo_conv-50_q086", "cell": "retrieval", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 7401, "prompt_tokens": 479, "completion_tokens": 193, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F1\",\n  \"rationale\": \"The model claims the information is not mentioned, explicitly abstaining from answering, even though the context clearly indicates skiing.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F1", "parsed_rationale": "The model claims the information is not mentioned, explicitly abstaining from answering, even though the context clearly indicates skiing."}
+{"instance_id": "locomo_conv-26_q059", "cell": "retrieval", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 16196, "prompt_tokens": 564, "completion_tokens": 430, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F3\",\n  \"rationale\": \"The model states 'No', which contradicts the ground truth that she is 'somewhat, but not extremely religious', thus getting the derived fact wrong.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F3", "parsed_rationale": "The model states 'No', which contradicts the ground truth that she is 'somewhat, but not extremely religious', thus getting the derived fact wrong."}
+{"instance_id": "locomo_conv-44_q000", "cell": "retrieval", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 6454, "prompt_tokens": 443, "completion_tokens": 205, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F1\",\n  \"rationale\": \"The model explicitly states the information is not mentioned, failing to derive the year 2020 from the context (2023 minus 3 years).\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F1", "parsed_rationale": "The model explicitly states the information is not mentioned, failing to derive the year 2020 from the context (2023 minus 3 years)."}
+{"instance_id": "locomo_conv-41_q080", "cell": "retrieval", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 11221, "prompt_tokens": 492, "completion_tokens": 341, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly states that Maria made a painting but misses the required details that it was a painting of a castle on a hill.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly states that Maria made a painting but misses the required details that it was a painting of a castle on a hill."}
+{"instance_id": "locomo_conv-50_q086", "cell": "agentic", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 9421, "prompt_tokens": 489, "completion_tokens": 242, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F1\",\n  \"rationale\": \"The model failed to provide an answer, instead returning a system message indicating it reached the maximum tool turns, which constitutes an abstention.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F1", "parsed_rationale": "The model failed to provide an answer, instead returning a system message indicating it reached the maximum tool turns, which constitutes an abstention."}
+{"instance_id": "locomo_conv-26_q059", "cell": "agentic", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 19790, "prompt_tokens": 597, "completion_tokens": 510, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F4\",\n  \"rationale\": \"The model's answer relies on fabricated details not present in the context, such as the necklace from Sweden, and mischaracterizes the stained glass window as a painting.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F4", "parsed_rationale": "The model's answer relies on fabricated details not present in the context, such as the necklace from Sweden, and mischaracterizes the stained glass window as a painting."}
+{"instance_id": "locomo_conv-44_q000", "cell": "agentic", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 17473, "prompt_tokens": 453, "completion_tokens": 721, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F1\",\n  \"rationale\": \"The model failed to provide an answer to the question, instead returning a system message indicating it reached the maximum tool turns.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F1", "parsed_rationale": "The model failed to provide an answer to the question, instead returning a system message indicating it reached the maximum tool turns."}
+{"instance_id": "locomo_conv-41_q080", "cell": "agentic", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 8686, "prompt_tokens": 492, "completion_tokens": 339, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly identifies that Maria made a painting but misses the required details that it was a painting of a castle on a hill.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly identifies that Maria made a painting but misses the required details that it was a painting of a castle on a hill."}
--- a/benchmarks/probes/judge-swap-validation/zhipu-split-responses.jsonl
+++ b/benchmarks/probes/judge-swap-validation/zhipu-split-responses.jsonl
@@ -0,0 +1,7 @@
+{"instance_id": "locomo_conv-43_q155", "cell": "agentic", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 14784, "prompt_tokens": 453, "completion_tokens": 556, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly identifies that John felt great but misses the core ground-truth fact that jogging without pain was a 'huge success'.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly identifies that John felt great but misses the core ground-truth fact that jogging without pain was a 'huge success'.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-50_q015", "cell": "agentic", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 26449, "prompt_tokens": 553, "completion_tokens": 1024, "raw_text": "The user wants to evaluate if the model's answer \"to relax and recharge\" is correct against the ground-truth answer \"because it relaxes and calms him\".\n\nStep 1: Determine if the model's answer is correct.\nGround truth: \"because it relaxes and calms him\"\nModel's answer: \"to relax and recharge\"\n\n\"relax\" matches \"relaxes\".\n\"recharge\" vs \"calms him\". Does \"recharge\" mean the same as \"calms him\"? Not exactly. \"Calms\" implies finding peace and tranquility, while \"recharge\" implies regaining energy. However, in the context of visiting parks to relax, \"recharge\" is a very common synonym/associated concept. Let's look at the context: \"so calming\", \"to relax - it's so peaceful being surrounded by nature\". \"Recharge\" is an extra detail that is a reasonable inference from relaxing and being peaceful, but is it factually incorrect? The instructions say \"Extra detail is acceptable ONLY if it is factually correct.\" Is \"recharge\" factually correct based on the context? It's a plausible inference, but the ground truth specifically says \"calms him\". Wait, \"recharge\" is not explicitly in the text, but is it an incorrect claim? It's a very minor difference. Let's consider if \"recharge\" is an acceptable alternative formulation for \"calms\". Often, people say \"relax and recharge\" as a set phrase. But \"calms\" is more about reducing stress, \"recharge\" is about gaining energy.\nLet's check the failure modes if incorrect:\nF4: relies on claims that do NOT appear in the ground-truth context (fabrication). \"recharge\" does not appear in the context.\nF2: correctly states SOME required facts but misses others, without stating incorrect facts. \"relax\" is stated, \"calms\" is missed. Is \"recharge\" an incorrect fact? It's an addition.\n\nWait, is \"recharge\" considered an incorrect claim? If I say \"I visit parks to relax and recharge\", it's not contradicted by the text, but it's an addition. The ground truth says \"relaxes and calms him\". The model says \"to relax and recharge\". It captures \"relax\" but replaces \"calms\" with \"recharge\".\nIs this a PARTIAL (F2) because it missed \"calms\" and added \"recharge\"? Or is it correct because \"recharge\" is an acceptable synonym/extra detail?\nActually, \"calm\" and \"recharge\" have slightly different meanings. Calm is about peace, recharge is about energy. The context says \"so calming\" and \"so peaceful\". Recharging is not explicitly mentioned.\nIf we consider \"recharge\" as an extra detail that is not factually supported (fabrication), it might be F4. But usually, \"relax and recharge\" is seen as a minor paraphrase. Let's be strict. The ground truth is \"because it relaxes and calms him\". The model missed \"calms\" and added \"recharge\".\nIf it's incorrect, it's F2 (PARTIAL) because it correctly states \"relax\" but misses \"calms\", and \"recharge\" is an extra detail. Wait, if \"recharge\" is an incorrect claim, it's not F2. F2 is \"without stating any incorrect facts\". Is \"recharge\" an incorrect fact? It's not in the text.\nLet's reconsider if it's \"correct\". \"Minor phrasing differences, synonyms, or alternative but equivalent formulations are acceptable.\" Is \"recharge\" an alternative but equivalent formulation for \"calms\"? Not really, calming is about reducing arousal, recharging is about restoring resources. However, in casual conversation, they are often grouped.\nLet's look at the instruction: \"Extra detail is acceptable ONLY if it is factually correct.\" Is it factually correct that he visits parks to recharge? The text doesn't say that. It says he visits to relax and because it's calming/peaceful. So \"recharge\" is an unsupported addition.\nIf it's an unsupported addition, it's a fabrication (F4) or just incorrect (F3). But it also misses \"calms\".\nActually, many evaluators would mark \"to relax and recharge\" as correct because it captures the gist. But strictly speaking, \"calms\" is replaced by \"recharge\".\nLet's evaluate if it's F2 (PARTIAL). It states \"relax\" (correct), misses \"calms\", and adds \"recharge\". If \"recharge\" is considered an incorrect fact, it fails the F2 condition (\"without stating any incorrect facts\").\nIs \"recharge\" an incorrect fact? It's an inference. A reasonable inference, but not explicitly stated.\nWhat if \"recharge\" is just a synonym for \"re-energize", "parsed_verdict": null, "parsed_failure_mode": null, "parsed_rationale": null, "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-43_q155", "cell": "full-context", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 22581, "prompt_tokens": 453, "completion_tokens": 868, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly identifies that John felt 'great' but misses the core ground-truth fact that jogging without pain was a 'huge success'.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly identifies that John felt 'great' but misses the core ground-truth fact that jogging without pain was a 'huge success'.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-50_q015", "cell": "full-context", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 22543, "prompt_tokens": 551, "completion_tokens": 737, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model's answer captures the 'relax' part but misses the 'calms him' part of the ground-truth answer.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model's answer captures the 'relax' part but misses the 'calms him' part of the ground-truth answer.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-43_q155", "cell": "oracle-context", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 15651, "prompt_tokens": 453, "completion_tokens": 569, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model captures the positive sentiment but misses the specific required fact from the ground truth that it was 'a huge success'.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model captures the positive sentiment but misses the specific required fact from the ground truth that it was 'a huge success'.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-47_q092", "cell": "oracle-context", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 6011, "prompt_tokens": 457, "completion_tokens": 234, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model correctly identifies that John created an application but misses the required details that it is a computer application on smartphones.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model correctly identifies that John created an application but misses the required details that it is a computer application on smartphones.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
+{"instance_id": "locomo_conv-50_q015", "cell": "oracle-context", "provider": "zhipu", "model_id": "glm-5.1", "routing": "direct", "http_status": 200, "error": null, "retries": 0, "latency_ms": 20114, "prompt_tokens": 551, "completion_tokens": 902, "raw_text": "```json\n{\n  \"verdict\": \"incorrect\",\n  \"failure_mode\": \"F2\",\n  \"rationale\": \"The model captures the fact that it relaxes him, but misses the additional required fact from the ground truth that it also calms him.\"\n}\n```", "parsed_verdict": "incorrect", "parsed_failure_mode": "F2", "parsed_rationale": "The model captures the fact that it relaxes him, but misses the additional required fact from the ground truth that it also calms him.", "opus_verdict_ref": "correct", "gpt_verdict_ref": "incorrect"}
--- a/benchmarks/probes/vertex-batch-eligibility/eligibility-memo.md
+++ b/benchmarks/probes/vertex-batch-eligibility/eligibility-memo.md
@@ -0,0 +1,32 @@
+# Vertex AI Batch Eligibility Probe — gemini-3.1-pro-preview
+
+**Date:** 2026-04-24 · **Target:** PM-RATIFY-VERTEX-BATCH-ELIGIBILITY gate · **Budget actual:** <$0.02 (control job only).
+
+## Verdict: **INFEASIBLE**
+
+`publishers/google/models/gemini-3.1-pro-preview` exists in the Vertex
+publisher catalog (REST `GET /v1/publishers/google/models/gemini-3.1-pro-
+preview` with `X-Goog-User-Project: gen-lang-client-0674908699` → **HTTP
+200**) but is **NOT** registered for Batch Prediction. SDK
+`vertexai.batch_prediction.BatchPredictionJob.submit(source_model=…)`
+returns `NotFound: 404 The PublisherModel gemini-3.1-pro-preview does not
+exist.` on both bare name and fully-qualified forms.
+
+## Control proof (plumbing OK)
+
+Identical SDK path with `source_model="gemini-2.5-flash"` submitted
+successfully: `projects/565729498549/locations/us-central1/
+batchPredictionJobs/7779966002640453632` (state=PENDING). Vertex Batch
+maintains a batch-eligibility registry distinct from the publisher
+catalog; Gemini 3.1 Pro Preview is absent from the former.
+
+## SDK versions (audit)
+
+`google-cloud-aiplatform 1.148.1`, `google-cloud-storage 3.10.1`,
+Python 3.11.9.
+
+## Consequence
+
+Branch A (§1.3e fallback) closed. Stage 3 re-kick remains gated on
+Google quota ticket. Re-probe when Google adds 3.1 Pro Preview to batch
+registry; monitor Vertex release notes.
--- a/benchmarks/probes/vertex-batch-eligibility/job-state-trace.log
+++ b/benchmarks/probes/vertex-batch-eligibility/job-state-trace.log
@@ -0,0 +1,23 @@
+2026-04-24T11:52:53.225915+00:00 [probe] START — PROJECT=gen-lang-client-0674908699 LOCATION=us-central1 MODEL=publishers/google/models/gemini-3.1-pro-preview
+2026-04-24T11:53:12.466058+00:00 [probe] sdk versions: aiplatform=1.148.1 storage=3.10.1
+2026-04-24T11:53:12.475509+00:00 [build] wrote 5 requests to D:\Projects\waggle-os\benchmarks\probes\vertex-batch-eligibility\probe-input.jsonl
+2026-04-24T11:53:13.396231+00:00 [upload] FATAL UnicodeEncodeError: 'charmap' codec can't encode character '\u2192' in position 125: character maps to <undefined>
+2026-04-24T11:53:13.396734+00:00 [probe] VERDICT=INCONCLUSIVE reason=gcs_upload_fail
+2026-04-24T11:53:54.790890+00:00 [probe] START — PROJECT=gen-lang-client-0674908699 LOCATION=us-central1 MODEL=publishers/google/models/gemini-3.1-pro-preview
+2026-04-24T11:54:13.959901+00:00 [probe] sdk versions: aiplatform=1.148.1 storage=3.10.1
+2026-04-24T11:54:13.960905+00:00 [build] wrote 5 requests to D:\Projects\waggle-os\benchmarks\probes\vertex-batch-eligibility\probe-input.jsonl
+2026-04-24T11:54:14.922979+00:00 [upload] D:\Projects\waggle-os\benchmarks\probes\vertex-batch-eligibility\probe-input.jsonl ->gs://egzakta-vertex-batch-probe-2026-04/probe-input.jsonl (9801 bytes)
+2026-04-24T11:54:16.176968+00:00 [submit:preview] ERROR NotFound: 404 The PublisherModel gemini-3.1-pro-preview does not exist.
+2026-04-24T11:54:16.179084+00:00 [submit:legacy] ERROR RuntimeError: BatchPredictionJob resource has not been created.
+2026-04-24T11:54:16.180087+00:00 [probe] SUBMISSION FAILED on both preview and legacy APIs
+2026-04-24T11:54:16.181088+00:00 [probe] VERDICT=INFEASIBLE reason=submission_rejected
+2026-04-24T11:57:19.970025+00:00 [probe] START - PROJECT=gen-lang-client-0674908699 LOCATION=us-central1 MODEL=publishers/google/models/gemini-3.1-pro-preview
+2026-04-24T11:57:39.425526+00:00 [probe] sdk versions: aiplatform=1.148.1 storage=3.10.1
+2026-04-24T11:57:39.425526+00:00 [build] wrote 5 requests to D:\Projects\waggle-os\benchmarks\probes\vertex-batch-eligibility\probe-input.jsonl
+2026-04-24T11:57:40.931139+00:00 [upload] D:\Projects\waggle-os\benchmarks\probes\vertex-batch-eligibility\probe-input.jsonl ->gs://egzakta-vertex-batch-probe-2026-04/probe-input.jsonl (9801 bytes)
+2026-04-24T11:57:42.065981+00:00 [submit:preview:gemini-3.1-pro-preview] ERROR NotFound: 404 The PublisherModel gemini-3.1-pro-preview does not exist.
+2026-04-24T11:57:42.844454+00:00 [submit:preview:publishers/google/models/gemini-3.1-pro-preview] ERROR NotFound: 404 The PublisherModel gemini-3.1-pro-preview does not exist.
+2026-04-24T11:57:42.845455+00:00 [submit:preview] all variants exhausted; trying legacy API
+2026-04-24T11:57:42.846958+00:00 [submit:legacy] ERROR RuntimeError: BatchPredictionJob resource has not been created.
+2026-04-24T11:57:42.848577+00:00 [probe] SUBMISSION FAILED on all preview model variants + legacy API
+2026-04-24T11:57:42.848577+00:00 [probe] VERDICT=INFEASIBLE reason=submission_rejected_all_variants
--- a/benchmarks/probes/vertex-batch-eligibility/probe-input.jsonl
+++ b/benchmarks/probes/vertex-batch-eligibility/probe-input.jsonl
@@ -0,0 +1,5 @@
+{"request": {"contents": [{"role": "user", "parts": [{"text": "You are evaluating whether an LLM's answer is correct against ground truth.\n\n## Question\nWhen did Caroline go to the LGBTQ support group?\n\n## Ground-truth answer\n7 May 2023\n\n## Ground-truth supporting context (excerpt shown to the model)\nSession 1 (1:56 pm on 8 May, 2023):\nCaroline: I went to a LGBTQ support group yesterday and it was so powerful.\n\n## Model's answer\nunknown\n\n## Your task\n\nStep 1: Determine if the model's answer is correct.\n- \"correct\" means the model's answer contains all required facts from ground truth, with no additional incorrect claims.\n- Minor phrasing differences, synonyms, or alternative but equivalent formulations are acceptable.\n- Extra detail is acceptable ONLY if it is factually correct.\n\nStep 2: If incorrect, assign exactly one failure mode using this decision tree:\n\n1. Does the model explicitly refuse or say it does not know? ->F1 (ABSTAIN)\n2. Does the model answer a DIFFERENT question than was asked (coherent but off-topic)? ->F5 (OFF-TOPIC)\n3. Does the model rely on entities, names, dates, or claims that do NOT appear in the ground-truth context (fabrication)? ->F4 (HALLUCINATED)\n4. Does the model correctly state SOME required facts but miss others, without stating any incorrect facts? ->F2 (PARTIAL)\n5. Otherwise (model states facts derived from the context but gets them wrong): ->F3 (INCORRECT)\n\nStep 3: Return JSON only, no prose, in this exact schema:\n\n{\n  \"verdict\": \"correct\" | \"incorrect\",\n  \"failure_mode\": null | \"F1\" | \"F2\" | \"F3\" | \"F4\" | \"F5\",\n  \"rationale\": \"one sentence explaining the verdict\"\n}\n\nIf verdict is \"correct\", failure_mode MUST be null.\nIf verdict is \"incorrect\", failure_mode MUST be one of F1-F5."}]}], "generationConfig": {"temperature": 0.0, "maxOutputTokens": 512}}}
+{"request": {"contents": [{"role": "user", "parts": [{"text": "You are evaluating whether an LLM's answer is correct against ground truth.\n\n## Question\nWhen did Melanie paint a sunrise?\n\n## Ground-truth answer\n2022\n\n## Ground-truth supporting context (excerpt shown to the model)\nSession 1 (1:56 pm on 8 May, 2023):\nMelanie: You'd be a great counselor! Your empathy and understanding will really help the people you work with. By the way, take a look at this. [image: a photo of a painting of a sunset over a lake]\n\n## Model's answer\nunknown\n\n## Your task\n\nStep 1: Determine if the model's answer is correct.\n- \"correct\" means the model's answer contains all required facts from ground truth, with no additional incorrect claims.\n- Minor phrasing differences, synonyms, or alternative but equivalent formulations are acceptable.\n- Extra detail is acceptable ONLY if it is factually correct.\n\nStep 2: If incorrect, assign exactly one failure mode using this decision tree:\n\n1. Does the model explicitly refuse or say it does not know? ->F1 (ABSTAIN)\n2. Does the model answer a DIFFERENT question than was asked (coherent but off-topic)? ->F5 (OFF-TOPIC)\n3. Does the model rely on entities, names, dates, or claims that do NOT appear in the ground-truth context (fabrication)? ->F4 (HALLUCINATED)\n4. Does the model correctly state SOME required facts but miss others, without stating any incorrect facts? ->F2 (PARTIAL)\n5. Otherwise (model states facts derived from the context but gets them wrong): ->F3 (INCORRECT)\n\nStep 3: Return JSON only, no prose, in this exact schema:\n\n{\n  \"verdict\": \"correct\" | \"incorrect\",\n  \"failure_mode\": null | \"F1\" | \"F2\" | \"F3\" | \"F4\" | \"F5\",\n  \"rationale\": \"one sentence explaining the verdict\"\n}\n\nIf verdict is \"correct\", failure_mode MUST be null.\nIf verdict is \"incorrect\", failure_mode MUST be one of F1-F5."}]}], "generationConfig": {"temperature": 0.0, "maxOutputTokens": 512}}}
+{"request": {"contents": [{"role": "user", "parts": [{"text": "You are evaluating whether an LLM's answer is correct against ground truth.\n\n## Question\nWhat fields would Caroline be likely to pursue in her educaton?\n\n## Ground-truth answer\nPsychology, counseling certification\n\n## Ground-truth supporting context (excerpt shown to the model)\nSession 1 (1:56 pm on 8 May, 2023):\nCaroline: Gonna continue my edu and check out career options, which is pretty exciting!\nCaroline: I'm keen on counseling or working in mental health - I'd love to support those with similar issues.\n\n## Model's answer\nunknown\n\n## Your task\n\nStep 1: Determine if the model's answer is correct.\n- \"correct\" means the model's answer contains all required facts from ground truth, with no additional incorrect claims.\n- Minor phrasing differences, synonyms, or alternative but equivalent formulations are acceptable.\n- Extra detail is acceptable ONLY if it is factually correct.\n\nStep 2: If incorrect, assign exactly one failure mode using this decision tree:\n\n1. Does the model explicitly refuse or say it does not know? ->F1 (ABSTAIN)\n2. Does the model answer a DIFFERENT question than was asked (coherent but off-topic)? ->F5 (OFF-TOPIC)\n3. Does the model rely on entities, names, dates, or claims that do NOT appear in the ground-truth context (fabrication)? ->F4 (HALLUCINATED)\n4. Does the model correctly state SOME required facts but miss others, without stating any incorrect facts? ->F2 (PARTIAL)\n5. Otherwise (model states facts derived from the context but gets them wrong): ->F3 (INCORRECT)\n\nStep 3: Return JSON only, no prose, in this exact schema:\n\n{\n  \"verdict\": \"correct\" | \"incorrect\",\n  \"failure_mode\": null | \"F1\" | \"F2\" | \"F3\" | \"F4\" | \"F5\",\n  \"rationale\": \"one sentence explaining the verdict\"\n}\n\nIf verdict is \"correct\", failure_mode MUST be null.\nIf verdict is \"incorrect\", failure_mode MUST be one of F1-F5."}]}], "generationConfig": {"temperature": 0.0, "maxOutputTokens": 512}}}
+{"request": {"contents": [{"role": "user", "parts": [{"text": "You are evaluating whether an LLM's answer is correct against ground truth.\n\n## Question\nWhat did Caroline research?\n\n## Ground-truth answer\nAdoption agencies\n\n## Ground-truth supporting context (excerpt shown to the model)\nSession 2 (1:14 pm on 25 May, 2023):\nCaroline: Researching adoption agencies — it's been a dream to have a family and give a loving home to kids who need it.\n\n## Model's answer\nunknown\n\n## Your task\n\nStep 1: Determine if the model's answer is correct.\n- \"correct\" means the model's answer contains all required facts from ground truth, with no additional incorrect claims.\n- Minor phrasing differences, synonyms, or alternative but equivalent formulations are acceptable.\n- Extra detail is acceptable ONLY if it is factually correct.\n\nStep 2: If incorrect, assign exactly one failure mode using this decision tree:\n\n1. Does the model explicitly refuse or say it does not know? ->F1 (ABSTAIN)\n2. Does the model answer a DIFFERENT question than was asked (coherent but off-topic)? ->F5 (OFF-TOPIC)\n3. Does the model rely on entities, names, dates, or claims that do NOT appear in the ground-truth context (fabrication)? ->F4 (HALLUCINATED)\n4. Does the model correctly state SOME required facts but miss others, without stating any incorrect facts? ->F2 (PARTIAL)\n5. Otherwise (model states facts derived from the context but gets them wrong): ->F3 (INCORRECT)\n\nStep 3: Return JSON only, no prose, in this exact schema:\n\n{\n  \"verdict\": \"correct\" | \"incorrect\",\n  \"failure_mode\": null | \"F1\" | \"F2\" | \"F3\" | \"F4\" | \"F5\",\n  \"rationale\": \"one sentence explaining the verdict\"\n}\n\nIf verdict is \"correct\", failure_mode MUST be null.\nIf verdict is \"incorrect\", failure_mode MUST be one of F1-F5."}]}], "generationConfig": {"temperature": 0.0, "maxOutputTokens": 512}}}
+{"request": {"contents": [{"role": "user", "parts": [{"text": "You are evaluating whether an LLM's answer is correct against ground truth.\n\n## Question\nWhat is Caroline's identity?\n\n## Ground-truth answer\nTransgender woman\n\n## Ground-truth supporting context (excerpt shown to the model)\nSession 1 (1:56 pm on 8 May, 2023):\nCaroline: The transgender stories were so inspiring! I was so happy and thankful for all the support. [image: a photo of a dog walking past a wall with a painting of a woman]\n\n## Model's answer\nunknown\n\n## Your task\n\nStep 1: Determine if the model's answer is correct.\n- \"correct\" means the model's answer contains all required facts from ground truth, with no additional incorrect claims.\n- Minor phrasing differences, synonyms, or alternative but equivalent formulations are acceptable.\n- Extra detail is acceptable ONLY if it is factually correct.\n\nStep 2: If incorrect, assign exactly one failure mode using this decision tree:\n\n1. Does the model explicitly refuse or say it does not know? ->F1 (ABSTAIN)\n2. Does the model answer a DIFFERENT question than was asked (coherent but off-topic)? ->F5 (OFF-TOPIC)\n3. Does the model rely on entities, names, dates, or claims that do NOT appear in the ground-truth context (fabrication)? ->F4 (HALLUCINATED)\n4. Does the model correctly state SOME required facts but miss others, without stating any incorrect facts? ->F2 (PARTIAL)\n5. Otherwise (model states facts derived from the context but gets them wrong): ->F3 (INCORRECT)\n\nStep 3: Return JSON only, no prose, in this exact schema:\n\n{\n  \"verdict\": \"correct\" | \"incorrect\",\n  \"failure_mode\": null | \"F1\" | \"F2\" | \"F3\" | \"F4\" | \"F5\",\n  \"rationale\": \"one sentence explaining the verdict\"\n}\n\nIf verdict is \"correct\", failure_mode MUST be null.\nIf verdict is \"incorrect\", failure_mode MUST be one of F1-F5."}]}], "generationConfig": {"temperature": 0.0, "maxOutputTokens": 512}}}
--- a/benchmarks/probes/vertex-batch-eligibility/probe-script.py
+++ b/benchmarks/probes/vertex-batch-eligibility/probe-script.py
@@ -0,0 +1,378 @@
+"""
+Vertex AI Batch Prediction Eligibility Probe - gemini-3.1-pro-preview
+=====================================================================
+
+§1.3f per PM-RATIFY-V5-RPD fallback Branch A (2026-04-24).
+
+Empirical question: does `publishers/google/models/gemini-3.1-pro-preview`
+accept Vertex Batch Prediction? If ELIGIBLE, unlocks an async path that
+sidesteps the 25 RPM online per-model cap (§1.3 probe root cause) and
+the current 250 RPD ceiling.
+
+Scope discipline:
+  - No §11 frozen path touched (no runner, no judge-runner, no litellm-
+    config.yaml, no failure-mode-judge.ts).
+  - No benchmark adapter. Self-contained standalone script.
+  - All artefacts emitted to `benchmarks/probes/vertex-batch-eligibility/`.
+
+Execution:
+  1. Build 5-instance Vertex Batch JSONL from the canonical LoCoMo fixture.
+  2. Upload to GCS.
+  3. Submit BatchPredictionJob via the newer Gemini `submit()` surface
+     (vertexai.batch_prediction), with fallback to the legacy
+     `aiplatform.BatchPredictionJob.create()` if submit is unavailable
+     or rejects the preview model.
+  4. Poll up to `initial_poll_seconds` (default 450s = 7.5 min, within
+     Bash 10-min timeout). On INCOMPLETE, emit resume-ready job name for
+     a follow-up invocation with --resume <job_name>.
+
+Usage:
+  python probe-script.py                 # fresh submit + poll
+  python probe-script.py --resume <jrn>  # poll-only resume of running job
+
+Judge prompt: verbatim reproduction of
+`packages/server/src/benchmarks/judge/failure-mode-judge.ts:93-140`
+(buildJudgePrompt). Identical to §1.3 + §1.3c probe payload so any
+eligibility difference is routing, not prompt content.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+import traceback
+from datetime import datetime, timezone
+from pathlib import Path
+
+# ── Bound parameters (all from PM direktiva; no placeholders) ─────────────
+
+PROJECT_ID = "gen-lang-client-0674908699"
+LOCATION = "us-central1"
+GCS_BUCKET = "egzakta-vertex-batch-probe-2026-04"
+GCS_INPUT = f"gs://{GCS_BUCKET}/probe-input.jsonl"
+GCS_OUTPUT_PREFIX = f"gs://{GCS_BUCKET}/probe-output/"
+MODEL_ID = "publishers/google/models/gemini-3.1-pro-preview"
+JOB_DISPLAY_NAME = "locomo-mini-vertex-batch-eligibility-probe-2026-04-24"
+
+FIXTURE_PATH = Path("D:/Projects/waggle-os/benchmarks/data/locomo/locomo-1540.jsonl")
+PROBE_DIR = Path("D:/Projects/waggle-os/benchmarks/probes/vertex-batch-eligibility")
+LOG_PATH = PROBE_DIR / "job-state-trace.log"
+INPUT_PATH = PROBE_DIR / "probe-input.jsonl"
+OUTPUT_PATH = PROBE_DIR / "job-output.jsonl"
+
+TERMINAL_STATES = {
+    "JobState.JOB_STATE_SUCCEEDED",
+    "JobState.JOB_STATE_FAILED",
+    "JobState.JOB_STATE_CANCELLED",
+    "JobState.JOB_STATE_EXPIRED",
+}
+
+# Verbatim from failure-mode-judge.ts:93-140 (buildJudgePrompt).
+JUDGE_PROMPT_TEMPLATE = "\n".join([
+    "You are evaluating whether an LLM's answer is correct against ground truth.",
+    "",
+    "## Question",
+    "{question}",
+    "",
+    "## Ground-truth answer",
+    "{ground_truth}",
+    "",
+    "## Ground-truth supporting context (excerpt shown to the model)",
+    "{context}",
+    "",
+    "## Model's answer",
+    "{model_answer}",
+    "",
+    "## Your task",
+    "",
+    "Step 1: Determine if the model's answer is correct.",
+    "- \"correct\" means the model's answer contains all required facts from ground truth, with no additional incorrect claims.",
+    "- Minor phrasing differences, synonyms, or alternative but equivalent formulations are acceptable.",
+    "- Extra detail is acceptable ONLY if it is factually correct.",
+    "",
+    "Step 2: If incorrect, assign exactly one failure mode using this decision tree:",
+    "",
+    "1. Does the model explicitly refuse or say it does not know? ->F1 (ABSTAIN)",
+    "2. Does the model answer a DIFFERENT question than was asked (coherent but off-topic)? ->F5 (OFF-TOPIC)",
+    "3. Does the model rely on entities, names, dates, or claims that do NOT appear in the ground-truth context (fabrication)? ->F4 (HALLUCINATED)",
+    "4. Does the model correctly state SOME required facts but miss others, without stating any incorrect facts? ->F2 (PARTIAL)",
+    "5. Otherwise (model states facts derived from the context but gets them wrong): ->F3 (INCORRECT)",
+    "",
+    "Step 3: Return JSON only, no prose, in this exact schema:",
+    "",
+    "{{",
+    "  \"verdict\": \"correct\" | \"incorrect\",",
+    "  \"failure_mode\": null | \"F1\" | \"F2\" | \"F3\" | \"F4\" | \"F5\",",
+    "  \"rationale\": \"one sentence explaining the verdict\"",
+    "}}",
+    "",
+    "If verdict is \"correct\", failure_mode MUST be null.",
+    "If verdict is \"incorrect\", failure_mode MUST be one of F1-F5.",
+])
+
+
+def ts() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+# Windows cp1252 stdout chokes on Unicode arrows; reconfigure once at import.
+try:
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+    sys.stderr.reconfigure(encoding="utf-8", errors="replace")
+except Exception:
+    pass
+
+
+def logline(text: str) -> None:
+    line = f"{ts()} {text}"
+    try:
+        print(line, flush=True)
+    except UnicodeEncodeError:
+        print(line.encode("ascii", errors="replace").decode("ascii"), flush=True)
+    with LOG_PATH.open("a", encoding="utf-8") as f:
+        f.write(line + "\n")
+
+
+def build_input_jsonl() -> int:
+    PROBE_DIR.mkdir(parents=True, exist_ok=True)
+    count = 0
+    with FIXTURE_PATH.open("r", encoding="utf-8") as f, INPUT_PATH.open("w", encoding="utf-8") as g:
+        for line in f:
+            if count >= 5:
+                break
+            line = line.strip()
+            if not line:
+                continue
+            rec = json.loads(line)
+            prompt = JUDGE_PROMPT_TEMPLATE.format(
+                question=rec["question"],
+                ground_truth=rec.get("gold_answer", ""),
+                context=rec.get("context", ""),
+                model_answer="unknown",
+            )
+            # Vertex Batch JSONL request shape for Gemini models.
+            req = {
+                "request": {
+                    "contents": [
+                        {"role": "user", "parts": [{"text": prompt}]},
+                    ],
+                    "generationConfig": {"temperature": 0.0, "maxOutputTokens": 512},
+                }
+            }
+            g.write(json.dumps(req, ensure_ascii=False) + "\n")
+            count += 1
+    return count
+
+
+def upload_to_gcs(local_path: Path, gcs_uri: str) -> None:
+    from google.cloud import storage
+    assert gcs_uri.startswith("gs://")
+    rest = gcs_uri[len("gs://"):]
+    bucket_name, _, blob_name = rest.partition("/")
+    client = storage.Client(project=PROJECT_ID)
+    bucket = client.bucket(bucket_name)
+    blob = bucket.blob(blob_name)
+    blob.upload_from_filename(str(local_path))
+    logline(f"[upload] {local_path} ->{gcs_uri} ({local_path.stat().st_size} bytes)")
+
+
+def download_first_output_jsonl(gcs_prefix: str, local_path: Path) -> int:
+    from google.cloud import storage
+    assert gcs_prefix.startswith("gs://")
+    rest = gcs_prefix[len("gs://"):]
+    bucket_name, _, prefix = rest.partition("/")
+    client = storage.Client(project=PROJECT_ID)
+    bucket = client.bucket(bucket_name)
+    rows = 0
+    with local_path.open("w", encoding="utf-8") as out:
+        for blob in client.list_blobs(bucket, prefix=prefix):
+            if blob.name.endswith(".jsonl") or "predictions" in blob.name:
+                data = blob.download_as_text()
+                out.write(data)
+                rows += sum(1 for line in data.splitlines() if line.strip())
+                logline(f"[download] {blob.name} ->{local_path} ({len(data)} bytes)")
+    return rows
+
+
+def submit_via_preview_api(model_variant: str) -> object | None:
+    """Try the Gemini-native BatchPredictionJob.submit() API with a given
+    model identifier form. The SDK may want bare name (`gemini-3.1-pro-
+    preview`) vs fully-qualified (`publishers/google/models/gemini-3.1-pro-
+    preview`); caller loops through variants."""
+    try:
+        import vertexai
+        from vertexai.batch_prediction import BatchPredictionJob
+    except ImportError as e:
+        logline(f"[submit:preview:{model_variant}] ImportError {e}")
+        return None
+    try:
+        vertexai.init(project=PROJECT_ID, location=LOCATION)
+        job = BatchPredictionJob.submit(
+            source_model=model_variant,
+            input_dataset=GCS_INPUT,
+            output_uri_prefix=GCS_OUTPUT_PREFIX,
+            job_display_name=JOB_DISPLAY_NAME,
+        )
+        logline(f"[submit:preview:{model_variant}] OK job_resource_name={job.resource_name}")
+        return job
+    except Exception as e:
+        logline(f"[submit:preview:{model_variant}] ERROR {type(e).__name__}: {str(e)[:500]}")
+        return None
+
+
+def submit_via_legacy_api() -> object | None:
+    """Fall back to the legacy aiplatform.BatchPredictionJob.create() API."""
+    try:
+        from google.cloud import aiplatform
+    except ImportError as e:
+        logline(f"[submit:legacy] ImportError {e}")
+        return None
+    try:
+        aiplatform.init(project=PROJECT_ID, location=LOCATION)
+        job = aiplatform.BatchPredictionJob.create(
+            job_display_name=JOB_DISPLAY_NAME,
+            model_name=MODEL_ID,
+            instances_format="jsonl",
+            gcs_source=GCS_INPUT,
+            gcs_destination_prefix=GCS_OUTPUT_PREFIX,
+            predictions_format="jsonl",
+            sync=False,
+        )
+        logline(f"[submit:legacy] OK - job_resource_name={job.resource_name}")
+        return job
+    except Exception as e:
+        logline(f"[submit:legacy] ERROR {type(e).__name__}: {str(e)[:500]}")
+        return None
+
+
+def poll_until_terminal(job, max_seconds: int, cadence_s: int = 60) -> str:
+    t0 = time.time()
+    last_state = None
+    while time.time() - t0 < max_seconds:
+        try:
+            # Refresh from Vertex.
+            job._sync_gca_resource()
+        except Exception as e:
+            logline(f"[poll] sync error {type(e).__name__}: {str(e)[:200]}")
+        state = str(job.state) if hasattr(job, "state") else "UNKNOWN"
+        if state != last_state:
+            logline(f"[poll] t+{int(time.time()-t0):>4}s state={state}")
+            last_state = state
+        if state in TERMINAL_STATES:
+            return state
+        time.sleep(cadence_s)
+    logline(f"[poll] INCOMPLETE after {max_seconds}s - state={last_state}")
+    return "INCOMPLETE"
+
+
+def resume_job(resource_name: str):
+    """Re-attach to an existing BatchPredictionJob by its resource name."""
+    logline(f"[resume] reattaching to {resource_name}")
+    try:
+        from vertexai.batch_prediction import BatchPredictionJob as PreviewBPJ
+        import vertexai
+        vertexai.init(project=PROJECT_ID, location=LOCATION)
+        return PreviewBPJ(resource_name)
+    except Exception as e:
+        logline(f"[resume:preview] fail {type(e).__name__}: {e}")
+    try:
+        from google.cloud import aiplatform
+        aiplatform.init(project=PROJECT_ID, location=LOCATION)
+        return aiplatform.BatchPredictionJob(resource_name)
+    except Exception as e:
+        logline(f"[resume:legacy] fail {type(e).__name__}: {e}")
+    return None
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--resume", default=None, help="resource_name of an already-submitted BPJ")
+    ap.add_argument("--max-seconds", type=int, default=450, help="max wall-clock in poll loop")
+    args = ap.parse_args()
+
+    logline(f"[probe] START - PROJECT={PROJECT_ID} LOCATION={LOCATION} MODEL={MODEL_ID}")
+
+    # Package version stamp for audit.
+    try:
+        import google.cloud.aiplatform as aip
+        import google.cloud.storage as gcs
+        logline(f"[probe] sdk versions: aiplatform={aip.__version__} storage={gcs.__version__}")
+    except Exception:
+        logline("[probe] sdk version stamp failed")
+
+    if args.resume:
+        job = resume_job(args.resume)
+        if job is None:
+            logline("[probe] resume failed - exit 2")
+            return 2
+    else:
+        # Phase 1: build input.
+        n = build_input_jsonl()
+        logline(f"[build] wrote {n} requests to {INPUT_PATH}")
+
+        # Phase 2: upload.
+        try:
+            upload_to_gcs(INPUT_PATH, GCS_INPUT)
+        except Exception as e:
+            logline(f"[upload] FATAL {type(e).__name__}: {str(e)[:500]}")
+            logline("[probe] VERDICT=INCONCLUSIVE reason=gcs_upload_fail")
+            return 3
+
+        # Phase 3: submit. Try model-ID variants in order (SDK preferences
+        # vary — bare name vs fully-qualified). The Vertex publisher catalog
+        # GET confirmed `gemini-3.1-pro-preview` exists at HTTP 200; if all
+        # variants via the SDK still 404, that's evidence of
+        # "model exists but batch-prediction not enabled for it" -> INFEASIBLE.
+        model_variants = [
+            "gemini-3.1-pro-preview",                         # bare name
+            "publishers/google/models/gemini-3.1-pro-preview", # fully-qualified
+        ]
+        job = None
+        for variant in model_variants:
+            job = submit_via_preview_api(variant)
+            if job is not None:
+                break
+        if job is None:
+            logline("[submit:preview] all variants exhausted; trying legacy API")
+            job = submit_via_legacy_api()
+        if job is None:
+            logline("[probe] SUBMISSION FAILED on all preview model variants + legacy API")
+            logline("[probe] VERDICT=INFEASIBLE reason=submission_rejected_all_variants")
+            return 4
+
+    # Phase 4: poll.
+    final_state = poll_until_terminal(job, max_seconds=args.max_seconds)
+    logline(f"[probe] TERMINAL_STATE={final_state} job_resource_name={job.resource_name}")
+
+    if final_state == "INCOMPLETE":
+        logline(f"[probe] RESUMABLE - re-run with: python probe-script.py --resume {job.resource_name}")
+        logline("[probe] VERDICT=PARTIAL_PENDING wall_clock_exhausted")
+        return 5
+
+    # Phase 5: classify.
+    if final_state == "JobState.JOB_STATE_SUCCEEDED":
+        logline("[probe] downloading output JSONL(s)...")
+        rows = download_first_output_jsonl(GCS_OUTPUT_PREFIX, OUTPUT_PATH)
+        logline(f"[probe] downloaded {rows} output rows ->{OUTPUT_PATH}")
+        logline("[probe] VERDICT=ELIGIBLE")
+        return 0
+    elif final_state == "JobState.JOB_STATE_FAILED":
+        err = getattr(job, "error", None)
+        logline(f"[probe] JOB FAILED - error={err}")
+        logline("[probe] VERDICT=INFEASIBLE reason=job_state_failed")
+        return 6
+    else:
+        logline(f"[probe] VERDICT=INCONCLUSIVE reason=unexpected_state:{final_state}")
+        return 7
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except Exception as e:
+        logline(f"[FATAL] {type(e).__name__}: {str(e)[:500]}")
+        logline(f"[FATAL] traceback: {traceback.format_exc()[:2000]}")
+        logline("[probe] VERDICT=INCONCLUSIVE reason=fatal_exception")
+        sys.exit(99)