moving

2026-09-02 10:10:29 +02:00
commit 0c3e2ead3b
3841 changed files with 970576 additions and 0 deletions
--- a/benchmarks/calibration/v6-kappa-recal/kappa-v6-compute.py
+++ b/benchmarks/calibration/v6-kappa-recal/kappa-v6-compute.py
@@ -0,0 +1,330 @@
+"""
+Manifest v6 Phase 1 — κ re-calibration computation
+===================================================
+
+Computes three pairwise Cohen's κ on 100-instance sample:
+  κ(Opus, GPT)       — should match v5 historical baseline ~0.74-0.82
+  κ(Opus, MiniMax)   — new measurement
+  κ(GPT, MiniMax)    — new measurement
+
+Conservative trio κ = min of the three.
+
+Also reports:
+  - Raw agreement % per pair
+  - Confusion matrix per pair
+  - Per-cell breakdown (no-context / oracle-context / full-context /
+    retrieval / agentic)
+  - MiniMax operational metrics: parse rate, latency p50/p95, routing
+    errors, token usage
+
+Writes:
+  kappa-v6-analysis.md — detailed matrix + per-cell breakdown
+  _summary-v6-kappa.json — machine-readable for halt ping
+"""
+from __future__ import annotations
+
+import json
+import statistics
+from collections import Counter
+from pathlib import Path
+
+OUT_DIR = Path("D:/Projects/waggle-os/benchmarks/calibration/v6-kappa-recal")
+SAMPLE_PATH = OUT_DIR / "kappa-sample-instances.jsonl"
+RESPONSES_PATH = OUT_DIR / "minimax-kappa-responses.jsonl"
+ANALYSIS_PATH = OUT_DIR / "kappa-v6-analysis.md"
+SUMMARY_JSON = OUT_DIR / "_summary-v6-kappa.json"
+
+
+def load_jsonl(path: Path) -> list[dict]:
+    out = []
+    with path.open("r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                out.append(json.loads(line))
+    return out
+
+
+def cohen_kappa(pairs: list[tuple[str, str]]) -> tuple[float, dict]:
+    """Cohen's κ on 2-class (correct/incorrect) pairs. Returns (κ, detail)."""
+    if not pairs:
+        return (float("nan"), {"n": 0, "agree": 0, "po": 0.0, "pe": 0.0})
+    n = len(pairs)
+    agree = sum(1 for a, b in pairs if a == b)
+    po = agree / n
+    a_counts = {"correct": 0, "incorrect": 0}
+    b_counts = {"correct": 0, "incorrect": 0}
+    for a, b in pairs:
+        a_counts[a] = a_counts.get(a, 0) + 1
+        b_counts[b] = b_counts.get(b, 0) + 1
+    pe = sum(
+        (a_counts.get(v, 0) / n) * (b_counts.get(v, 0) / n)
+        for v in ("correct", "incorrect")
+    )
+    if pe >= 1.0:
+        return (1.0 if po == 1.0 else float("nan"),
+                {"n": n, "agree": agree, "po": po, "pe": pe,
+                 "a_counts": a_counts, "b_counts": b_counts})
+    kappa = (po - pe) / (1.0 - pe)
+    return (kappa, {"n": n, "agree": agree, "po": po, "pe": pe,
+                    "a_counts": a_counts, "b_counts": b_counts})
+
+
+def confusion_matrix(pairs: list[tuple[str, str]]) -> dict:
+    """2x2 confusion (rows = judge A, cols = judge B)."""
+    cm = {"correct_correct": 0, "correct_incorrect": 0,
+          "incorrect_correct": 0, "incorrect_incorrect": 0}
+    for a, b in pairs:
+        key = f"{a}_{b}"
+        cm[key] = cm.get(key, 0) + 1
+    return cm
+
+
+def classify_verdict(trio_kappa: float) -> str:
+    if trio_kappa != trio_kappa:  # NaN
+        return "INCONCLUSIVE"
+    if trio_kappa >= 0.70:
+        return "PASS"
+    if trio_kappa >= 0.60:
+        return "BORDERLINE"
+    return "FAIL"
+
+
+def fmt_k(x: float) -> str:
+    if x != x:
+        return "NaN"
+    return f"{x:.4f}"
+
+
+def pct_str(num: int, denom: int) -> str:
+    if denom == 0:
+        return "—"
+    return f"{num}/{denom} ({num * 100.0 / denom:.1f}%)"
+
+
+def main() -> int:
+    sample = load_jsonl(SAMPLE_PATH)
+    mm = load_jsonl(RESPONSES_PATH)
+
+    # Index MiniMax responses
+    mm_by_key = {(r["instance_id"], r["cell"]): r for r in mm}
+
+    # Build pair lists for 3 pairwise κ
+    pairs_og = []  # (Opus, GPT)
+    pairs_om = []  # (Opus, MiniMax)
+    pairs_gm = []  # (GPT, MiniMax)
+    per_cell = {"no-context": [], "oracle-context": [], "full-context": [],
+                "retrieval": [], "agentic": []}
+
+    mm_parse_ok = 0
+    mm_lat = []
+    mm_retries = 0
+    mm_routing_errors = 0
+    mm_prompt_tok = []
+    mm_comp_tok = []
+
+    for s in sample:
+        op_v = s.get("opus_verdict")
+        gp_v = s.get("gpt_verdict")
+        if op_v not in ("correct", "incorrect") or gp_v not in ("correct", "incorrect"):
+            continue
+        pairs_og.append((op_v, gp_v))
+        per_cell.setdefault(s["cell"], []).append(("og", op_v, gp_v))
+
+        key = (s["instance_id"], s["cell"])
+        m = mm_by_key.get(key)
+        if m is None:
+            continue
+        mm_v = m.get("parsed_verdict")
+        mm_lat.append(m.get("latency_ms") or 0)
+        mm_retries += m.get("retries") or 0
+        if m.get("http_status") != 200:
+            mm_routing_errors += 1
+        if m.get("prompt_tokens"):
+            mm_prompt_tok.append(m["prompt_tokens"])
+        if m.get("completion_tokens"):
+            mm_comp_tok.append(m["completion_tokens"])
+        if mm_v in ("correct", "incorrect"):
+            mm_parse_ok += 1
+            pairs_om.append((op_v, mm_v))
+            pairs_gm.append((gp_v, mm_v))
+            per_cell.setdefault(s["cell"], []).append(("om", op_v, mm_v))
+            per_cell.setdefault(s["cell"], []).append(("gm", gp_v, mm_v))
+
+    k_og, k_og_det = cohen_kappa(pairs_og)
+    k_om, k_om_det = cohen_kappa(pairs_om)
+    k_gm, k_gm_det = cohen_kappa(pairs_gm)
+    cm_og = confusion_matrix(pairs_og)
+    cm_om = confusion_matrix(pairs_om)
+    cm_gm = confusion_matrix(pairs_gm)
+
+    kappas = [k for k in (k_og, k_om, k_gm) if k == k]
+    k_trio = min(kappas) if kappas else float("nan")
+    verdict = classify_verdict(k_trio)
+
+    # Per-cell pairwise
+    per_cell_rows = []
+    cells_order = ["no-context", "oracle-context", "full-context", "retrieval", "agentic"]
+    for cell in cells_order:
+        triples = per_cell.get(cell, [])
+        pog = [(a, b) for t, a, b in triples if t == "og"]
+        pom = [(a, b) for t, a, b in triples if t == "om"]
+        pgm = [(a, b) for t, a, b in triples if t == "gm"]
+        k_c_og, _ = cohen_kappa(pog) if pog else (float("nan"), {})
+        k_c_om, _ = cohen_kappa(pom) if pom else (float("nan"), {})
+        k_c_gm, _ = cohen_kappa(pgm) if pgm else (float("nan"), {})
+        n_cell = len(pog)
+        mm_cell_parsed = len(pom)
+        per_cell_rows.append({
+            "cell": cell,
+            "n": n_cell,
+            "mm_parsed": mm_cell_parsed,
+            "k_og": k_c_og,
+            "k_om": k_c_om,
+            "k_gm": k_c_gm,
+        })
+
+    # Operational metrics
+    lat_p50 = int(statistics.median(mm_lat)) if mm_lat else 0
+    lat_p95 = int(sorted(mm_lat)[max(0, int(len(mm_lat) * 0.95) - 1)]) if mm_lat else 0
+    prompt_tok_total = sum(mm_prompt_tok)
+    comp_tok_total = sum(mm_comp_tok)
+    # OR MiniMax M2.7 pricing: $0.30/M prompt, $1.20/M completion
+    cost_actual = round(
+        (prompt_tok_total / 1_000_000) * 0.30 + (comp_tok_total / 1_000_000) * 1.20, 4
+    )
+
+    # ── Write kappa-v6-analysis.md ─────────────────────────────────────
+
+    lines = []
+    lines.append("# Manifest v6 κ Re-Calibration Analysis")
+    lines.append("")
+    lines.append("**Date:** 2026-04-24  **Parent:** `38a830e` (v6 Phase 1 Commit 2)  **v6 anchor:** `60d061e`")
+    lines.append("")
+    lines.append(f"**Sample:** {len(sample)} instances from `benchmarks/results/locomo-mini-n20-retry-2026-04-24T00-02-12Z.jsonl` (authoritative v5 κ set; zero new Opus/GPT calls).")
+    lines.append("")
+    lines.append(f"**MiniMax verdicts:** {len(mm)} calls via OpenRouter `minimax/minimax-m2.7` (v6 alias: `minimax-m27-via-openrouter`); direct HTTP probe (LiteLLM proxy not in loop — isolates model behavior from middleware).")
+    lines.append("")
+    lines.append(f"**Prompt:** verbatim `failure-mode-judge.ts:245-258` (same as §1.3g / §1.3h / §1.3h-C).")
+    lines.append(f"**Parameters:** `temperature=0.0`, `max_tokens=4096`.")
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+    lines.append("## §1 Three pairwise Cohen's κ")
+    lines.append("")
+    lines.append("| Pair | n | Agree | Raw % | κ |")
+    lines.append("|------|---|-------|-------|-----|")
+    lines.append(f"| Opus vs GPT | {k_og_det['n']} | {k_og_det['agree']} | {k_og_det['po']*100:.2f}% | **{fmt_k(k_og)}** |")
+    lines.append(f"| Opus vs MiniMax | {k_om_det['n']} | {k_om_det['agree']} | {k_om_det['po']*100:.2f}% | **{fmt_k(k_om)}** |")
+    lines.append(f"| GPT vs MiniMax | {k_gm_det['n']} | {k_gm_det['agree']} | {k_gm_det['po']*100:.2f}% | **{fmt_k(k_gm)}** |")
+    lines.append("")
+    lines.append(f"**Conservative trio κ = min = {fmt_k(k_trio)}**")
+    lines.append("")
+    lines.append(f"## §2 Verdict: **{verdict}**")
+    lines.append("")
+    lines.append("Per v6 §5.4 gate criteria:")
+    lines.append("- `κ_trio ≥ 0.70` → PASS, halt with PM-RATIFY-V6-KAPPA")
+    lines.append("- `0.60 ≤ κ_trio < 0.70` → BORDERLINE, halt with PM adjudication")
+    lines.append("- `κ_trio < 0.60` → FAIL, halt with swap-path-re-evaluation")
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+    lines.append("## §3 Confusion matrices")
+    lines.append("")
+    lines.append("### Opus vs GPT")
+    lines.append("")
+    lines.append("| | GPT=correct | GPT=incorrect |")
+    lines.append("|---|---|---|")
+    lines.append(f"| **Opus=correct** | {cm_og.get('correct_correct', 0)} | {cm_og.get('correct_incorrect', 0)} |")
+    lines.append(f"| **Opus=incorrect** | {cm_og.get('incorrect_correct', 0)} | {cm_og.get('incorrect_incorrect', 0)} |")
+    lines.append("")
+    lines.append("### Opus vs MiniMax")
+    lines.append("")
+    lines.append("| | MiniMax=correct | MiniMax=incorrect |")
+    lines.append("|---|---|---|")
+    lines.append(f"| **Opus=correct** | {cm_om.get('correct_correct', 0)} | {cm_om.get('correct_incorrect', 0)} |")
+    lines.append(f"| **Opus=incorrect** | {cm_om.get('incorrect_correct', 0)} | {cm_om.get('incorrect_incorrect', 0)} |")
+    lines.append("")
+    lines.append("### GPT vs MiniMax")
+    lines.append("")
+    lines.append("| | MiniMax=correct | MiniMax=incorrect |")
+    lines.append("|---|---|---|")
+    lines.append(f"| **GPT=correct** | {cm_gm.get('correct_correct', 0)} | {cm_gm.get('correct_incorrect', 0)} |")
+    lines.append(f"| **GPT=incorrect** | {cm_gm.get('incorrect_correct', 0)} | {cm_gm.get('incorrect_incorrect', 0)} |")
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+    lines.append("## §4 Per-cell κ breakdown (n=20 per cell)")
+    lines.append("")
+    lines.append("| Cell | n | MiniMax parsed | κ(Opus,GPT) | κ(Opus,MiniMax) | κ(GPT,MiniMax) |")
+    lines.append("|------|---|-----------------|----------------|-------------------|------------------|")
+    for row in per_cell_rows:
+        lines.append(
+            f"| {row['cell']} | {row['n']} | {row['mm_parsed']} "
+            f"| {fmt_k(row['k_og'])} | {fmt_k(row['k_om'])} | {fmt_k(row['k_gm'])} |"
+        )
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+    lines.append("## §5 MiniMax operational metrics")
+    lines.append("")
+    lines.append(f"- Calls: {len(mm)} total, parsed OK: **{pct_str(mm_parse_ok, len(mm))}**")
+    lines.append(f"- Routing errors (non-200 HTTP): **{mm_routing_errors}/{len(mm)}** ({mm_routing_errors*100/len(mm):.1f}%)")
+    lines.append(f"- Total retries: {mm_retries}")
+    lines.append(f"- Latency p50: **{lat_p50/1000:.1f} s**  |  p95: **{lat_p95/1000:.1f} s**")
+    lines.append(f"- Token usage: prompt = {prompt_tok_total:,}, completion = {comp_tok_total:,}")
+    lines.append(f"- Cost actual (OR MiniMax pricing $0.30/$1.20 per 1M): **~${cost_actual}**")
+    lines.append("")
+    lines.append(f"Per brief §3.5 operational hedge thresholds:")
+    lines.append(f"- parse ≥95/100 target: **{'MET' if mm_parse_ok >= 95 else 'MISS (below target)'}** — actual {mm_parse_ok}/100")
+    lines.append(f"- parse ≥90/100 halt: **{'MET' if mm_parse_ok >= 90 else 'FAIL (halt)'}** — actual {mm_parse_ok}/100")
+    lines.append(f"- latency p50 ≤25s: **{'MET' if lat_p50 <= 25000 else 'MISS'}** — actual {lat_p50/1000:.1f}s")
+    lines.append(f"- OR routing errors <5%: **{'MET' if mm_routing_errors < 5 else 'FLAG'}** — actual {mm_routing_errors/len(mm)*100:.1f}%")
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+    lines.append(f"## §6 Comparison to v5 historical baseline")
+    lines.append("")
+    lines.append(f"v5 κ baseline reference: Fleiss' κ=0.7458 on three-way Opus+GPT+Gemini ensemble.")
+    lines.append(f"v6 κ(Opus, GPT) pairwise: **{fmt_k(k_og)}** — sanity check. If significantly different from v5 baseline range (~0.74-0.82 for a high-agreement pair), investigate.")
+    lines.append(f"v6 conservative trio κ (Opus+GPT+MiniMax): **{fmt_k(k_trio)}**.")
+
+    ANALYSIS_PATH.write_text("\n".join(lines), encoding="utf-8")
+    print(f"Wrote {ANALYSIS_PATH}")
+
+    # Machine-readable summary
+    summary = {
+        "verdict": verdict,
+        "k_opus_gpt": k_og,
+        "k_opus_minimax": k_om,
+        "k_gpt_minimax": k_gm,
+        "k_conservative_trio": k_trio,
+        "minimax_parse_success": mm_parse_ok,
+        "minimax_n_total": len(mm),
+        "minimax_lat_p50_ms": lat_p50,
+        "minimax_lat_p95_ms": lat_p95,
+        "minimax_routing_errors": mm_routing_errors,
+        "minimax_retries_total": mm_retries,
+        "minimax_prompt_tokens_total": prompt_tok_total,
+        "minimax_completion_tokens_total": comp_tok_total,
+        "cost_actual_usd": cost_actual,
+        "per_cell": per_cell_rows,
+        "confusion_opus_gpt": cm_og,
+        "confusion_opus_minimax": cm_om,
+        "confusion_gpt_minimax": cm_gm,
+    }
+    SUMMARY_JSON.write_text(json.dumps(summary, indent=2, default=str), encoding="utf-8")
+    print(f"Wrote {SUMMARY_JSON}")
+    print(f"\nVerdict: {verdict}")
+    print(f"k(Opus, GPT)     = {fmt_k(k_og)}")
+    print(f"k(Opus, MiniMax) = {fmt_k(k_om)}")
+    print(f"k(GPT, MiniMax)  = {fmt_k(k_gm)}")
+    print(f"k_trio (min)     = {fmt_k(k_trio)}")
+    print(f"MiniMax parse    = {mm_parse_ok}/100")
+    print(f"Cost             = ${cost_actual}")
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+    sys.exit(main())