moving

2026-09-02 10:10:29 +02:00
commit 0c3e2ead3b
3841 changed files with 970576 additions and 0 deletions
--- a/benchmarks/harness/config/models.json
+++ b/benchmarks/harness/config/models.json
@@ -0,0 +1,154 @@
+{
+  "qwen3.6-35b-a3b": {
+    "id": "qwen3.6-35b-a3b",
+    "displayName": "Qwen3.6-35B-A3B (canonical)",
+    "provider": "alibaba",
+    "litellmModel": "dashscope/qwen3.6-35b-a3b",
+    "pricePerMillionInput": 0.20,
+    "pricePerMillionOutput": 0.80,
+    "contextWindow": 262144,
+    "pinning_surface": "floating_alias",
+    "pinning_surface_carve_out_reason": "DashScope does not expose immutable model snapshots; floating alias mandated by B3 addendum § 5"
+  },
+  "qwen3.6-35b-a3b-stage2": {
+    "id": "qwen3.6-35b-a3b-stage2",
+    "displayName": "Qwen3.6-35B-A3B (Stage 2 LOCKED config — thinking=on, 64K)",
+    "provider": "alibaba",
+    "litellmModel": "qwen3.6-35b-a3b-via-openrouter",
+    "pricePerMillionInput": 0.20,
+    "pricePerMillionOutput": 0.80,
+    "contextWindow": 262144,
+    "stage2Config": {
+      "thinking": true,
+      "maxTokens": 64000,
+      "reasoningShape": "openrouter-unified"
+    },
+    "pinning_surface": "floating_alias",
+    "pinning_surface_carve_out_reason": "OpenRouter bridge to DashScope does not expose immutable model snapshots; floating alias mandated by B3 addendum § 5"
+  },
+  "qwen3.6-35b-a3b-via-openrouter": {
+    "id": "qwen3.6-35b-a3b-via-openrouter",
+    "displayName": "Qwen3.6-35B-A3B (C3 Stage 2 mini subject — alias name matches LiteLLM route, stage2 LOCKED config applied)",
+    "provider": "alibaba",
+    "litellmModel": "qwen3.6-35b-a3b-via-openrouter",
+    "pricePerMillionInput": 0.20,
+    "pricePerMillionOutput": 0.80,
+    "contextWindow": 262144,
+    "stage2Config": {
+      "thinking": true,
+      "maxTokens": 64000,
+      "reasoningShape": "openrouter-unified"
+    },
+    "pinning_surface": "floating_alias",
+    "pinning_surface_carve_out_reason": "Sprint 12 Task 2 C3 Stage 2 mini (2026-04-23): sibling alias of `qwen3.6-35b-a3b-stage2` that exposes the LiteLLM route name as the models.json key — lets the manifest `target_model: qwen3.6-35b-a3b-via-openrouter` match both the pre-registration audit trail AND the CLI --model arg verbatim. Same routing: LiteLLM -> OpenRouter -> Alibaba bridge (slug `openrouter/qwen/qwen3.5-35b-a3b`). OpenRouter bridge to DashScope does not expose immutable model snapshots; floating alias mandated by B3 addendum § 5."
+  },
+  "qwen3.6-35b-a3b-via-dashscope-direct": {
+    "id": "qwen3.6-35b-a3b-via-dashscope-direct",
+    "displayName": "Qwen3.6-35B-A3B (C3 Stage 2 Mini Retry v3 PRIMARY — DashScope-intl direct, TRUE 3.6)",
+    "provider": "alibaba",
+    "litellmModel": "qwen3.6-35b-a3b-via-dashscope-direct",
+    "pricePerMillionInput": 0.20,
+    "pricePerMillionOutput": 0.80,
+    "contextWindow": 262144,
+    "stage2Config": {
+      "thinking": true,
+      "maxTokens": 16000,
+      "reasoningShape": "dashscope-native"
+    },
+    "pinning_surface": "floating_alias",
+    "pinning_surface_carve_out_reason": "Sprint 12 Task 2 C3 Stage 2 Mini Retry v3 (2026-04-23): models.json key for the DashScope-intl direct route (litellm-config.yaml:228 alias `qwen3.6-35b-a3b-via-dashscope-direct` routes to `openai/qwen3.6-35b-a3b` @ https://dashscope-intl.aliyuncs.com/compatible-mode/v1 with DASHSCOPE_API_KEY). Delivers TRUE Qwen 3.6-35B-A3B (not the 3.5 regress of the OpenRouter bridge). thinking=on, max_tokens=16000 per v3 manifest §2.1 (reduced from v2 64000 to avoid tail-latency timeouts). DashScope-intl does not expose immutable model snapshots; floating alias mandated by B3 addendum § 5."
+  },
+  "qwen3.6-35b-a3b-local": {
+    "id": "qwen3.6-35b-a3b-local",
+    "displayName": "Qwen3.6-35B-A3B (local vLLM)",
+    "provider": "local",
+    "litellmModel": "openai/qwen3.6-35b-a3b",
+    "pricePerMillionInput": 0.0,
+    "pricePerMillionOutput": 0.0,
+    "contextWindow": 262144,
+    "pinning_surface": "floating_alias",
+    "pinning_surface_carve_out_reason": "Local vLLM endpoint does not expose immutable model snapshots; floating alias mandated by B3 addendum § 5"
+  },
+  "llama-3.1-8b-instruct": {
+    "id": "llama-3.1-8b-instruct",
+    "displayName": "Llama 3.1 8B Instruct (Week 2 placeholder)",
+    "provider": "local",
+    "litellmModel": "openai/llama-3.1-8b-instruct",
+    "pricePerMillionInput": 0.0,
+    "pricePerMillionOutput": 0.0,
+    "contextWindow": 131072,
+    "pinning_surface": "floating_alias",
+    "pinning_surface_carve_out_reason": "Local vLLM endpoint does not expose immutable model snapshots; floating alias mandated by B3 addendum § 5"
+  },
+  "claude-opus-4-6": {
+    "id": "claude-opus-4-6",
+    "displayName": "Claude Opus 4.6 (Week 2 placeholder, frontier baseline)",
+    "provider": "anthropic",
+    "litellmModel": "claude-opus-4-6",
+    "pricePerMillionInput": 15.0,
+    "pricePerMillionOutput": 75.0,
+    "contextWindow": 200000,
+    "pinning_surface": "anthropic_immutable",
+    "pinning_surface_carve_out_reason": null
+  },
+  "claude-opus-4-7": {
+    "id": "claude-opus-4-7",
+    "displayName": "Claude Opus 4.7 (Sprint 11 Task 2.2 primary judge)",
+    "provider": "anthropic",
+    "litellmModel": "claude-opus-4-7",
+    "pricePerMillionInput": 15.0,
+    "pricePerMillionOutput": 75.0,
+    "contextWindow": 200000,
+    "pinning_surface": "anthropic_immutable",
+    "pinning_surface_carve_out_reason": null,
+    "judge_role": "primary"
+  },
+  "gpt-5.4": {
+    "id": "gpt-5.4",
+    "displayName": "GPT-5.4 (Sprint 11 Task 2.2 primary judge, direct via LiteLLM→OpenAI)",
+    "provider": "openai_via_openrouter",
+    "litellmModel": "gpt-5.4",
+    "pricePerMillionInput": 10.0,
+    "pricePerMillionOutput": 30.0,
+    "contextWindow": 200000,
+    "pinning_surface": "floating_alias",
+    "pinning_surface_carve_out_reason": "Sprint 12 Task 2 C3 Stage 2 mini (2026-04-23): routing pivoted from OpenRouter bridge to direct OpenAI Chat Completions via LiteLLM local alias `gpt-5.4` → `openai/gpt-5.4` (eliminates ~5-15% OpenRouter middleware markup + preserves token-level telemetry for EU AI Act Art. 14 replay). OpenAI does not expose immutable model snapshots for the gpt-5.x family; floating alias mandated by B3 addendum § 5.",
+    "judge_role": "primary"
+  },
+  "gemini-3.1": {
+    "id": "gemini-3.1",
+    "displayName": "Gemini 3.1 Pro Preview (legacy OpenRouter bridge route)",
+    "provider": "google_via_openrouter",
+    "litellmModel": "openrouter/google/gemini-3.1-pro-preview",
+    "pricePerMillionInput": 3.5,
+    "pricePerMillionOutput": 10.5,
+    "contextWindow": 1000000,
+    "pinning_surface": "floating_alias",
+    "pinning_surface_carve_out_reason": "Legacy OpenRouter-bridge route; Sprint 12 Task 2 C3 Stage 2 mini (2026-04-23) pivoted judge routing to direct via LiteLLM local alias `gemini-3.1-pro` — see sibling entry below. This entry kept for backward compat with pre-Sprint-12 artefacts; floating alias mandated by B3 addendum § 5",
+    "judge_role": "primary"
+  },
+  "gemini-3.1-pro": {
+    "id": "gemini-3.1-pro",
+    "displayName": "Gemini 3.1 Pro Preview (Sprint 12 Task 2 primary judge, direct via LiteLLM→Google AI Studio)",
+    "provider": "google_via_openrouter",
+    "litellmModel": "gemini-3.1-pro",
+    "pricePerMillionInput": 3.5,
+    "pricePerMillionOutput": 10.5,
+    "contextWindow": 1000000,
+    "pinning_surface": "floating_alias",
+    "pinning_surface_carve_out_reason": "Sprint 12 Task 2 C3 Stage 2 mini (2026-04-23): direct-provider routing via LiteLLM local alias `gemini-3.1-pro` → `gemini/gemini-3.1-pro-preview` (Google AI Studio). Google ships the 3.1 Pro generation as `-preview` suffix only — no stable alias exists as of 2026-04-23 01:47 UTC (GET openrouter.ai/v1/models). Direct routing eliminates OpenRouter middleware markup + preserves native reasoning_tokens telemetry. Verify at replay time that `-preview` still resolves to the same underlying checkpoint; Google guarantees preview stability within a release window but not across release cycles. Floating alias mandated by B3 addendum § 5 (no immutable upstream snapshot surface).",
+    "judge_role": "primary"
+  },
+  "grok-4.20": {
+    "id": "grok-4.20",
+    "displayName": "Grok 4.20 (Sprint 11 B2 LOCK quadri-vendor tie-break reserve)",
+    "provider": "xai_via_openrouter",
+    "litellmModel": "openrouter/x-ai/grok-4.20",
+    "pricePerMillionInput": 5.0,
+    "pricePerMillionOutput": 15.0,
+    "contextWindow": 131072,
+    "pinning_surface": "floating_alias",
+    "pinning_surface_carve_out_reason": "xAI does not expose immutable model snapshots through OpenRouter routing layer; floating alias mandated by B3 addendum § 5",
+    "judge_role": "reserve"
+  }
+}