oleg.m/waggle-os
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

moving
Some checks failed
Installer Smoke / installer-smoke (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Oleg Maslov
2026-09-02 10:10:29 +02:00
commit 0c3e2ead3b
3841 changed files with 970576 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
benchmarks/harness/config/datasets.json Normal file
View File

@@ -0,0 +1,32 @@
{
"synthetic": {
"id": "synthetic",
"displayName": "Synthetic scaffold dataset (60 instances, built-in)",
"dataPath": "synthetic/placeholder.jsonl",
"source": "synthetic"
},
"locomo": {
"id": "locomo",
"displayName": "LoCoMo (snap-research/locomo, canonical 1531-instance non-adversarial eval set)",
"dataPath": "locomo/locomo-1540.jsonl",
"source": "external"
},
"longmemeval": {
"id": "longmemeval",
"displayName": "LongMemEval V1 S-variant (xiaowu0162/longmemeval-cleaned, 500 questions, 115K tokens)",
"dataPath": "longmemeval/longmemeval.jsonl",
"source": "external"
},
"beam-128k": {
"id": "beam-128k",
"displayName": "BEAM 128K (mohammadtavakoli78/BEAM, ICLR 2026, 20 conversations ~128K tokens)",
"dataPath": "beam/beam-128K.jsonl",
"source": "external"
},
"beam-1m": {
"id": "beam-1m",
"displayName": "BEAM 1M (mohammadtavakoli78/BEAM, ICLR 2026, 35 conversations ~1M tokens)",
"dataPath": "beam/beam-1M.jsonl",
"source": "external"
}
}

154
benchmarks/harness/config/models.json Normal file
View File

@@ -0,0 +1,154 @@
{
"qwen3.6-35b-a3b": {
"id": "qwen3.6-35b-a3b",
"displayName": "Qwen3.6-35B-A3B (canonical)",
"provider": "alibaba",
"litellmModel": "dashscope/qwen3.6-35b-a3b",
"pricePerMillionInput": 0.20,
"pricePerMillionOutput": 0.80,
"contextWindow": 262144,
"pinning_surface": "floating_alias",
"pinning_surface_carve_out_reason": "DashScope does not expose immutable model snapshots; floating alias mandated by B3 addendum § 5"
},
"qwen3.6-35b-a3b-stage2": {
"id": "qwen3.6-35b-a3b-stage2",
"displayName": "Qwen3.6-35B-A3B (Stage 2 LOCKED config — thinking=on, 64K)",
"provider": "alibaba",
"litellmModel": "qwen3.6-35b-a3b-via-openrouter",
"pricePerMillionInput": 0.20,
"pricePerMillionOutput": 0.80,
"contextWindow": 262144,
"stage2Config": {
"thinking": true,
"maxTokens": 64000,
"reasoningShape": "openrouter-unified"
},
"pinning_surface": "floating_alias",
"pinning_surface_carve_out_reason": "OpenRouter bridge to DashScope does not expose immutable model snapshots; floating alias mandated by B3 addendum § 5"
},
"qwen3.6-35b-a3b-via-openrouter": {
"id": "qwen3.6-35b-a3b-via-openrouter",
"displayName": "Qwen3.6-35B-A3B (C3 Stage 2 mini subject — alias name matches LiteLLM route, stage2 LOCKED config applied)",
"provider": "alibaba",
"litellmModel": "qwen3.6-35b-a3b-via-openrouter",
"pricePerMillionInput": 0.20,
"pricePerMillionOutput": 0.80,
"contextWindow": 262144,
"stage2Config": {
"thinking": true,
"maxTokens": 64000,
"reasoningShape": "openrouter-unified"
},
"pinning_surface": "floating_alias",
"pinning_surface_carve_out_reason": "Sprint 12 Task 2 C3 Stage 2 mini (2026-04-23): sibling alias of `qwen3.6-35b-a3b-stage2` that exposes the LiteLLM route name as the models.json key — lets the manifest `target_model: qwen3.6-35b-a3b-via-openrouter` match both the pre-registration audit trail AND the CLI --model arg verbatim. Same routing: LiteLLM -> OpenRouter -> Alibaba bridge (slug `openrouter/qwen/qwen3.5-35b-a3b`). OpenRouter bridge to DashScope does not expose immutable model snapshots; floating alias mandated by B3 addendum § 5."
},
"qwen3.6-35b-a3b-via-dashscope-direct": {
"id": "qwen3.6-35b-a3b-via-dashscope-direct",
"displayName": "Qwen3.6-35B-A3B (C3 Stage 2 Mini Retry v3 PRIMARY — DashScope-intl direct, TRUE 3.6)",
"provider": "alibaba",
"litellmModel": "qwen3.6-35b-a3b-via-dashscope-direct",
"pricePerMillionInput": 0.20,
"pricePerMillionOutput": 0.80,
"contextWindow": 262144,
"stage2Config": {
"thinking": true,
"maxTokens": 16000,
"reasoningShape": "dashscope-native"
},
"pinning_surface": "floating_alias",
"pinning_surface_carve_out_reason": "Sprint 12 Task 2 C3 Stage 2 Mini Retry v3 (2026-04-23): models.json key for the DashScope-intl direct route (litellm-config.yaml:228 alias `qwen3.6-35b-a3b-via-dashscope-direct` routes to `openai/qwen3.6-35b-a3b` @ https://dashscope-intl.aliyuncs.com/compatible-mode/v1 with DASHSCOPE_API_KEY). Delivers TRUE Qwen 3.6-35B-A3B (not the 3.5 regress of the OpenRouter bridge). thinking=on, max_tokens=16000 per v3 manifest §2.1 (reduced from v2 64000 to avoid tail-latency timeouts). DashScope-intl does not expose immutable model snapshots; floating alias mandated by B3 addendum § 5."
},
"qwen3.6-35b-a3b-local": {
"id": "qwen3.6-35b-a3b-local",
"displayName": "Qwen3.6-35B-A3B (local vLLM)",
"provider": "local",
"litellmModel": "openai/qwen3.6-35b-a3b",
"pricePerMillionInput": 0.0,
"pricePerMillionOutput": 0.0,
"contextWindow": 262144,
"pinning_surface": "floating_alias",
"pinning_surface_carve_out_reason": "Local vLLM endpoint does not expose immutable model snapshots; floating alias mandated by B3 addendum § 5"
},
"llama-3.1-8b-instruct": {
"id": "llama-3.1-8b-instruct",
"displayName": "Llama 3.1 8B Instruct (Week 2 placeholder)",
"provider": "local",
"litellmModel": "openai/llama-3.1-8b-instruct",
"pricePerMillionInput": 0.0,
"pricePerMillionOutput": 0.0,
"contextWindow": 131072,
"pinning_surface": "floating_alias",
"pinning_surface_carve_out_reason": "Local vLLM endpoint does not expose immutable model snapshots; floating alias mandated by B3 addendum § 5"
},
"claude-opus-4-6": {
"id": "claude-opus-4-6",
"displayName": "Claude Opus 4.6 (Week 2 placeholder, frontier baseline)",
"provider": "anthropic",
"litellmModel": "claude-opus-4-6",
"pricePerMillionInput": 15.0,
"pricePerMillionOutput": 75.0,
"contextWindow": 200000,
"pinning_surface": "anthropic_immutable",
"pinning_surface_carve_out_reason": null
},
"claude-opus-4-7": {
"id": "claude-opus-4-7",
"displayName": "Claude Opus 4.7 (Sprint 11 Task 2.2 primary judge)",
"provider": "anthropic",
"litellmModel": "claude-opus-4-7",
"pricePerMillionInput": 15.0,
"pricePerMillionOutput": 75.0,
"contextWindow": 200000,
"pinning_surface": "anthropic_immutable",
"pinning_surface_carve_out_reason": null,
"judge_role": "primary"
},
"gpt-5.4": {
"id": "gpt-5.4",
"displayName": "GPT-5.4 (Sprint 11 Task 2.2 primary judge, direct via LiteLLM→OpenAI)",
"provider": "openai_via_openrouter",
"litellmModel": "gpt-5.4",
"pricePerMillionInput": 10.0,
"pricePerMillionOutput": 30.0,
"contextWindow": 200000,
"pinning_surface": "floating_alias",
"pinning_surface_carve_out_reason": "Sprint 12 Task 2 C3 Stage 2 mini (2026-04-23): routing pivoted from OpenRouter bridge to direct OpenAI Chat Completions via LiteLLM local alias `gpt-5.4` → `openai/gpt-5.4` (eliminates ~5-15% OpenRouter middleware markup + preserves token-level telemetry for EU AI Act Art. 14 replay). OpenAI does not expose immutable model snapshots for the gpt-5.x family; floating alias mandated by B3 addendum § 5.",
"judge_role": "primary"
},
"gemini-3.1": {
"id": "gemini-3.1",
"displayName": "Gemini 3.1 Pro Preview (legacy OpenRouter bridge route)",
"provider": "google_via_openrouter",
"litellmModel": "openrouter/google/gemini-3.1-pro-preview",
"pricePerMillionInput": 3.5,
"pricePerMillionOutput": 10.5,
"contextWindow": 1000000,
"pinning_surface": "floating_alias",
"pinning_surface_carve_out_reason": "Legacy OpenRouter-bridge route; Sprint 12 Task 2 C3 Stage 2 mini (2026-04-23) pivoted judge routing to direct via LiteLLM local alias `gemini-3.1-pro` — see sibling entry below. This entry kept for backward compat with pre-Sprint-12 artefacts; floating alias mandated by B3 addendum § 5",
"judge_role": "primary"
},
"gemini-3.1-pro": {
"id": "gemini-3.1-pro",
"displayName": "Gemini 3.1 Pro Preview (Sprint 12 Task 2 primary judge, direct via LiteLLM→Google AI Studio)",
"provider": "google_via_openrouter",
"litellmModel": "gemini-3.1-pro",
"pricePerMillionInput": 3.5,
"pricePerMillionOutput": 10.5,
"contextWindow": 1000000,
"pinning_surface": "floating_alias",
"pinning_surface_carve_out_reason": "Sprint 12 Task 2 C3 Stage 2 mini (2026-04-23): direct-provider routing via LiteLLM local alias `gemini-3.1-pro` → `gemini/gemini-3.1-pro-preview` (Google AI Studio). Google ships the 3.1 Pro generation as `-preview` suffix only — no stable alias exists as of 2026-04-23 01:47 UTC (GET openrouter.ai/v1/models). Direct routing eliminates OpenRouter middleware markup + preserves native reasoning_tokens telemetry. Verify at replay time that `-preview` still resolves to the same underlying checkpoint; Google guarantees preview stability within a release window but not across release cycles. Floating alias mandated by B3 addendum § 5 (no immutable upstream snapshot surface).",
"judge_role": "primary"
},
"grok-4.20": {
"id": "grok-4.20",
"displayName": "Grok 4.20 (Sprint 11 B2 LOCK quadri-vendor tie-break reserve)",
"provider": "xai_via_openrouter",
"litellmModel": "openrouter/x-ai/grok-4.20",
"pricePerMillionInput": 5.0,
"pricePerMillionOutput": 15.0,
"contextWindow": 131072,
"pinning_surface": "floating_alias",
"pinning_surface_carve_out_reason": "xAI does not expose immutable model snapshots through OpenRouter routing layer; floating alias mandated by B3 addendum § 5",
"judge_role": "reserve"
}
}