189 lines
11 KiB
YAML
189 lines
11 KiB
YAML
# C3 Stage 2 Mini Retry v3 Per-Run Manifest — machine-readable twin
|
|
# Canonical markdown surface: 2026-04-23-stage2-mini-manifest-v3.md
|
|
# Parent bench-spec lock: 2026-04-22-bench-spec-locked.manifest.yaml
|
|
# Supersedes: 2026-04-23-stage2-mini-manifest.manifest.yaml (v1, partial-run aborted 2026-04-23T01:33Z)
|
|
# Sync guard: scripts/check-manifest-sync.mjs (authorized, pending implementation)
|
|
# Any change to this file requires new markdown decision doc + PM ratification
|
|
# and a new hash binding (old runs' --manifest-hash binding becomes invalid).
|
|
|
|
manifest_version: v3.0.0
|
|
manifest_type: per_run_mini_retry
|
|
inherits_from_parent: decisions/2026-04-22-bench-spec-locked.manifest.yaml
|
|
supersedes_v1: decisions/2026-04-23-stage2-mini-manifest.manifest.yaml
|
|
supersedes_v1_reason: "v1 run aborted 2026-04-23T01:33Z after 64 cell-raw instances. Rolling 50-eval error rate 20% > 15% threshold due to OpenRouter→Alibaba bridge tail-latency degradation under thinking=on + max_tokens=64000. v3 pivots to DashScope-direct primary + reduces max_tokens to 16000 + raises HTTP timeout to 300s."
|
|
locked_date: 2026-04-23
|
|
authority: "PM (Marko Marković) — v3 brief 2026-04-23, GATE-0 adjudication (2-route roster, Option 4), GATE-1 adjudication (dashscope-direct primary)"
|
|
sprint: 12
|
|
track: A
|
|
task: 2
|
|
run_stage: mini_retry
|
|
|
|
# ── Field 7: subject model + routing (v3 EXACT values per brief §2.2) ──────
|
|
subject_model: qwen3.6-35b-a3b-via-dashscope-direct
|
|
subject_fallback_1: qwen3.6-35b-a3b-via-openrouter
|
|
subject_fallback_2: NOT_AVAILABLE
|
|
subject_thinking: on
|
|
subject_max_tokens: 16000
|
|
subject_http_timeout_ms: 300000
|
|
subject_parallel_concurrency: 2
|
|
subject_routing_path: alibaba-dashscope-direct
|
|
subject_quantization: FP16-cloud
|
|
subject_quantization_source_note: "DashScope-intl compatible-mode API does not expose provider-side quantization metadata in response objects as of 2026-04-23. Operational assumption: cloud inference on FP16 weights; verify at replay time via DashScope console / billing-line level-of-detail if auditor-requested."
|
|
|
|
subject_routing_detail:
|
|
litellm_alias: qwen3.6-35b-a3b-via-dashscope-direct
|
|
upstream_provider: alibaba_dashscope_intl_direct
|
|
upstream_slug: openai/qwen3.6-35b-a3b
|
|
api_base: https://dashscope-intl.aliyuncs.com/compatible-mode/v1
|
|
api_key_env: DASHSCOPE_API_KEY
|
|
routing_path: "LiteLLM (localhost:4000) → DashScope-intl compatible-mode API direct → Alibaba inference (TRUE Qwen 3.6-35B-A3B)"
|
|
smoke_verified_at: 2026-04-23T12:10:27Z
|
|
smoke_reliability_3_of_3: 1.00
|
|
smoke_latency_median_ms: 5528
|
|
smoke_latency_max_ms: 11655
|
|
smoke_reasoning_chars_median: 1971
|
|
|
|
subject_fallback_1_detail:
|
|
litellm_alias: qwen3.6-35b-a3b-via-openrouter
|
|
upstream_provider: openrouter_bridge
|
|
upstream_slug: openrouter/qwen/qwen3.5-35b-a3b
|
|
api_key_env: OPENROUTER_API_KEY
|
|
routing_path: "LiteLLM → OpenRouter → Alibaba inference"
|
|
regression_disclosure: "OpenRouter catalog does NOT carry 3.6-35b-a3b slug as of 2026-04-23 01:47 UTC. This route lands on qwen3.5-35b-a3b — a one-minor regression. Acceptable for transient DashScope outage fallback only; NOT acceptable as primary."
|
|
smoke_reliability_2_of_3: 0.67
|
|
smoke_fastfail_observed_on: locomo_conv-26_q059
|
|
smoke_fastfail_http_code: 500
|
|
smoke_fastfail_latency_ms: 58
|
|
smoke_latency_median_ms_healthy: 3353
|
|
|
|
subject_fallback_2_detail:
|
|
status: NOT_AVAILABLE
|
|
reason: "Ollama cloud does not publish Qwen 3.6 variants as of 2026-04-23 (cloud catalog check via ollama.com/library/qwen3.6/tags: only local-inference tags e.g. qwen3.6:35b-a3b, qwen3.6:35b-a3b-bf16, etc.; only Qwen 3.5 has `:cloud`-suffix routed-remote variants). 2-route roster confirmed by PM adjudication at GATE-0 Option 4 (2026-04-23)."
|
|
revisit_sprint: 13
|
|
revisit_trigger: "Ollama cloud publishes `qwen3.6*:cloud` OR Alibaba partners with Ollama for remote-routed 3.6 inference"
|
|
|
|
# ── Judge ensemble — DIRECT routing via LiteLLM local aliases ──────────────
|
|
judge_primary:
|
|
id: claude-opus-4-7
|
|
routing_path: anthropic-direct
|
|
litellm_alias: claude-opus-4-7
|
|
upstream_slug: anthropic/claude-opus-4-7
|
|
api_key_env: ANTHROPIC_API_KEY
|
|
pinning_surface: anthropic_immutable
|
|
pinning_surface_carve_out_reason: null
|
|
smoke_verified_at: 2026-04-23T01:53:27Z
|
|
smoke_latency_ms: 1639
|
|
|
|
judge_secondary:
|
|
id: gpt-5.4
|
|
routing_path: openai-direct
|
|
litellm_alias: gpt-5.4
|
|
upstream_slug: openai/gpt-5.4
|
|
api_key_env: OPENAI_API_KEY
|
|
pinning_surface: floating_alias
|
|
pinning_surface_carve_out_reason: "OpenAI does not expose immutable model snapshots for the gpt-5.x family via Chat Completions surface. Floating alias mandated by B3 addendum § 5."
|
|
smoke_verified_at: 2026-04-23T01:53:29Z
|
|
smoke_latency_ms: 3178
|
|
|
|
judge_tie_breaker:
|
|
id: gemini-3.1-pro-preview
|
|
routing_path: google-ai-studio-direct
|
|
litellm_alias: gemini-3.1-pro-preview
|
|
upstream_slug: gemini/gemini-3.1-pro-preview
|
|
api_key_env: GEMINI_API_KEY
|
|
pinning_surface: floating_alias
|
|
pinning_surface_carve_out_reason: "Gemini 3.1 Pro ships only as `-preview` suffix as of 2026-04-23 (no stable variant). Google guarantees preview-alias stability within release window, not across cycles. Replay-time verification required. Floating alias mandated by B3 addendum § 5."
|
|
smoke_verified_at: 2026-04-23T01:53:32Z
|
|
smoke_latency_ms: 2531
|
|
|
|
# ── Run scope ──────────────────────────────────────────────────────────────
|
|
target_N: 400
|
|
cells: [raw, context, retrieval, agentic]
|
|
cells_audit_note: "v3 brief §2.2 specifies cell names `raw, context, retrieval, agentic`. Existing harness runner (benchmarks/harness/src/runner.ts) uses the v1 cell vocabulary `raw | filtered | compressed | full-context`. Stage 3 runner invocation must either (a) add the v3 name mapping to cells.ts isCellName() + cells dict, or (b) invoke with the existing v1 names plus a documented mapping in the exit ping (v3 `raw=raw`, `context=full-context`, `retrieval=filtered`, `agentic=compressed`). GATE-2 is appropriate place for PM to confirm which mapping to take."
|
|
|
|
dataset: locomo
|
|
dataset_path: benchmarks/data/locomo/locomo-1540.jsonl
|
|
dataset_instance_count_total: 1531
|
|
dataset_version_hash_source: benchmarks/harness/src/datasets.ts::getDatasetVersion
|
|
|
|
seed: 42
|
|
ci_method:
|
|
wilson_95: true
|
|
cluster_bootstrap_95: true
|
|
bootstrap_iterations: 10000
|
|
bootstrap_seed: 42
|
|
cluster_unit: conversation_id
|
|
|
|
# ── Budget ─────────────────────────────────────────────────────────────────
|
|
expected_budget_usd: "100-200"
|
|
budget_cap_usd: 250
|
|
budget_abort_threshold_usd: 325
|
|
|
|
# ── Abort triggers (v3 brief §3.3 EXACT values) ────────────────────────────
|
|
abort_triggers:
|
|
- name: rolling_50_error_rate_exceeds
|
|
threshold: 0.10
|
|
semantic: "Rolling 50-eval error rate > 10%. Tightened from v2 (15%) per v3 §3.3. Mid-run HALT + page Marko."
|
|
- name: cell_completion_p50_exceeds
|
|
threshold_minutes: 30
|
|
semantic: "Any single cell p50 completion time > 30 min. Mid-run HALT + page Marko."
|
|
- name: total_spend_exceeds
|
|
threshold_usd: 325
|
|
semantic: "Cumulative run spend > $325 (130% of $250 cap). HARD HALT."
|
|
|
|
# ── Retention ──────────────────────────────────────────────────────────────
|
|
retention_policy: A2-Q5-tier-2-full-preserved
|
|
retention_horizon_floor_months: 12
|
|
retention_horizon_active_claim: indefinite_plus_24mo_post_decommissioning
|
|
|
|
# ── Failure taxonomy surface (A3 LOCK § 6) ─────────────────────────────────
|
|
failure_taxonomy_version: F1-F6+other v1
|
|
a3_namespace_split_commit: 7b7436d
|
|
a3_failure_code_column: a3_failure_code
|
|
a3_rationale_column: a3_rationale
|
|
sprint9_legacy_columns_preserved: true
|
|
sprint9_legacy_columns_populated_on_this_run: true
|
|
known_scope_out_f6_f_other: "Judge response parser (failure-mode-judge.ts Zod + buildJudgePrompt) still targets Sprint 9 5-value space. rubric.ts::buildJudgeRubricBlock is shipped but not spliced into Task 2 runtime. F6 and F_other counts will be 0 in aggregate.failure_distribution. Distribution remains structurally valid; review-flag gate stays OFF trivially. Activation = Task 2 Phase 2 rubric splice."
|
|
|
|
# ── resolveTieBreak wire ───────────────────────────────────────────────────
|
|
resolve_tie_break_live: true
|
|
resolve_tie_break_wire_commit: 80896f1
|
|
resolve_tie_break_audit_slug_alignment_commit: 89268ae
|
|
|
|
# ── Stage-1 smoke record (summary; detail in binary-subject-smoke.md) ──────
|
|
stage1_smoke_cost_usd: 0.0024
|
|
stage1_smoke_total_calls: 6
|
|
stage1_smoke_subject_only: true
|
|
stage1_smoke_dashscope_direct_reliability: 1.00
|
|
stage1_smoke_openrouter_reliability: 0.67
|
|
stage1_smoke_artefact: benchmarks/results/smoke-binary-2026-04-23T12-10-15Z.jsonl
|
|
|
|
# ── Pre-kick commits (prior to v3 manifest emit) ───────────────────────────
|
|
pre_v3_commits:
|
|
- sha: 7b7436d
|
|
scope: "§2.1 A3 failure taxonomy namespace split"
|
|
- sha: 68f26ba
|
|
scope: "§2.2 OpenRouter slug verification patch"
|
|
- sha: 89268ae
|
|
scope: "§2.3 resolveTieBreak audit-slug alignment"
|
|
- sha: 34ba083
|
|
scope: "§3.5 direct-provider judge routing audit + models.json pre-registration refresh"
|
|
- sha: 01ccf59
|
|
scope: "qwen3.6-35b-a3b-via-openrouter models.json alias (v2)"
|
|
- sha: 5ec069e
|
|
scope: "v3 Stage 0: qwen3.6-35b-a3b-via-dashscope-direct + gemini-3.1-pro-preview LiteLLM alias adds"
|
|
|
|
# ── Stage 3 invocation (v3 brief §3.2 template, NOT YET EXECUTED) ──────────
|
|
runtime_invocation_template: >
|
|
npx tsx scripts/run-mini-locomo.ts
|
|
--manifest decisions/2026-04-23-stage2-mini-manifest-v3.yaml
|
|
--subject qwen3.6-35b-a3b-via-dashscope-direct
|
|
--judge-ensemble claude-opus-4-7,gpt-5.4,gemini-3.1-pro-preview
|
|
--N 100 --cells raw,context,retrieval,agentic
|
|
--parallel-concurrency 2
|
|
--output benchmarks/results/raw-locomo-retry-v3-<ISO>.jsonl
|
|
|
|
runtime_invocation_audit_note: "v3 brief §3.2 invokes `scripts/run-mini-locomo.ts` which does NOT currently exist in the waggle-os repo. Existing runner is `benchmarks/harness/src/runner.ts`. Stage 3 kickoff requires either (a) creating `scripts/run-mini-locomo.ts` as a thin wrapper that reads the manifest YAML and maps v3 cell names to the existing runner, OR (b) invoking runner.ts directly with equivalent flags. GATE-2 ratification should specify which path. If (b), the required flag translations are: --subject → --model, --cells → --cell/--all-cells (v3 cell-name mapping per `cells_audit_note` above), --N → --limit, --parallel-concurrency has no existing flag in runner.ts (sequential execution is the current default; concurrency requires runner.ts patch or post-launch parallelism via shell job control)."
|
|
|
|
emitted_at: 2026-04-23T13:10:00Z
|