# Bench-Spec LOCK v1 — machine-readable twin # Canonical markdown surface: 2026-04-22-bench-spec-locked.md # Sync guard: scripts/check-manifest-sync.mjs (authorized, pending implementation) # Any change to this file requires new markdown decision doc + PM ratification. manifest_version: v1.0.0 manifest_type: bench_spec_lock_parent locked_date: 2026-04-22 authority: PM (Marko Marković) — A3 interview 7/7 closed 2026-04-22 sprint: 11 track: A task: A3 # Per-run manifest instances (mini + full) will be emitted at run kickoff # and will inherit from this parent manifest, resolving dated snapshots. threshold_tiering: reference: mem0_locomo_91_6 reference_point_pct: 91.6 tiers: strong_publishable: point_min_pct: 91.6 wilson_lower_min_pct: 91.6 bootstrap_lower_min_pct: 91.6 publishable: point_min_pct: 91.6 wilson_lower_min_pct: 89.0 weak: point_min_pct: 89.0 point_max_pct: 91.5 requires: pm_review_gate fail: point_max_pct: 89.0 requires: post_mortem conservative_rule: "If Wilson and cluster-bootstrap disagree on tier, more conservative tier prevails." confidence_intervals: primary: method: wilson_score_95 description: "Frequentist binomial CI on instance-level binary verdicts." secondary: method: cluster_bootstrap_95 iterations: 10000 seed: 42 cluster_unit: conversation_id resample_mode: cluster_level_with_replacement quantiles: [2.5, 97.5] instance_counts: mini_c3: cells: 4 per_cell: 100 total_evaluations: 400 cell_names: [raw, filtered, compressed, full_context] budget_expected_usd: [120, 200] budget_cap_usd: 250 full_h42: qwen_n: 1540 qwen_runs: 3 qwen_total_evaluations: 4620 opus_probe_n: 500 opus_probe_runs: 3 opus_probe_total_evaluations: 1500 grand_total_evaluations: 6120 budget_expected_usd: [1300, 2300] budget_cap_usd: 2600 budget_hard_abort_usd: 2600 budget_breakdown_full: qwen_primary: expected_usd: [600, 1100] ceiling_usd: 1400 opus_probe: expected_usd: [200, 350] ceiling_usd: 450 judge_triple: expected_usd: [450, 750] ceiling_usd: 900 tiebreak_grok: expected_usd: [5, 15] ceiling_usd: 40 buffer_retries: expected_usd: [45, 85] ceiling_usd: 110 multiple_comparisons: mini_declaration: exploratory_descriptive_no_gating_no_correction full_declaration: primary_confirmatory_hypothesis_count: 1 h1_statement: "Qwen3.6-35B-A3B-Thinking achieves >= 91.6% point estimate on LoCoMo with Wilson 95% lower bound >= 89.0% (PUBLISHABLE tier)." secondary_metrics_treatment: descriptive_no_correction_required correction_family: none_required rationale: "Only one confirmatory hypothesis declared on full run; no multiple-comparisons correction needed." preregistration: v1_frozen_at: bench_spec_lock_2026_04_22 v2_issue_condition: material_change_surfaced_at_mini_exit v2_requires: new_pm_ratified_decision_doc mid_run_amendment_policy: halt_restart_required manifest_hash_event: bench.preregistration.manifest_hash h_audit_2_integration: required judge_ensemble: primary: - provider: anthropic floating_alias: anthropic/claude-opus-4-7 role: primary_judge_1 - provider: openai floating_alias: openai/gpt-5.4 role: primary_judge_2 - provider: google floating_alias: google/gemini-3.1 role: primary_judge_3 tiebreak: provider: xai floating_alias: xai/grok-4.20 trigger: three_way_split_1_1_1 path_enum: quadri-vendor defensive_2_2_path: pm-escalation consistency_constraint: same_physical_models_mini_and_full snapshot_drift_policy: manifest_flag_and_mini_rerun kappa_monitoring: baseline_reference: sprint_10_task_2_2_kappa_0_7458 compute: fleiss_kappa_on_pre_tiebreak_vote_matrix thresholds: pass_no_flag_kappa_min: 0.65 pass_with_flag_kappa_range: [0.60, 0.65] halt_kappa_max: 0.60 halt_drop_from_baseline_max_pp: 10 halt_protocol: preserve_partial_jsonl_write_halted_session_ping_notify_pm failure_taxonomy: version: v1 categories: - code: F1 name: contradicts_ground_truth - code: F2 name: partial_answer - code: F3 name: off_topic - code: F4 name: refusal - code: F5 name: tool_use_error scope: tool_permitted_cells_only - code: F6 name: format_violation special: null_correct: description: judge_majority_verdict_correct_no_f_code f_other: description: failure_outside_f1_f6_taxonomy mandatory_rationale_min_words: 10 rate_threshold_for_taxonomy_review_pct: 10 jsonl_schema_extension: fields: verdict: [correct, incorrect] failure_code: [null, F1, F2, F3, F4, F5, F6, F_other] failure_rationale: null_unless_f_other reproducibility_manifest: format: hybrid_markdown_plus_yaml canonical_surface: markdown machine_surface: yaml per_run_path_convention: mini: PM-Waggle-OS/decisions/-stage2-mini-manifest.md full: PM-Waggle-OS/decisions/-stage2-full-manifest.md freeze_timing: run_kickoff mid_run_change_policy: halt_and_restart required_fields_count: 16 required_fields: - manifest_version - manifest_hash - run_id - run_stage - target_model - target_model_thinking_mode - judge_primary - judge_tiebreak - judge_rubric_path - dataset - dataset_version - instance_count - cells - ci_method - failure_taxonomy_version - budget_cap - retention_policy ci_sync_guard: script_path: scripts/check-manifest-sync.mjs status: authorized_pending_implementation repo: waggle-os interim_policy: manual_sync_verified_in_commit_message reasoning_content_retention: inherits_from: a2_q5_tier_2_locked archive_content: full_jsonl_with_reasoning_content_preserved_unpruned bundle_layout: runs_dir: runs/ aggregates_dir: aggregates/ manifest_yaml: manifest.yaml manifest_md: manifest.md exit_ping: exit-ping.md git_state: git-state.txt docker_state: docker-state.txt readme: README.md bundle_path_convention: waggle-os/benchmarks/archive/-stage2-.tar.gz access_policy: internal_egzakta: open external_regulator_partner_auditor: pm_signoff_plus_audit_log_entry audit_log_path_convention: PM-Waggle-OS/audit-log/--.md hybrid_external_consultant: default_external_tier_pm_override_with_written_rationale retention_horizon: minimum_months: 12 active_launch_claim: indefinite post_decommissioning_additional_months: 24 decommissioning_trigger: pm_supersession_decision_doc_or_product_retirement validation_gates: before_c3_mini_kickoff: - parallel_yaml_manifest_committed - manifest_hash_recorded_in_commit - ci_sync_guard_spec_documented - h_audit_2_manifest_hash_event_spot_verified - kickoff_brief_cites_lock - exit_ping_reports_kappa_wilson_bootstrap_f_distribution_hash_match_budget before_h42_full_kickoff: - c3_mini_exit_pass_with_pm_review - v2_issued_if_material_change_else_v1_carryforward_noted - dated_snapshots_re_resolved_and_pinned - budget_guard_configured_hard_abort_2600usd out_of_scope: - provider_rotation_timing - judge_rubric_evolution_beyond_f_taxonomy - tertiary_metrics_beyond_f1_f6 - third_party_replication - landing_copy_marketing_derivation related_decisions: a1_h_audit_design: PM-Waggle-OS/decisions/2026-04-22-h-audit-1-design-ratified.md b1_stage_2_primary_config: PM-Waggle-OS/decisions/2026-04-22-stage-2-primary-config-locked.md b2_tie_break_policy: PM-Waggle-OS/decisions/2026-04-22-tie-break-policy-locked.md b3_model_route_naming: PM-Waggle-OS/decisions/2026-04-22-model-route-naming-locked.md related_exit_pings: a2_h_audit_1: PM-Waggle-OS/sessions/2026-04-22-sprint-11-h-audit-1-exit.md b1_stage_2_config: PM-Waggle-OS/sessions/2026-04-22-sprint-11-b1-stage2-config-exit.md b2_tiebreak: PM-Waggle-OS/sessions/2026-04-22-sprint-11-b2-tiebreak-exit.md b3_opus46_audit: PM-Waggle-OS/sessions/2026-04-22-sprint-11-b3-opus46-audit-exit.md c2_mikroeval: PM-Waggle-OS/sessions/2026-04-22-sprint-11-c2-stage1-mikroeval-exit.md sprint_11_impact: pre_lock_exit_criteria_closed: 8_of_10 post_lock_exit_criteria_closed: 9_of_10_pending_c3_execution unblocks: c3_stage_2_4_cell_mini gates_remaining_after_lock: [b4_stage_2_kickoff_memo, c3_execution]