This commit is contained in:
Oleg Maslov
457
benchmarks/results/manifest-v4-preregistration.yaml
Normal file
457
benchmarks/results/manifest-v4-preregistration.yaml
Normal file
@@ -0,0 +1,457 @@
|
||||
# Manifest v4 — Task 2.5 Stage 3 N=400 Pre-Registration (machine-readable twin)
|
||||
# Canonical markdown surface: manifest-v4-preregistration.md
|
||||
# Any change to this file requires new PM-ratified decision doc + manifest v5.
|
||||
# SHA-256 of this YAML's bytes is recorded in the anchor commit message.
|
||||
|
||||
manifest_version: v4.0.0-preregistration
|
||||
manifest_type: stage_3_n400_preregistration
|
||||
locked_date: 2026-04-24
|
||||
authority: PM (Marko Markovic) — Option 1 (N=400 direct) ratified 2026-04-24 on Stage 2-Retry Gate C PARTIAL PASS exit
|
||||
sprint: 12
|
||||
task: 2.5
|
||||
stage: 3
|
||||
branch: feature/c3-v3-wrapper
|
||||
code_freeze_head: 373516c2784807da8536dbc0c194c54f4e4cd4be
|
||||
code_freeze_head_short: 373516c
|
||||
supersedes: stage_2_retry_n20_gate_c
|
||||
inherits_from: bench_spec_lock_v1_2026_04_22
|
||||
|
||||
# ── Field 7 slots (preregistration.ts PreregistrationManifestPayload) ───────
|
||||
|
||||
manifest_path: benchmarks/results/manifest-v4-preregistration.yaml
|
||||
manifest_locked_at: 2026-04-24T00:00:00Z
|
||||
|
||||
dataset:
|
||||
name: locomo
|
||||
source_url: https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json
|
||||
raw_archive_path: benchmarks/data/locomo10.json
|
||||
raw_archive_sha256: 79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4
|
||||
raw_archive_bytes: 2805274
|
||||
canonical_path: benchmarks/data/locomo/locomo-1540.jsonl
|
||||
canonical_sha256: 39e415e2f3a0fa1bd3cb1804a58d0b440b50d3070b2100698437e4ec402a5b24
|
||||
canonical_instance_count: 1531
|
||||
paper_total_claim: 1540
|
||||
paper_reference: "Maharana et al., ACL-2024 — Evaluating Very Long-Term Conversational Memory of LLM Agents"
|
||||
category_distribution:
|
||||
single_hop: 841
|
||||
multi_hop: 281
|
||||
temporal: 320
|
||||
open_ended: 89
|
||||
|
||||
# ── Primary hypothesis (directional, confirmatory) ──────────────────────────
|
||||
|
||||
primary_hypothesis:
|
||||
name: memory_lift_retrieval_vs_no_context
|
||||
direction: one_sided_positive
|
||||
statement: "retrieval_judge_accuracy − no-context_judge_accuracy ≥ 5pp"
|
||||
test: fisher_exact_one_sided
|
||||
alpha_threshold: 0.10
|
||||
effect_size_threshold_pp: 5
|
||||
justification_ex_ante:
|
||||
- gate_b_dry_run_conv_scope_20_of_20_vs_whole_corpus_8_of_20_leak_2026_04_24
|
||||
- gate_c_monotonicity_no_context_0_10_lt_retrieval_0_35_lt_agentic_0_40_lt_oracle_0_55
|
||||
|
||||
# ── Secondary endpoints (ex-ante, non-blocking, descriptive) ────────────────
|
||||
|
||||
secondary_endpoints:
|
||||
S1_monotonicity_no_context_leq_retrieval:
|
||||
direction: one_sided_positive
|
||||
threshold_pp: 0
|
||||
test: fisher_exact_one_sided
|
||||
alpha: 0.20
|
||||
S2_monotonicity_retrieval_leq_agentic:
|
||||
direction: one_sided_positive
|
||||
threshold_pp: 0
|
||||
test: fisher_exact_one_sided
|
||||
alpha: 0.20
|
||||
S3_monotonicity_agentic_leq_oracle_context:
|
||||
direction: one_sided_positive
|
||||
threshold_pp: 0
|
||||
test: fisher_exact_one_sided
|
||||
alpha: 0.20
|
||||
S4_agentic_lift_over_retrieval:
|
||||
direction: descriptive
|
||||
threshold_pp: 0
|
||||
report: [point_estimate, wilson_95_ci]
|
||||
S5_abstain_penalty_oracle_minus_full_context:
|
||||
direction: descriptive_expected_positive
|
||||
report: [point_estimate]
|
||||
rationale: "SYSTEM_EVOLVED strict-abstain prompt is expected to reduce full-context accuracy below oracle-context on instances where the oracle answer is ambiguous; diagnostic only, no gating"
|
||||
|
||||
# ── Sample design ───────────────────────────────────────────────────────────
|
||||
|
||||
sample:
|
||||
cells:
|
||||
- no-context
|
||||
- oracle-context
|
||||
- full-context
|
||||
- retrieval
|
||||
- agentic
|
||||
n_per_cell: 400
|
||||
total_evaluations: 2000
|
||||
instance_selection_seed: 42
|
||||
instance_selection_method: "shuffle-then-take-first-N, deterministic given seed"
|
||||
matched_pairs: true
|
||||
concurrency: 2
|
||||
|
||||
# ── Cells semantics (frozen at HEAD 373516c) ────────────────────────────────
|
||||
|
||||
cells_semantics:
|
||||
no_context:
|
||||
system_prompt: SYSTEM_BASELINE
|
||||
user_prompt: "Question: {question}"
|
||||
memory_injection: none
|
||||
added_at: stage_2_retry_1_1_2026_04_24
|
||||
oracle_context:
|
||||
system_prompt: SYSTEM_BASELINE
|
||||
user_prompt: "Context: {instance.context}\\n\\nQuestion: {instance.question}"
|
||||
memory_injection: oracle_fed_by_locomo
|
||||
harness_alias: raw
|
||||
note: "PM-facing alias for harness `raw` (oracle-fed on LoCoMo; not actually zero-memory)"
|
||||
full_context:
|
||||
system_prompt: SYSTEM_EVOLVED
|
||||
user_prompt: "# Recalled Memories\\n- [memory:synth] {instance.context}\\n\\nQuestion: {instance.question}"
|
||||
memory_injection: oracle_fed_plus_evolved_abstain
|
||||
retrieval:
|
||||
system_prompt: SYSTEM_BASELINE
|
||||
substrate: waggle_core_hybrid_search
|
||||
scope: conversation_scoped_via_gopId
|
||||
top_k_default: 20
|
||||
top_k_upper_clamp: 50
|
||||
agentic:
|
||||
system_prompt: SYSTEM_AGENTIC_softened_stage2_retry
|
||||
system_prompt_sha256: 6facae6decc44a6404290514accb4f7cb364081b32d02847a20f8e871633e328
|
||||
system_prompt_bytes: 1467
|
||||
tool_allowlist:
|
||||
- search_memory
|
||||
tool_binding: "search_memory is bound to instance.conversation_id via makeSearchMemoryTool(substrate, 20, boundToGopId); non-overridable by agent"
|
||||
max_turns: 3
|
||||
timeout_ms: 180000
|
||||
forced_answer_fallback:
|
||||
enabled: true
|
||||
system_prompt: SYSTEM_AGENTIC_FORCED_FALLBACK
|
||||
trigger: "resp.content.trim() === '' && capturedToolResults.length > 0 on runAgentLoop exit"
|
||||
gate_c_firing_rate: 0
|
||||
|
||||
# ── Model stack ─────────────────────────────────────────────────────────────
|
||||
|
||||
subject_model: qwen3.6-35b-a3b-via-dashscope-direct
|
||||
subject_fallback_1: qwen3.6-35b-a3b-via-openrouter
|
||||
subject_fallback_2: NOT_AVAILABLE
|
||||
|
||||
subject_route_table:
|
||||
primary:
|
||||
alias: qwen3.6-35b-a3b-via-dashscope-direct
|
||||
litellm_model: qwen3.6-35b-a3b-via-dashscope-direct
|
||||
upstream_route: "LiteLLM local alias -> openai/qwen3.6-35b-a3b @ https://dashscope-intl.aliyuncs.com/compatible-mode/v1"
|
||||
provider: alibaba
|
||||
thinking: on
|
||||
max_tokens: 16000
|
||||
reasoning_shape: dashscope-native
|
||||
price_per_million_input_usd: 0.20
|
||||
price_per_million_output_usd: 0.80
|
||||
context_window: 262144
|
||||
pinning_surface: floating_alias
|
||||
pinning_surface_carve_out_reason: "DashScope-intl does not expose immutable model snapshots; floating alias mandated by B3 addendum § 5"
|
||||
note: "TRUE Qwen 3.6-35B-A3B (not the 3.5 regress of the OpenRouter bridge)"
|
||||
fallback_1:
|
||||
alias: qwen3.6-35b-a3b-via-openrouter
|
||||
litellm_model: qwen3.6-35b-a3b-via-openrouter
|
||||
upstream_route: "LiteLLM -> OpenRouter bridge (openrouter/qwen/qwen3.5-35b-a3b)"
|
||||
provider: alibaba_via_openrouter
|
||||
thinking: on
|
||||
max_tokens: 64000
|
||||
reasoning_shape: openrouter-unified
|
||||
pinning_surface: floating_alias
|
||||
note: "Known 3.5 regress via OpenRouter bridge; only used on primary network failure"
|
||||
trigger_condition: fetch_error_on_primary
|
||||
fallback_2:
|
||||
alias: NOT_AVAILABLE
|
||||
note: "No third fallback; halt if fallback_1 also fails"
|
||||
|
||||
judge_ensemble:
|
||||
primary:
|
||||
- judge_role: primary
|
||||
slot: primary_judge_1
|
||||
model_id: claude-opus-4-7
|
||||
provider: anthropic
|
||||
litellm_model: claude-opus-4-7
|
||||
pinning_surface: anthropic_immutable
|
||||
pinning_surface_carve_out_reason: null
|
||||
price_per_million_input_usd: 15.00
|
||||
price_per_million_output_usd: 75.00
|
||||
- judge_role: primary
|
||||
slot: primary_judge_2
|
||||
model_id: gpt-5.4
|
||||
provider: openai_via_openrouter
|
||||
litellm_model: gpt-5.4
|
||||
pinning_surface: floating_alias
|
||||
pinning_surface_carve_out_reason: "Direct OpenAI Chat Completions via LiteLLM local alias `gpt-5.4` -> `openai/gpt-5.4`; OpenAI does not expose immutable model snapshots for gpt-5.x; B3 addendum § 5"
|
||||
price_per_million_input_usd: 10.00
|
||||
price_per_million_output_usd: 30.00
|
||||
- judge_role: primary
|
||||
slot: primary_judge_3
|
||||
model_id: gemini-3.1-pro
|
||||
provider: google_via_openrouter
|
||||
litellm_model: gemini-3.1-pro
|
||||
pinning_surface: floating_alias
|
||||
pinning_surface_carve_out_reason: "Direct Google AI Studio via LiteLLM local alias `gemini-3.1-pro` -> `gemini/gemini-3.1-pro-preview`; no stable alias (preview-only); B3 addendum § 5"
|
||||
price_per_million_input_usd: 3.50
|
||||
price_per_million_output_usd: 10.50
|
||||
tiebreak:
|
||||
judge_role: reserve
|
||||
model_id: grok-4.20
|
||||
provider: xai_via_openrouter
|
||||
litellm_model: openrouter/x-ai/grok-4.20
|
||||
pinning_surface: floating_alias
|
||||
pinning_surface_carve_out_reason: "xAI does not expose immutable model snapshots through OpenRouter routing layer; floating alias mandated by B3 addendum § 5"
|
||||
price_per_million_input_usd: 5.00
|
||||
price_per_million_output_usd: 15.00
|
||||
trigger: three_way_split_1_1_1
|
||||
path_enum: quadri-vendor
|
||||
defensive_2_2_path: pm-escalation
|
||||
consistency_constraint: same_physical_judge_models_as_stage_1_stage_1_5_stage_2_stage_2_retry
|
||||
vote_policy: majority_with_grok_reserve_on_1_1_1_split
|
||||
snapshot_drift_policy: halt_and_manifest_flag
|
||||
|
||||
# Convenience flat list for preregistration.ts emitter compatibility
|
||||
judge_primary:
|
||||
id: claude-opus-4-7
|
||||
judge_secondary:
|
||||
id: gpt-5.4
|
||||
judge_tie_breaker:
|
||||
id: gemini-3.1-pro
|
||||
|
||||
# ── Substrate ───────────────────────────────────────────────────────────────
|
||||
|
||||
substrate:
|
||||
implementation: "@waggle/core::HybridSearch (RRF-fused FTS5 + vec0)"
|
||||
ingest_keying: "gop_id = conversation_id"
|
||||
scope_filter:
|
||||
parameter: gopId
|
||||
source_location: packages/core/src/mind/search.ts:14
|
||||
field_name: SearchOptions.gopId
|
||||
non_overridable: true
|
||||
benchmark_binding: instance.conversation_id
|
||||
top_k_default: 20
|
||||
top_k_upper_clamp: 50
|
||||
embedder:
|
||||
factory: createOllamaEmbedder
|
||||
base_url: http://localhost:11434
|
||||
model: nomic-embed-text
|
||||
dims: 1024
|
||||
cost: zero_local_inference
|
||||
ingest_batch_size: 200
|
||||
code_reference: benchmarks/harness/src/substrate.ts
|
||||
|
||||
# ── κ monitoring (inherited from Bench-Spec LOCK v1) ────────────────────────
|
||||
|
||||
kappa_monitoring:
|
||||
baseline_reference: sprint_10_task_2_2_kappa_0_7458
|
||||
compute: fleiss_kappa_on_pre_tiebreak_vote_matrix
|
||||
thresholds:
|
||||
pass_no_flag_kappa_min: 0.65
|
||||
pass_with_flag_kappa_range: [0.60, 0.65]
|
||||
halt_kappa_max: 0.60
|
||||
halt_drop_from_baseline_max_pp: 10
|
||||
halt_protocol: preserve_partial_jsonl_write_halted_session_ping_notify_pm
|
||||
|
||||
# ── Confidence intervals (inherited from Bench-Spec LOCK v1) ────────────────
|
||||
|
||||
confidence_intervals:
|
||||
primary:
|
||||
method: wilson_score_95
|
||||
description: "Frequentist binomial CI on instance-level binary verdicts."
|
||||
secondary:
|
||||
method: cluster_bootstrap_95
|
||||
iterations: 10000
|
||||
seed: 42
|
||||
cluster_unit: conversation_id
|
||||
resample_mode: cluster_level_with_replacement
|
||||
quantiles: [2.5, 97.5]
|
||||
|
||||
# ── Failure taxonomy (inherited from Bench-Spec LOCK v1) ────────────────────
|
||||
|
||||
failure_taxonomy:
|
||||
version: v1
|
||||
categories:
|
||||
- {code: F1, name: contradicts_ground_truth}
|
||||
- {code: F2, name: partial_answer}
|
||||
- {code: F3, name: off_topic}
|
||||
- {code: F4, name: refusal}
|
||||
- {code: F5, name: tool_use_error, scope: tool_permitted_cells_only}
|
||||
- {code: F6, name: format_violation}
|
||||
special:
|
||||
null_correct:
|
||||
description: judge_majority_verdict_correct_no_f_code
|
||||
f_other:
|
||||
description: failure_outside_f1_f6_taxonomy
|
||||
mandatory_rationale_min_words: 10
|
||||
rate_threshold_for_taxonomy_review_pct: 10
|
||||
|
||||
# ── Stopping rules (ex-ante, no interim looks) ──────────────────────────────
|
||||
|
||||
stopping_rules:
|
||||
budget_hard_halt_usd: 28.00
|
||||
budget_cap_usd: 30.00
|
||||
streak_halt: "3 consecutive fetch failures on same model -> halt (streak-tracker.ts)"
|
||||
pre_cell_health_check: "GET /health/liveliness + POST /v1/chat/completions ping per model -> halt before cell on any 5xx/fetch-error (health-check.ts)"
|
||||
runner_lock: "concurrent_runners: FORBIDDEN (runner-lock.ts); contention -> halt"
|
||||
deviation_from_preregistration: "any change to manifest v4 §1-§9 during run -> immediate halt + PM raise"
|
||||
no_interim_looks: true
|
||||
mid_run_amendment_policy: halt_restart_required
|
||||
|
||||
# ── Post-hoc exclusion policy: NONE ─────────────────────────────────────────
|
||||
|
||||
post_hoc_exclusion:
|
||||
policy: none
|
||||
evaluator_loss_handling:
|
||||
included_in_denominator: true
|
||||
reported_separately: true
|
||||
denominator_formula: "correct + incorrect + evaluator_loss"
|
||||
subject_failures_classified_as: failure_mode_per_f1_f6_taxonomy
|
||||
instance_whitelist: none
|
||||
instance_blacklist: none
|
||||
|
||||
# ── Budget ──────────────────────────────────────────────────────────────────
|
||||
|
||||
budget:
|
||||
cap_usd: 30.00
|
||||
hard_halt_usd: 28.00
|
||||
expected_burn_usd: 23.00
|
||||
variance_ceiling_usd: 28.00
|
||||
breakdown_expected:
|
||||
subject_qwen_dashscope_direct_usd: 2.50
|
||||
judge_triple_opus_gpt5_gemini_usd: 20.00
|
||||
embedding_ollama_local_usd: 0.00
|
||||
tie_break_grok_reserve_usd: 0.50
|
||||
scale_from_stage_2_retry_gate_c: "Gate C $1.1637 / 100 evals × 20 = $23.27"
|
||||
|
||||
# ── Target sample + CLI invocation ──────────────────────────────────────────
|
||||
|
||||
target_N: 400
|
||||
target_cells:
|
||||
- no-context
|
||||
- oracle-context
|
||||
- full-context
|
||||
- retrieval
|
||||
- agentic
|
||||
target_total_evaluations: 2000
|
||||
target_concurrency: 2
|
||||
|
||||
cli_invocation_template: >
|
||||
npx tsx scripts/run-mini-locomo.ts
|
||||
--manifest benchmarks/results/manifest-v4-preregistration.yaml
|
||||
--subject qwen3.6-35b-a3b-via-dashscope-direct
|
||||
--subject-fallback-1 qwen3.6-35b-a3b-via-openrouter
|
||||
--judge-ensemble claude-opus-4-7,gpt-5.4,gemini-3.1-pro
|
||||
--v3-cells --N 400 --parallel-concurrency 2 --seed 42
|
||||
|
||||
# ── Code freeze assertions ──────────────────────────────────────────────────
|
||||
|
||||
code_freeze:
|
||||
head: 373516c2784807da8536dbc0c194c54f4e4cd4be
|
||||
branch: feature/c3-v3-wrapper
|
||||
frozen_paths:
|
||||
- benchmarks/harness/src/cells.ts
|
||||
- benchmarks/harness/src/substrate.ts
|
||||
- benchmarks/harness/src/judge-client.ts
|
||||
- benchmarks/harness/src/judge-runner.ts
|
||||
- benchmarks/harness/src/health-check.ts
|
||||
- benchmarks/harness/src/streak-tracker.ts
|
||||
- benchmarks/harness/src/runner-lock.ts
|
||||
- benchmarks/harness/src/runner.ts
|
||||
- benchmarks/harness/config/models.json
|
||||
- packages/agent/src/agent-loop.ts
|
||||
- packages/agent/src/tools.ts
|
||||
- packages/core/src/mind/search.ts
|
||||
- packages/core/src/mind/frames.ts
|
||||
- packages/core/src/mind/sessions.ts
|
||||
- packages/core/src/mind/db.ts
|
||||
- litellm-config.yaml
|
||||
permitted_delta_during_run:
|
||||
- "new JSONL files emitted to benchmarks/results/ by the N=400 run"
|
||||
- "no other file modifications"
|
||||
|
||||
# ── Deviation policy ────────────────────────────────────────────────────────
|
||||
|
||||
deviation_policy:
|
||||
on_detection:
|
||||
- immediate_halt
|
||||
- pm_raise
|
||||
- re_preregister_new_manifest_v5_if_accepted
|
||||
consistency_with_bench_spec_lock_v1: preregistration.mid_run_amendment_policy == halt_restart_required
|
||||
|
||||
# ── PM gates ────────────────────────────────────────────────────────────────
|
||||
|
||||
pm_gates:
|
||||
gate_p_pre_run:
|
||||
trigger: "anchor commit of manifest v4 md + yaml on feature/c3-v3-wrapper"
|
||||
action: "CC-1 halts; awaits PM GO on pre-registration content before N=400 kick"
|
||||
pm_checks:
|
||||
- "pre-registration content matches Stage 2-Retry Gate C ratifications"
|
||||
- "§1-§10 locked unambiguously for Gate D adjudication"
|
||||
outcomes:
|
||||
- GO: "§1.2 N=400 kickoff authorized"
|
||||
- REVISE: "Gate P loop"
|
||||
gate_d_post_run:
|
||||
trigger: "N=400 run exit (clean completion or halted per stopping_rules)"
|
||||
action: "CC-1 writes Gate D exit report at PM-Waggle-OS/sessions/2026-04-24-task25-stage3-n400-complete.md; halts"
|
||||
pm_checks:
|
||||
- "primary endpoint pass/fail (Fisher one-sided p < 0.10 on retrieval - no-context ≥ 5pp)"
|
||||
- "secondary endpoint summary (monotonicity chain, agentic lift, abstain penalty)"
|
||||
- "deviation count (expected: 0)"
|
||||
- "budget usage vs $28 hard halt / $30 cap"
|
||||
- "evaluator-loss count"
|
||||
outcomes:
|
||||
- compose_sota_claim_authority_pm
|
||||
- publish_gate
|
||||
- further_scope_matched_mem0_or_multi_model
|
||||
cc1_self_advance: forbidden_at_both_gates
|
||||
|
||||
# ── Scope boundaries ────────────────────────────────────────────────────────
|
||||
|
||||
scope_boundaries:
|
||||
claimable_at_gate_d:
|
||||
- memory_lift_magnitude_and_significance_conv_scope_qwen_harness_head_373516c
|
||||
- per_cell_judge_accuracy_wilson_95
|
||||
- monotonicity_chain_observation_5_cell
|
||||
- conv_scope_fair_comparison_methodology
|
||||
- agentic_discipline_search_rate_turns_unknown_fallback
|
||||
not_claimable_at_gate_d:
|
||||
- direct_comparability_to_mem0_91_6_different_scope_and_memory_layer
|
||||
- multi_model_generalization_stage_3_is_qwen_only
|
||||
- production_waggle_orchestrator_performance
|
||||
reserved_for_pm_at_gate_d:
|
||||
- public_claim_phrasing_venue
|
||||
- matched_scope_mem0_co_run_stage_4
|
||||
- publication_timing
|
||||
cc1_does_not_compose_public_sota_claim: true
|
||||
|
||||
# ── Related artefacts ───────────────────────────────────────────────────────
|
||||
|
||||
related:
|
||||
bench_spec_lock_v1_parent: PM-Waggle-OS/decisions/2026-04-22-bench-spec-locked.manifest.yaml
|
||||
stage_2_retry_gate_c_exit: PM-Waggle-OS/sessions/2026-04-24-task25-stage2-retry-complete.md
|
||||
stage_3_brief: PM-Waggle-OS/briefs/2026-04-24-cc-task25-stage3-n400-kickoff.md
|
||||
stage_2_retry_commit: 373516c
|
||||
rollback_tag: checkpoint/pre-self-evolution-2026-04-14
|
||||
canonical_md_surface: benchmarks/results/manifest-v4-preregistration.md
|
||||
|
||||
# ── Validation gates inherited from Bench-Spec LOCK v1 ──────────────────────
|
||||
|
||||
validation_gates:
|
||||
before_n400_kickoff:
|
||||
- anchor_commit_sha_recorded
|
||||
- manifest_yaml_sha256_recorded_in_commit_message
|
||||
- pm_gate_p_ratification_received
|
||||
- health_check_green_on_subject_plus_3_judges
|
||||
- no_concurrent_runner
|
||||
at_gate_d_exit:
|
||||
- all_2000_evals_accounted_in_denominators
|
||||
- evaluator_loss_reported_separately
|
||||
- primary_fisher_one_sided_computed
|
||||
- secondary_endpoints_reported
|
||||
- budget_reconciled
|
||||
- deviation_count_reported
|
||||
- code_freeze_reverified_head_373516c
|
||||
Reference in New Issue
Block a user