This commit is contained in:
1
gepa-phase-5/canary-kickoff.jsonl
Normal file
1
gepa-phase-5/canary-kickoff.jsonl
Normal file
@@ -0,0 +1 @@
|
||||
{"ts":"2026-04-29T13:51:57Z","event":"canary_kickoff_day_0","canary_pct":10,"variants_in_scope":["claude::gen1-v1","qwen-thinking::gen1-v1"],"variants_withheld":["gpt::gen1-v2"],"manifest_pin_sha":"6bc20897d3851072eda34e80070faf39772bee66","manifest_pin_short":"6bc2089","manifest_pin_label":"Faza 1 Checkpoint C closure (15/15 held-out + manifest v7 11 amendments terminus)","implementation_authoring_sha":"19152cf59cbca5e68ce8faacd428166d747cdde4","implementation_authoring_short":"19152cf","pre_canary_kickoff_sha":"19152cf59cbca5e68ce8faacd428166d747cdde4","pre_canary_kickoff_short":"19152cf","branch":"phase-5-deployment-v2","authority_memo":"D:/Projects/PM-Waggle-OS/decisions/2026-04-30-phase-5-1-5-pm-signoff-canary-authorize.md","authority_dated":"2026-04-30","brief":"D:/Projects/PM-Waggle-OS/briefs/2026-04-29-phase-5-deployment-brief-v1.md","cost_amendment":"D:/Projects/PM-Waggle-OS/decisions/2026-04-30-phase-5-cost-amendment-LOCKED.md","cost_hard_cap_usd":75,"cost_halt_trigger_usd":60,"cost_to_date_usd":0.1628,"observation_window_hours":24,"next_target_canary_pct":25,"promotion_and_gate":"max(7_days_kickoff_offset, 30_samples_per_metric_per_variant)","halt_authority":["§4.2_automatic_thresholds","PM_manual","Marko_manual","CC_observed_anomaly"],"rollback_path":"set WAGGLE_PHASE5_CANARY_PCT=0 → process restart (no redeploy needed) per §2.3","monitoring_emit_root":"gepa-phase-5/monitoring/<ISO_date>/<variant>.jsonl","alerts_emit_root":"gepa-phase-5/phase-5-alerts/<ISO_date>.jsonl","daily_summary_emit_root":"gepa-phase-5/phase-5-daily-summary/<ISO_date>.md","halt_pm_stderr_signal":"PHASE5-ROLLBACK-TRIGGER <ts> <trigger_id> <variant> <metric>=<observed> threshold=<threshold>","wiring_commit_target":"feat(phase-5): canary kick-off Day 0 — pickShape→routeRequestToVariant + canary_pct default 10 + manifest SHA pin"}
|
||||
33
gepa-phase-5/cost-probe-2026-04-29-summary.md
Normal file
33
gepa-phase-5/cost-probe-2026-04-29-summary.md
Normal file
@@ -0,0 +1,33 @@
|
||||
# Phase 5 §0.3 Cost Probe Summary
|
||||
|
||||
**Date:** 2026-04-29T11:19:11.627Z
|
||||
**Pricing snapshot:** 2026-04-29
|
||||
**Branch:** phase-5-deployment-v2
|
||||
**Endpoint:** http://localhost:4000 (LiteLLM proxy, matches Faza 1 runner pattern)
|
||||
|
||||
## Per-variant statistics
|
||||
|
||||
| Variant | Model alias | OK | Errors | p50 | p95 | max | mean | total |
|
||||
|---|---|---|---|---|---|---|---|---|
|
||||
| claude::gen1-v1 | claude-opus-4-7 | 5 | 0 | $0.0239 | $0.0432 | $0.0432 | $0.0250 | $0.1251 |
|
||||
| qwen-thinking::gen1-v1 | qwen3.6-35b-a3b-via-dashscope-direct | 5 | 0 | $0.0064 | $0.0176 | $0.0176 | $0.0075 | $0.0377 |
|
||||
|
||||
## Canary cost ceiling (per brief §5.4)
|
||||
|
||||
Formula: `canary_cost_p95_ceiling = 740 requests × max(p95) × 1.2 (buffer)`
|
||||
|
||||
max(p95) = $0.0432
|
||||
canary_cost_p95_ceiling = 740 × $0.0432 × 1.2 = **$38.34**
|
||||
|
||||
## Verdict
|
||||
|
||||
**HARD-CAP-EXCEED** — ceiling $38.34 > hard cap $25.
|
||||
|
||||
## Probe spend (this script)
|
||||
|
||||
Total: **$0.1628**
|
||||
Brief §5.4 probe budget: $0.30-$0.50.
|
||||
|
||||
## JSONL anchor
|
||||
|
||||
Per-request rows: `gepa-phase-5/cost-probe-2026-04-29.jsonl`
|
||||
10
gepa-phase-5/cost-probe-2026-04-29.jsonl
Normal file
10
gepa-phase-5/cost-probe-2026-04-29.jsonl
Normal file
@@ -0,0 +1,10 @@
|
||||
{"variant":"claude::gen1-v1","complexity":"trivial","model_alias":"claude-opus-4-7","input_tokens":638,"output_tokens":39,"cost_usd":0.004165,"latency_ms":2404,"ts":"2026-04-29T11:15:29.751Z"}
|
||||
{"variant":"claude::gen1-v1","complexity":"medium-1","model_alias":"claude-opus-4-7","input_tokens":816,"output_tokens":600,"cost_usd":0.01908,"latency_ms":11367,"ts":"2026-04-29T11:15:41.119Z"}
|
||||
{"variant":"claude::gen1-v1","complexity":"medium-2","model_alias":"claude-opus-4-7","input_tokens":787,"output_tokens":800,"cost_usd":0.023935,"latency_ms":15666,"ts":"2026-04-29T11:15:56.786Z"}
|
||||
{"variant":"claude::gen1-v1","complexity":"complex","model_alias":"claude-opus-4-7","input_tokens":953,"output_tokens":1200,"cost_usd":0.034765,"latency_ms":21553,"ts":"2026-04-29T11:16:18.341Z"}
|
||||
{"variant":"claude::gen1-v1","complexity":"stretch","model_alias":"claude-opus-4-7","input_tokens":1135,"output_tokens":1500,"cost_usd":0.043175,"latency_ms":27587,"ts":"2026-04-29T11:16:45.930Z"}
|
||||
{"variant":"qwen-thinking::gen1-v1","complexity":"trivial","model_alias":"qwen3.6-35b-a3b-via-dashscope-direct","input_tokens":152,"output_tokens":950,"cost_usd":0.001938,"latency_ms":7596,"ts":"2026-04-29T11:16:53.528Z"}
|
||||
{"variant":"qwen-thinking::gen1-v1","complexity":"medium-1","model_alias":"qwen3.6-35b-a3b-via-dashscope-direct","input_tokens":277,"output_tokens":2278,"cost_usd":0.00462525,"latency_ms":17009,"ts":"2026-04-29T11:17:10.538Z"}
|
||||
{"variant":"qwen-thinking::gen1-v1","complexity":"medium-2","model_alias":"qwen3.6-35b-a3b-via-dashscope-direct","input_tokens":237,"output_tokens":3189,"cost_usd":0.00643725,"latency_ms":24995,"ts":"2026-04-29T11:17:35.534Z"}
|
||||
{"variant":"qwen-thinking::gen1-v1","complexity":"complex","model_alias":"qwen3.6-35b-a3b-via-dashscope-direct","input_tokens":350,"output_tokens":3496,"cost_usd":0.0070795,"latency_ms":29499,"ts":"2026-04-29T11:18:05.034Z"}
|
||||
{"variant":"qwen-thinking::gen1-v1","complexity":"stretch","model_alias":"qwen3.6-35b-a3b-via-dashscope-direct","input_tokens":506,"output_tokens":8752,"cost_usd":0.0176305,"latency_ms":66591,"ts":"2026-04-29T11:19:11.626Z"}
|
||||
120
gepa-phase-5/cross-stream.md
Normal file
120
gepa-phase-5/cross-stream.md
Normal file
@@ -0,0 +1,120 @@
|
||||
# Phase 5 §5 Cross-Stream Dependencies
|
||||
|
||||
**Date:** 2026-04-30
|
||||
**Status:** Declared per `feedback_waggle_primary_framing` (NEW 2026-04-30)
|
||||
**Manifest:** `gepa-phase-5/manifest.yaml` § cross_stream_primary + § cross_stream_secondary
|
||||
**Brief:** `D:/Projects/PM-Waggle-OS/briefs/2026-04-29-phase-5-deployment-brief-v1.md` §6
|
||||
|
||||
This doc enumerates the cross-stream destinations for Phase 5 production-stable evidence in **primary** vs **secondary** order per the binding feedback rule. Phase 5 production-stable is a **Waggle-launch-readiness gate** with KVARK as a downstream beneficiary.
|
||||
|
||||
---
|
||||
|
||||
## Strategic framing (binding rule)
|
||||
|
||||
`feedback_waggle_primary_framing`:
|
||||
|
||||
> Waggle is the demand-creation engine for KVARK (per CLAUDE.md §1 strategic function). KVARK without Waggle has no funnel; the strategic order is Waggle-first because Waggle drives the inbound that qualifies into KVARK. Reversing primary/secondary in deployment narratives confuses positioning and risks framing the sovereign-enterprise tier as the entry point rather than the upgrade path.
|
||||
|
||||
**Bound:** any brief that inverts primary/secondary order (KVARK first, Waggle second) flagged for amendment. Phase 5 brief §6 lists the destinations but does not explicitly tag primary/secondary; this doc closes that authoring gap by declaring the order.
|
||||
|
||||
---
|
||||
|
||||
## Primary cross-stream destinations (Waggle-launch-readiness)
|
||||
|
||||
### P1 — Waggle landing v3 Proof Card 1 swap
|
||||
|
||||
| Field | Value |
|
||||
|---|---|
|
||||
| Brief anchor | §6.1 |
|
||||
| Trigger condition | `phase_5_production_stable + Pass II numbers + retrieval engagement validated samples` |
|
||||
| Action | Update Landing v2.3 Proof Card 1 sa **PROVENANCE → BENCHMARK** swap (concrete numbers from Phase 5) |
|
||||
| Owner | PM authoring + claude.ai/design generation |
|
||||
| Timing post-production-stable | 2-3 weeks |
|
||||
| Additional audit anchor | `project_landing_v2_basics_2026_04_28` |
|
||||
|
||||
### P2 — Waggle launch comms
|
||||
|
||||
| Field | Value |
|
||||
|---|---|
|
||||
| Brief anchor | §6.1 implicit extension (brief lists §6.1-§6.4 but doesn't separately enumerate launch comms — declared here per `feedback_waggle_primary_framing`) |
|
||||
| Trigger condition | `phase_5_production_stable` |
|
||||
| Action | Public launch comms — Waggle production-stable evidence as anchor; KVARK pitch is downstream consumption of the same evidence |
|
||||
| Owner | Marko + PM |
|
||||
| Timing post-production-stable | Aligned with landing v3 swap |
|
||||
|
||||
### P3 — arxiv §5.1-§5.2 (methodology + Faza 1 results)
|
||||
|
||||
| Field | Value |
|
||||
|---|---|
|
||||
| Brief anchor | §6.2 (§5.1 + §5.2 portion only — §5.3 KVARK enterprise narrative is **secondary**, see S2 below) |
|
||||
| Trigger condition | `phase_5_production_stable + Marko_arxiv_co_author_roster_ratification + endorsement_contact` |
|
||||
| Action | arxiv §5.1 (methodology + manifest v7 + 11 amendments transparent narrative) + §5.2 (Faza 1 results: cross-family generalization, claude + qwen-thinking pass; gpt selection bias scoping) — Waggle-primary academic credibility |
|
||||
| Owner | Marko authoring |
|
||||
| Timing post-production-stable | TBD |
|
||||
|
||||
---
|
||||
|
||||
## Secondary cross-stream destinations (KVARK-downstream)
|
||||
|
||||
### S1 — KVARK enterprise pitch deck
|
||||
|
||||
| Field | Value |
|
||||
|---|---|
|
||||
| Brief anchor | §6.3 |
|
||||
| Trigger condition | `phase_5_production_stable + Marko_pitch_deck_draft_approval` |
|
||||
| Action | KVARK enterprise pitch deck section: "On-prem Qwen + Waggle harness validated Opus-class on real production traffic, X% retrieval parity, Y% Pass II floor" |
|
||||
| Owner | Marko |
|
||||
| Timing post-production-stable | TBD post landing v3 swap |
|
||||
| Relationship to primary | Downstream beneficiary of landing v3 evidence (consumes same Pass II + retrieval numbers) |
|
||||
|
||||
### S2 — arxiv §5.3 KVARK enterprise narrative
|
||||
|
||||
| Field | Value |
|
||||
|---|---|
|
||||
| Brief anchor | §6.2 §5.3-only portion |
|
||||
| Trigger condition | `phase_5_production_stable` |
|
||||
| Action | arxiv §5.3 KVARK enterprise narrative — Qwen 35B = Opus-class on production traffic = KVARK pitch scientific anchor |
|
||||
| Relationship to primary | Downstream beneficiary of §5.1 + §5.2 (consumes same methodology + results sections) |
|
||||
|
||||
### S3 — Faza 2 sprint planning
|
||||
|
||||
| Field | Value |
|
||||
|---|---|
|
||||
| Brief anchor | §6.4 |
|
||||
| Trigger condition | `phase_5_production_stable + 2 weeks observation` |
|
||||
| Scope | gpt::gen1-v2 N=16 re-validation; qwen-non-thinking deeper investigation (retrieval-quality decoupling characterization); generic-simple investigation (necessary-but-not-sufficient retrieval scoping); arxiv §5.4 finalization |
|
||||
| Budget headroom | $71.51 USD (decoupled from Phase 5 cap; Faza 1 unspent of $115 cap) |
|
||||
| Timing post-production-stable | 2 weeks observation, then Faza 2 sprint kick-off |
|
||||
| Relationship to primary | Methodology continuation; not Waggle-launch trigger directly |
|
||||
|
||||
---
|
||||
|
||||
## Sequencing implications
|
||||
|
||||
If multiple primary destinations are time-pressured:
|
||||
|
||||
1. **Always P1 first** (landing v3 swap is the customer-facing surface).
|
||||
2. **P2 in parallel** (launch comms can author against the same evidence as P1).
|
||||
3. **P3 follows** (arxiv has longest authoring timeline; doesn't gate launch).
|
||||
|
||||
Secondary destinations cannot pre-empt primary:
|
||||
|
||||
- **S1 KVARK pitch deck draft** can begin authoring during P1-P2-P3 work, but cannot precede them publicly.
|
||||
- **S2 arxiv §5.3** authored alongside P3 §5.1-§5.2 (same artifact), but the §5.3 KVARK section never carries primary-narrative weight.
|
||||
- **S3 Faza 2 planning** is post-launch methodology continuation; sequenced after the 2-week observation window.
|
||||
|
||||
---
|
||||
|
||||
## Anti-pattern guards
|
||||
|
||||
Per `feedback_waggle_primary_framing` "How to apply":
|
||||
|
||||
- ❌ "Phase 5 production-stable is a KVARK-readiness gate" — wrong. It is a Waggle-launch-readiness gate.
|
||||
- ❌ KVARK pitch deck listed before Waggle landing v3 in cross-stream dependency tables — flag for amendment.
|
||||
- ❌ arxiv §5.3 weight pitched as primary academic narrative — §5.3 is a downstream lift; §5.1 + §5.2 are primary.
|
||||
- ✅ Primary order: Waggle landing v3 → Waggle launch comms → arxiv §5.1-§5.2.
|
||||
- ✅ Secondary order: KVARK pitch deck → arxiv §5.3 → Faza 2 sprint planning.
|
||||
|
||||
---
|
||||
|
||||
**End of cross-stream declaration. Manifest § cross_stream_primary + § cross_stream_secondary mirror this doc; if drift detected, manifest is canonical.**
|
||||
88
gepa-phase-5/exit-criteria-coverage.md
Normal file
88
gepa-phase-5/exit-criteria-coverage.md
Normal file
@@ -0,0 +1,88 @@
|
||||
# Phase 5 §4 Exit Criteria — Coverage Map
|
||||
|
||||
**Date:** 2026-04-30
|
||||
**Status:** §4 BIND — pre-registered, no-revisit-without-amendment per brief §4.4
|
||||
**Manifest:** `gepa-phase-5/manifest.yaml`
|
||||
**Brief:** `D:/Projects/PM-Waggle-OS/briefs/2026-04-29-phase-5-deployment-brief-v1.md` §4
|
||||
**Monitoring code:** `packages/agent/src/canary/phase-5-monitoring.ts`
|
||||
|
||||
This doc traces every brief §4 threshold to its manifest field + monitoring code reference. Any drift between brief / manifest / code is an authoring bug and must be reconciled before canary kick-off.
|
||||
|
||||
---
|
||||
|
||||
## §4.1 — Promotion criteria (canary → full enable)
|
||||
|
||||
Brief §4.1 binds **5 conditions + wall-clock floor** with ε = 1e-9 inclusive boundary.
|
||||
|
||||
| # | Condition | Brief §4.1 wording | Manifest field | Code reference |
|
||||
|---|---|---|---|---|
|
||||
| 1 | Pass II rate | `variant_pass_ii ≥ baseline + 0pp − ε`; stretch goal +5pp not blocking | `promotion_criteria.criteria[0].condition` + `inclusive_boundary_epsilon: 1.0e-9` | `PROMOTION_THRESHOLDS.pass_ii_delta_pp = 0` + `inclusive_boundary_epsilon = 1e-9` |
|
||||
| 2a | Retrieval engagement (qwen-thinking) | `variant_retrieval ≥ baseline_retrieval × 0.80` | `promotion_criteria.criteria[1].condition_qwen_thinking` | `PROMOTION_THRESHOLDS.retrieval_qwen_thinking_multiplier = 0.80` |
|
||||
| 2b | Retrieval engagement (claude) | `variant_retrieval ≥ baseline_retrieval` | `promotion_criteria.criteria[1].condition_claude` | `PROMOTION_THRESHOLDS.retrieval_claude_multiplier = 1.0` |
|
||||
| 3 | Latency p95 | `variant_p95 ≤ baseline_p95 × 1.20` | `promotion_criteria.criteria[2].condition` | `PROMOTION_THRESHOLDS.latency_p95_multiplier = 1.20` |
|
||||
| 4 | Cost per request | `variant_cost ≤ baseline_cost × 1.15` | `promotion_criteria.criteria[3].condition` | `PROMOTION_THRESHOLDS.cost_per_request_multiplier = 1.15` |
|
||||
| 5 | Error rate | `variant_error ≤ baseline_error + 1pp` | `promotion_criteria.criteria[4].condition` | `PROMOTION_THRESHOLDS.error_rate_delta_pp = 1` |
|
||||
| floor | Wall-clock | `max(7_days_since_canary_kickoff, 30_samples_per_metric)` ⚠️ brief §4.1 says `min(...)` — flagged as authoring typo | `promotion_criteria.wall_clock_floor.formula` (binds `max()` per brief §2.2 + intent) + `canonical_intent` field documenting the discrepancy | `PROMOTION_THRESHOLDS.sample_floor_per_metric = 30` + `days_min = 7` |
|
||||
|
||||
**Inconsistency flag:** Brief §2.2 says `max(7_days_floor, 30_samples_floor)` (correct: both must hold). Brief §4.1 says `min(...)` (incorrect: would allow promotion at 7 days even with < 30 samples). Manifest binds `max()` per intent ("sample floor je hard guard protiv small-sample-effect stage promotion"). PM correction recommended in brief §4.1 wording before canary kick-off.
|
||||
|
||||
---
|
||||
|
||||
## §4.2 — Rollback triggers (immediate)
|
||||
|
||||
Brief §4.2 binds **5 triggers** plus an Opcija C-specific 6th (per `decisions/2026-04-30-branch-architecture-opcija-c.md` §3 + manifest § rollback_triggers.opcija_c_long_task_loop_exhausted).
|
||||
|
||||
| # | Trigger | Brief §4.2 wording | Manifest field | Code reference (`packages/agent/src/canary/phase-5-monitoring.ts`) |
|
||||
|---|---|---|---|---|
|
||||
| 1 | Pass II rate collapse | `variant < baseline − 10pp` (consecutive 2 windows of 10-sample each) | `rollback_triggers.triggers[0]` | `ROLLBACK_THRESHOLDS.pass_ii_collapse_pp = -10`, `pass_ii_consecutive_windows = 2`, `pass_ii_window_size = 10` + `checkPassIIRateCollapse()` |
|
||||
| 2 | Error rate spike | `variant > baseline + 5pp` (consecutive 24h) | `rollback_triggers.triggers[1]` | `ROLLBACK_THRESHOLDS.error_rate_spike_pp = 5`, `error_consecutive_window_hours = 24` + `checkErrorRateSpike()` |
|
||||
| 3 | Cost per request spike | `variant > baseline × 2.0` (immediate single-window) | `rollback_triggers.triggers[2]` | `ROLLBACK_THRESHOLDS.cost_per_request_multiplier = 2.0` + `checkSingleEventRollback()` for `metricName === 'cost_usd'` |
|
||||
| 4 | Latency p95 spike | `variant > baseline × 3.0` (immediate single-window) | `rollback_triggers.triggers[3]` | `ROLLBACK_THRESHOLDS.latency_p95_multiplier = 3.0` + `checkSingleEventRollback()` for `metricName === 'latency_ms'` |
|
||||
| 5 | Manual halt | PM, Marko, or CC observed anomaly outside thresholds | `rollback_triggers.triggers[4]` (`manual_halt_or_pm`) | Out-of-band: PM + Marko discretion; CC emits halt-and-PM via `emitAlert` with `is_rollback_trigger: true` |
|
||||
| 6 | Opcija C long-task | `loop_exhausted_rate > 5% baseline` → halt diagnostic "long-task fixes potrebni" + cherry-pick option | `rollback_triggers.triggers[5]` (`opcija_c_long_task_loop_exhausted`) including cherry_pick_candidate_set commits | `ROLLBACK_THRESHOLDS.opcija_c_loop_exhausted_rate_pct = 5` + `checkLoopExhaustedRate()` (diagnostic includes `c9bda3d, be8f702, e906114, 4d0542f, 8b8a940`) |
|
||||
|
||||
All rollback alerts emit `PHASE5-ROLLBACK-TRIGGER` stderr line for halt-and-PM automation hook.
|
||||
|
||||
---
|
||||
|
||||
## §4.3 — Production-stable definition
|
||||
|
||||
Brief §4.3 binds **3 conditions** for "production-stable" status that unblocks §6 cross-stream dependencies.
|
||||
|
||||
| # | Condition | Brief §4.3 wording | Manifest field |
|
||||
|---|---|---|---|
|
||||
| 1 | Days zero rollback | 30 days zero rollback events | `production_stable_status.conditions.days_zero_rollback: 30` |
|
||||
| 2 | Metrics within pass bands | All 5 §4.1 metrics maintained within pass bands | `production_stable_status.conditions.metrics_within_pass_bands_count: 5` |
|
||||
| 3 | Halt-and-PM emissions | Zero halt-and-PM trigger emissions | `production_stable_status.conditions.halt_and_pm_emissions: 0` |
|
||||
|
||||
Production-stable status `unblocks: cross_stream_dependencies` per manifest § production_stable_status.unblocks.
|
||||
|
||||
---
|
||||
|
||||
## §4.4 — No-revisit-without-amendment binding
|
||||
|
||||
Brief §4.4 binds: "Sve §4.1, §4.2, §4.3 thresholdi su LOCKED. Ako tokom Phase 5 CC discover-uje da neki threshold treba relaxation ili tightening, halt-and-PM proceduru sa amendment proposal. Marko ratifikuje, novi LOCKED memo, restart canary phase ako threshold change znači redo."
|
||||
|
||||
**Bound by manifest:**
|
||||
- `promotion_criteria.binding_rule: no_revisit_without_amendment`
|
||||
- `rollback_triggers.binding_rule: automatic_rollback_per_section_2_3`
|
||||
- Audit anchor: `audit_anchors.brief_LOCKED` + `audit_anchors.scope_LOCKED` + `audit_anchors.cost_amendment_LOCKED`
|
||||
|
||||
**Bound by code:** ROLLBACK_THRESHOLDS + PROMOTION_THRESHOLDS are `as const` frozen registries; mid-flight code mutation would require source-edit + commit (visible in audit trail).
|
||||
|
||||
**Precedent:** Faza 1 Amendment 11 terminal_calibration_clause — strict pre-registration discipline applies.
|
||||
|
||||
---
|
||||
|
||||
## Cost cap discipline (related: §5.4 amendment)
|
||||
|
||||
Per `feedback_production_vs_research_cost_discipline` (NEW 2026-04-30):
|
||||
|
||||
- Cost cap (§5.4) is **production deployment** regime — operational projection, amendable via PM decision memo + ratification (Phase 5 amendment $25→$75 precedent).
|
||||
- §4.1-§4.4 thresholds are **methodology binding** — strict no-revisit-without-amendment per Faza 1 Amendment 11 precedent.
|
||||
|
||||
Two regimes; only the cost cap was amended 2026-04-30 (per `decisions/2026-04-30-phase-5-cost-amendment-LOCKED.md`). All §4 promotion + rollback thresholds remain LOCKED at brief §4.1-§4.3 values.
|
||||
|
||||
---
|
||||
|
||||
**End of coverage map. All 5 promotion criteria + 6 rollback triggers + 3 production-stable conditions captured in manifest + code with explicit cross-references.**
|
||||
419
gepa-phase-5/manifest.yaml
Normal file
419
gepa-phase-5/manifest.yaml
Normal file
@@ -0,0 +1,419 @@
|
||||
# Phase 5 deployment manifest — GEPA-evolved variants production canary
|
||||
# Inherits Faza 1 manifest v7 LOCKED scope (claude::gen1-v1 + qwen-thinking::gen1-v1)
|
||||
# Authority: PM (Marko Markovic) — brief LOCKED 2026-04-29; cost amendment LOCKED 2026-04-30
|
||||
# Drives: §2 canary toggle + §3 monitoring + §4 exit criteria coverage + §5 cross-stream
|
||||
|
||||
manifest_version: phase-5-deployment-v1
|
||||
manifest_type: production_canary_deployment
|
||||
locked_date: 2026-04-29
|
||||
cost_amendment_locked_date: 2026-04-30
|
||||
authority: |
|
||||
PM (Marko Markovic).
|
||||
Brief ratified "sve ok idemo dalje" 2026-04-29.
|
||||
Cost cap amendment ratified "stavi visi slobodno" 2026-04-30.
|
||||
sprint: phase_5_canary
|
||||
task: variant_canary_deployment
|
||||
phase: phase_5_canary
|
||||
branch: phase-5-deployment-v2
|
||||
substrate_anchor_short: 6bc2089
|
||||
implementation_authoring_sha: 19152cf59cbca5e68ce8faacd428166d747cdde4 # short=19152cf — final §1-§5 implementation commit "docs(phase-5): §4 exit criteria coverage map + §5 cross-stream Waggle-primary declaration"
|
||||
pre_canary_kickoff_sha: 19152cf59cbca5e68ce8faacd428166d747cdde4 # short=19152cf — HEAD parent of canary kick-off commit; rollback target per §2.3 procedure
|
||||
canary_kickoff_locked_date: 2026-04-30 # Day 0 LIVE — PM signoff D:/Projects/PM-Waggle-OS/decisions/2026-04-30-phase-5-1-5-pm-signoff-canary-authorize.md
|
||||
supersedes: NONE # First Phase 5 manifest
|
||||
|
||||
# ── Inherits ────────────────────────────────────────────────────────────────
|
||||
|
||||
inherits_from:
|
||||
- manifest_v7_gepa_faza1_2026_04_28_substrate_freeze_c9bda3d
|
||||
- faza_1_closure_2026_04_29_manifest_terminus_6bc2089
|
||||
- phase_5_brief_LOCKED_2026_04_29
|
||||
- phase_5_scope_LOCKED_2026_04_29
|
||||
- branch_architecture_opcija_c_2026_04_30
|
||||
- phase_5_cost_amendment_LOCKED_2026_04_30
|
||||
|
||||
# ── Substrate anchor ────────────────────────────────────────────────────────
|
||||
|
||||
substrate_anchor:
|
||||
branch: gepa-faza-1
|
||||
also_pinned_by: phase-5-deployment-v2 # Phase 5 branch HEAD = 6bc2089 at preflight time
|
||||
commit_sha: 6bc20897d3851072eda34e80070faf39772bee66
|
||||
commit_sha_short: 6bc2089
|
||||
phase_label: "Faza 1 Checkpoint C closure (held-out validation 15/15 + manifest v7 11 amendments terminus)"
|
||||
pin_method: branch_pointer + reachability_proof
|
||||
ancestry_verification_command: "git merge-base --is-ancestor 6bc2089 HEAD"
|
||||
ancestry_verification_result: PASS # verified §0.1 #4 in preflight-evidence.md
|
||||
derived_variants:
|
||||
- name: claude::gen1-v1
|
||||
file: packages/agent/src/prompt-shapes/gepa-evolved/claude-gen1-v1.ts
|
||||
shape_constant: claudeGen1V1Shape
|
||||
shape_name: claude-gen1-v1
|
||||
registration: "registerShape('claude::gen1-v1', claudeGen1V1Shape)"
|
||||
default_max_tokens: 4096
|
||||
substrate_evidence:
|
||||
pass_ii_in_sample_n8: "100%"
|
||||
pass_ii_held_out_n5: "100%"
|
||||
pass_ii_combined_n13: "100%"
|
||||
held_out_gap_pp: 0
|
||||
in_sample_uplift_vs_baseline_pp: 12.5
|
||||
f5_cond_2_overfitting_bound_check: PASS # 0pp gap < ±15pp bound
|
||||
- name: qwen-thinking::gen1-v1
|
||||
file: packages/agent/src/prompt-shapes/gepa-evolved/qwen-thinking-gen1-v1.ts
|
||||
shape_constant: qwenThinkingGen1V1Shape
|
||||
shape_name: qwen-thinking-gen1-v1
|
||||
registration: "registerShape('qwen-thinking::gen1-v1', qwenThinkingGen1V1Shape)"
|
||||
default_max_tokens: 16000
|
||||
substrate_evidence:
|
||||
pass_ii_in_sample_uplift_pp: 12.5
|
||||
held_out_gap_pp: 0
|
||||
retrieval_engagement_mean: 2.231 # = 96% of Opus parity 2.33
|
||||
opus_parity_pct: 96
|
||||
phase_4_5_mechanism_status: CONFIRMED_OUT_OF_DISTRIBUTION
|
||||
|
||||
# ── Scope (LOCKED, no-substitution rule) ────────────────────────────────────
|
||||
|
||||
scope_LOCKED:
|
||||
in_scope:
|
||||
- claude::gen1-v1
|
||||
- qwen-thinking::gen1-v1
|
||||
withheld:
|
||||
- name: gpt::gen1-v2
|
||||
reason: "Selection-biased on in-sample (in-sample +25pp → held-out +5pp = 20pp gap > ±15pp overfitting bound). WITHHELD until Faza 2 N=16 re-validation."
|
||||
audit_chain_anchor: gpt-gen1-v2_failure_methodology_working_as_designed
|
||||
no_substitution_rule: |
|
||||
Mid-flight scope changes (add gpt, swap variants, drugi Generation) require
|
||||
new LOCKED decision memo + Marko ratifikacija. CC must halt-and-PM if
|
||||
discovery suggests scope changes. Per Faza 1 Amendment 11 terminal_calibration_clause
|
||||
precedent.
|
||||
scope_LOCKED_anchor: D:/Projects/PM-Waggle-OS/decisions/2026-04-29-phase-5-scope-LOCKED.md
|
||||
|
||||
# ── Config differential vs Faza 1 manifest v7 (§0.2 PASS) ────────────────────
|
||||
|
||||
config_differential:
|
||||
inheritance_rule: "Production deployment task-type ≠ Faza 1 evolution task-type. Implicit defaults FORBIDDEN; every field below carries explicit value + justification."
|
||||
audit_anchor: gepa-phase-5/preflight-evidence.md#§0.2
|
||||
fields:
|
||||
temperature:
|
||||
faza_1_value: "agent default 0.3; corpus gen 0.7; ground truth 0.5; judge primary 1.0; oracle 1.0"
|
||||
phase_5_value: 0.3
|
||||
justification: "Production-default 0.3 inherits agent-side default. No corpus generation or judging in Phase 5. May be re-tuned post-canary if probe surfaces variant-specific recommendation."
|
||||
max_tokens:
|
||||
faza_1_value: "judge 3000, agent default 16000, oracle 8000, mutation-oracle response_format 6000"
|
||||
phase_5_value:
|
||||
claude_gen1_v1: 4096
|
||||
qwen_thinking_gen1_v1: 16000
|
||||
justification: "Aligns with Phase 4.5 retrieval engagement finding. Qwen 1.33×/task → target ≥1.7 (thinking budget required). Claude narrative+XML structure is more output-efficient, so lower default."
|
||||
judge_model_primary:
|
||||
faza_1_value: "trio = claude-opus-4-7 + gpt-5.4 + minimax-m27 (κ=0.7878)"
|
||||
phase_5_value: NOT_APPLICABLE
|
||||
justification: "Phase 5 production deployment has no inline judge. §3 monitoring metrics (Pass II rate proxy, retrieval engagement, latency, cost, error rate) replace evaluation judging. Judge invocation deferred to Faza 2 re-validation runs."
|
||||
evaluation_corpus_source:
|
||||
faza_1_value: "50-instance NorthLane CFO synthesis corpus + LoCoMo dataset (held-out 5 instances of slice 8-12 of seed=42 shuffle for Checkpoint C)"
|
||||
phase_5_value: production_live_traffic
|
||||
justification: "Inherent task-type shift. Phase 5 evaluates in-the-wild against actual user prompts; production traffic distribution unknown a priori — §3 monitoring + §4 promotion criteria provide acceptance bounds."
|
||||
failure_mode_taxonomy:
|
||||
faza_1_value: "Amendment 4 texture audit — F1-F5 task family classifier + per-family Pass II rate; F6 = T2 modes"
|
||||
phase_5_value: production_threshold_based_classifier # see rollback_triggers below + opcija_c_addendum
|
||||
justification: "New mapping — production threshold-based detection vs. content-classifier."
|
||||
opcija_c_addendum:
|
||||
added_trigger: loop_exhausted_rate_gt_5pct_baseline
|
||||
rationale: "Phase 4 long-task fixes not inherited per Opcija C §3. Halt with 'long-task fixes potrebni' diagnostic; selective cherry-pick option from feature/c3-v3-wrapper available."
|
||||
cost_per_request_baseline:
|
||||
faza_1_value: "$0.27/instance (Amendment 3 probe-corrected)"
|
||||
phase_5_value:
|
||||
claude_gen1_v1_p50: 0.0239
|
||||
claude_gen1_v1_p95: 0.0432
|
||||
qwen_thinking_gen1_v1_p50: 0.0064
|
||||
qwen_thinking_gen1_v1_p95: 0.0176
|
||||
justification: "§0.3 probe-validated 2026-04-29 via LiteLLM proxy (10 requests). Cost may shift with production cache hit rates and conversation-context size."
|
||||
|
||||
# ── Cost economics (post-amendment LOCKED 2026-04-30) ────────────────────────
|
||||
|
||||
cost_governance:
|
||||
amendment_LOCKED_anchor: D:/Projects/PM-Waggle-OS/decisions/2026-04-30-phase-5-cost-amendment-LOCKED.md
|
||||
cost_regime_classification: production_deployment # per feedback_production_vs_research_cost_discipline
|
||||
hard_cap_usd: 75
|
||||
halt_trigger_usd: 60
|
||||
expected_total_usd_min: 35
|
||||
expected_total_usd_max: 45
|
||||
buffer_above_probe_validated_pct: 96 # = 75 / 38.34
|
||||
probe_validated_p95_ceiling_usd: 38.34
|
||||
probe_validated_anchor:
|
||||
formula: "740 requests × max(p95_per_variant) × 1.20 buffer"
|
||||
computation: "740 × $0.0432 × 1.20 = $38.34"
|
||||
p95_max_variant: claude_gen1_v1
|
||||
probe_artifact: gepa-phase-5/cost-probe-2026-04-29.jsonl
|
||||
probe_summary: gepa-phase-5/cost-probe-2026-04-29-summary.md
|
||||
probe_total_spend_usd: 0.1628
|
||||
pricing_snapshot:
|
||||
snapshot_date: 2026-04-29
|
||||
claude_opus_4_7:
|
||||
input_per_million_tokens: 5.00
|
||||
output_per_million_tokens: 25.00
|
||||
cache_hits_per_million_tokens: 0.50
|
||||
source_url: https://platform.claude.com/docs/en/docs/about-claude/pricing
|
||||
dashscope_qwen_35b_a3b_intl_thinking:
|
||||
input_per_million_tokens: 0.25
|
||||
output_per_million_tokens: 2.00
|
||||
source_url: https://www.alibabacloud.com/help/en/model-studio/billing-for-model-studio
|
||||
cost_dominance_mechanism: |
|
||||
claude::gen1-v1 stretch case (1500 output tokens × Opus 4.7 $25/M output) dominates
|
||||
canary cost ceiling. qwen-thinking::gen1-v1 ~3× cheaper per request even at higher
|
||||
output volume because $2/M output vs $25/M output.
|
||||
amendment_history:
|
||||
- version: v0
|
||||
hard_cap_usd: 25
|
||||
halt_trigger_usd: 20
|
||||
expected_total_usd: [8, 13]
|
||||
authored_in: phase_5_brief_v1_LOCKED_2026_04_29
|
||||
superseded_by: amendment_LOCKED_2026_04_30
|
||||
reason_for_supersession: probe_validated_ceiling_38.34_exceeded_25_hard_cap
|
||||
- version: v1_LOCKED
|
||||
hard_cap_usd: 75
|
||||
halt_trigger_usd: 60
|
||||
expected_total_usd: [35, 45]
|
||||
authored_in: D:/Projects/PM-Waggle-OS/decisions/2026-04-30-phase-5-cost-amendment-LOCKED.md
|
||||
ratification: "Marko: stavi visi slobodno (2026-04-30)"
|
||||
|
||||
# ── Volume estimate (brief §5.2) ─────────────────────────────────────────────
|
||||
|
||||
volume_estimate:
|
||||
canary_phase_total: 740 # across both variants, 7 days
|
||||
per_day_breakdown:
|
||||
day_0_1_canary_10pct: 40 # 10 req/day × 2 variants × 2 days
|
||||
day_1_3_canary_25pct: 100 # 25 × 2 × 2
|
||||
day_3_5_canary_50pct: 200 # 50 × 2 × 2
|
||||
day_5_plus_canary_100pct: 400 # 100 × 2 × 2 initial
|
||||
production_sustained_volume: TBD_post_canary_re_projected_at_4_3_production_stable_transition
|
||||
|
||||
# ── Canary toggle (§2.1, §0.4 #3) ────────────────────────────────────────────
|
||||
|
||||
canary_toggle:
|
||||
env_var: WAGGLE_PHASE5_CANARY_PCT
|
||||
allowed_values: [0, 10, 25, 50, 100]
|
||||
default_until_PM_ratification: 0 # historical; superseded post-kick-off — see binding_rule
|
||||
post_kick_off_code_default: 10
|
||||
post_kick_off_locked_date: 2026-04-30
|
||||
code_default_source: packages/agent/src/feature-flags.ts:parsePhase5CanaryPct # undefined-branch returns 10
|
||||
test_default_pin: 0 # vitest.setup.ts pins WAGGLE_PHASE5_CANARY_PCT='0' for shape-selection determinism
|
||||
routing_strategy: deterministic_per_request_id_bucket # preserves A/B paired-comparison validity
|
||||
hot_reconfig_method: process_restart_or_SIGHUP # no code redeploy required per §0.4 #3
|
||||
flip_authority: PM_canary_kick_off_ratification_per_§7.3 # exercised 2026-04-30 — kick-off complete
|
||||
binding_rule: |
|
||||
POST-KICK-OFF (canary Day 0 LIVE 2026-04-30): default_until_PM_ratification=0
|
||||
superseded by post_kick_off_code_default=10 in
|
||||
packages/agent/src/feature-flags.ts:parsePhase5CanaryPct undefined-branch
|
||||
(PM signoff D:/Projects/PM-Waggle-OS/decisions/2026-04-30-phase-5-1-5-pm-signoff-canary-authorize.md).
|
||||
Override path retained: WAGGLE_PHASE5_CANARY_PCT env var resolves first.
|
||||
Set WAGGLE_PHASE5_CANARY_PCT=0 to roll back canary OFF without redeploy
|
||||
per §2.3 procedure. Gradient advance to 25/50/100 requires PM ratification
|
||||
per §2.1 per-step AND-gate (max(7d, 30 samples) per metric per variant).
|
||||
Production wiring landed in same commit: pickShape() in
|
||||
packages/agent/src/retrieval-agent-loop.ts now delegates to
|
||||
routeRequestToVariant() so each LLM call buckets via FNV-1a hash of
|
||||
config.runId (Phase 3.4 checkpoint id, also used here for canary id).
|
||||
gradient_schedule:
|
||||
- day_offset: 0
|
||||
canary_pct: 10
|
||||
observation_window_hours: 24
|
||||
- day_offset_min: 1
|
||||
day_offset_max: 2
|
||||
canary_pct: 25
|
||||
observation_window_hours: 48
|
||||
- day_offset_min: 3
|
||||
day_offset_max: 5
|
||||
canary_pct: 50
|
||||
observation_window_hours: 72
|
||||
- day_offset_min: 5
|
||||
canary_pct_full_enable_candidate: 100
|
||||
condition: all_§4_1_promotion_criteria_PASS_per_pre_registered_thresholds
|
||||
|
||||
# ── Promotion criteria (§4.1 BIND, ε=1e-9 inclusive) ─────────────────────────
|
||||
|
||||
promotion_criteria:
|
||||
binding_rule: no_revisit_without_amendment
|
||||
amendment_authority: PM_ratification + LOCKED_memo
|
||||
inclusive_boundary_epsilon: 1.0e-9
|
||||
applies_at: canary_to_full_enable_promotion
|
||||
criteria:
|
||||
- id: pass_ii_rate
|
||||
condition: "variant_pass_ii ≥ baseline_pass_ii + 0pp − ε"
|
||||
stretch_goal_pp: 5 # not blocking
|
||||
sample_floor_per_metric: 30
|
||||
- id: retrieval_engagement
|
||||
condition_qwen_thinking: "variant_retrieval ≥ baseline_retrieval × 0.80"
|
||||
condition_claude: "variant_retrieval ≥ baseline_retrieval"
|
||||
rationale: "qwen-thinking is Phase 4.5 mechanism-validated retrieval-driver; claude::gen1-v1 evolution different type (narrative XML restructure, not retrieval-engagement-targeted)."
|
||||
qwen_thinking_baseline_reference_mean_calls: 2.231 # Faza 1 evidence
|
||||
- id: latency_p95
|
||||
condition: "variant_p95 ≤ baseline_p95 × 1.20"
|
||||
- id: cost_per_request
|
||||
condition: "variant_cost ≤ baseline_cost × 1.15"
|
||||
anchor: feedback_cost_projection_real_anchoring
|
||||
- id: error_rate
|
||||
condition: "variant_error ≤ baseline_error + 1pp"
|
||||
wall_clock_floor:
|
||||
formula: "max(7_days_since_canary_kickoff, 30_samples_per_variant_per_metric)"
|
||||
days_min: 7
|
||||
samples_per_metric_min: 30
|
||||
canonical_intent: |
|
||||
Both conditions must be met (whichever takes longer wins). Brief §2.2 specifies
|
||||
max(); brief §4.1 specifies min(). Manifest binds to max() per brief §2.2 + intent
|
||||
("sample floor je hard guard protiv small-sample-effect stage promotion"); §4.1
|
||||
wording 'min(...)' is treated as authoring typo. Flag for PM correction.
|
||||
|
||||
# ── Rollback triggers (§4.2 BIND, immediate) ─────────────────────────────────
|
||||
|
||||
rollback_triggers:
|
||||
binding_rule: automatic_rollback_per_section_2_3
|
||||
no_silent_rollback: true
|
||||
rollback_log_path: gepa-phase-5/rollback-log.jsonl
|
||||
triggers:
|
||||
- id: pass_ii_collapse
|
||||
condition: "variant_pass_ii < baseline_pass_ii − 10pp"
|
||||
consecutive_windows: 2
|
||||
window_size: 10
|
||||
- id: error_rate_spike
|
||||
condition: "variant_error > baseline_error + 5pp"
|
||||
consecutive_window_hours: 24
|
||||
- id: cost_per_request_spike
|
||||
condition: "variant_cost > baseline_cost × 2.0"
|
||||
window: single_immediate
|
||||
- id: latency_p95_spike
|
||||
condition: "variant_p95 > baseline_p95 × 3.0"
|
||||
window: single_immediate
|
||||
- id: manual_halt_or_pm
|
||||
authority: [PM, Marko, CC_observed_anomaly_outside_pre_registered_thresholds]
|
||||
condition: anomaly_outside_pre_registered_thresholds
|
||||
- id: opcija_c_long_task_loop_exhausted
|
||||
condition: "loop_exhausted_rate > 5%_baseline"
|
||||
rationale: "Phase 4 long-task fixes not inherited per Opcija C §3; halt with 'long-task fixes potrebni' diagnostic + selective cherry-pick option"
|
||||
cherry_pick_candidate_set:
|
||||
- c9bda3d # already inherited via gen1-v1 shape baseline pins
|
||||
- be8f702
|
||||
- e906114
|
||||
- 4d0542f
|
||||
- 8b8a940
|
||||
|
||||
# ── Production-stable definition (§4.3) ──────────────────────────────────────
|
||||
|
||||
production_stable_status:
|
||||
unblocks: cross_stream_dependencies
|
||||
conditions:
|
||||
days_zero_rollback: 30
|
||||
metrics_within_pass_bands_count: 5 # all 5 §4.1 metrics maintained
|
||||
halt_and_pm_emissions: 0
|
||||
|
||||
# ── Cross-stream dependencies (§5/§6, Waggle primary per feedback rule) ──────
|
||||
|
||||
cross_stream_primary:
|
||||
rationale_anchor: feedback_waggle_primary_framing
|
||||
rationale: |
|
||||
Waggle is the demand-creation engine for KVARK (CLAUDE.md §1 strategic function).
|
||||
Waggle landing/launch comms are PRIMARY trigger surface; Phase 5 production-stable
|
||||
is a 'Waggle-launch-readiness gate' with KVARK as downstream beneficiary.
|
||||
destinations:
|
||||
- id: waggle_landing_v3_proof_card_swap
|
||||
brief_section_anchor: §6.1
|
||||
trigger_condition: "phase_5_production_stable + Pass_II_numbers + retrieval_engagement_validated_samples"
|
||||
action: "Update Landing v2.3 Proof Card 1 sa PROVENANCE → BENCHMARK swap (concrete numbers from Phase 5)"
|
||||
owner: PM_authoring + claude.ai_design_generation
|
||||
timing_post_production_stable: 2_to_3_weeks
|
||||
additional_audit_anchor: project_landing_v2_basics_2026_04_28
|
||||
- id: waggle_launch_comms
|
||||
brief_section_anchor: §6.1_implicit_extension
|
||||
trigger_condition: phase_5_production_stable
|
||||
action: "Public launch comms — Waggle production-stable evidence as anchor; KVARK pitch is downstream consumption of same evidence"
|
||||
owner: Marko + PM
|
||||
timing_post_production_stable: aligned_with_landing_v3_swap
|
||||
- id: arxiv_5_1_5_2_methodology_results
|
||||
brief_section_anchor: §6.2 # §6.2 §5.1 + §5.2 portion only — §5.3 is secondary
|
||||
trigger_condition: "phase_5_production_stable + Marko_arxiv_co_author_roster_ratification + endorsement_contact"
|
||||
action: "arxiv §5.1 (methodology + manifest v7 + 11 amendments transparent narrative) + §5.2 (Faza 1 results: cross-family generalization, claude + qwen-thinking pass; gpt selection bias scoping) — Waggle-primary academic credibility"
|
||||
owner: Marko_authoring
|
||||
timing_post_production_stable: TBD
|
||||
|
||||
cross_stream_secondary:
|
||||
rationale_anchor: feedback_waggle_primary_framing
|
||||
rationale: |
|
||||
Downstream beneficiaries of Phase 5 production-stable evidence. Sequenced after
|
||||
Waggle-primary cross-stream destinations consume the evidence. Reversing primary/
|
||||
secondary order misframes KVARK as entry point rather than upgrade path.
|
||||
destinations:
|
||||
- id: kvark_pitch_deck
|
||||
brief_section_anchor: §6.3
|
||||
trigger_condition: "phase_5_production_stable + Marko_pitch_deck_draft_approval"
|
||||
action: "KVARK enterprise pitch deck section: 'On-prem Qwen + Waggle harness validated Opus-class on real production traffic, X% retrieval parity, Y% Pass II floor'"
|
||||
owner: Marko
|
||||
timing_post_production_stable: TBD_post_landing_v3_swap
|
||||
relationship_to_primary: downstream_beneficiary_of_landing_v3_evidence
|
||||
- id: arxiv_section_5_3_kvark_enterprise
|
||||
brief_section_anchor: §6.2_§5.3_only
|
||||
trigger_condition: phase_5_production_stable
|
||||
action: "arxiv §5.3 KVARK enterprise narrative — Qwen 35B = Opus-class on production traffic = KVARK pitch scientific anchor"
|
||||
relationship_to_primary: downstream_beneficiary_of_section_5_1_5_2_evidence
|
||||
- id: faza_2_sprint_planning
|
||||
brief_section_anchor: §6.4
|
||||
trigger_condition: "phase_5_production_stable + 2_weeks_observation"
|
||||
scope:
|
||||
- gpt_gen1_v2_n_16_re_validation
|
||||
- qwen_non_thinking_deeper_investigation_retrieval_quality_decoupling
|
||||
- generic_simple_investigation_necessary_but_not_sufficient_retrieval_scoping
|
||||
- arxiv_section_5_4_finalization
|
||||
budget_headroom_usd: 71.51 # decoupled from Phase 5 cap (Faza 1 unspent of $115 cap)
|
||||
timing_post_production_stable: 2_weeks_observation_then_kick_off
|
||||
|
||||
# ── Halt-and-PM trigger automation ───────────────────────────────────────────
|
||||
|
||||
halt_and_pm_triggers:
|
||||
active: true
|
||||
emit_path: phase-5-alerts/<ISO_date>.jsonl
|
||||
authority: automated_threshold_breach + manual_PM_or_Marko
|
||||
triggers:
|
||||
- rollback_triggering_threshold_breach # any §4.2 trigger
|
||||
- cost_overshoot_above_halt_60
|
||||
- cost_overshoot_above_hard_cap_75
|
||||
- probe_re_run_cost_shift_gt_25pct_from_baseline
|
||||
- discovery_outside_scope # prompt-shape registry mismatch, manifest emit fail, monitoring stub no-op
|
||||
binding_rule: |
|
||||
No silent degradation. Sve threshold breaches MUST emit THRESHOLD_BREACH log entry.
|
||||
Rollback-triggering breaches MUST invoke §2.3 rollback procedure (or stub equivalent
|
||||
pre-canary-kickoff: log + alert file write).
|
||||
|
||||
# ── Audit chain anchors ──────────────────────────────────────────────────────
|
||||
|
||||
audit_anchors:
|
||||
brief_LOCKED: D:/Projects/PM-Waggle-OS/briefs/2026-04-29-phase-5-deployment-brief-v1.md
|
||||
brief_LOCKED_ratification: D:/Projects/PM-Waggle-OS/decisions/2026-04-29-phase-5-brief-LOCKED.md
|
||||
scope_LOCKED: D:/Projects/PM-Waggle-OS/decisions/2026-04-29-phase-5-scope-LOCKED.md
|
||||
faza_1_closure: D:/Projects/PM-Waggle-OS/decisions/2026-04-29-gepa-faza1-results.md
|
||||
branch_architecture_opcija_c: D:/Projects/PM-Waggle-OS/decisions/2026-04-30-branch-architecture-opcija-c.md
|
||||
cost_amendment_LOCKED: D:/Projects/PM-Waggle-OS/decisions/2026-04-30-phase-5-cost-amendment-LOCKED.md
|
||||
preflight_evidence: D:/Projects/waggle-os/gepa-phase-5/preflight-evidence.md
|
||||
faza_1_manifest_v7: D:/Projects/waggle-os/benchmarks/preregistration/manifest-v7-gepa-faza1.yaml
|
||||
cost_probe_jsonl: D:/Projects/waggle-os/gepa-phase-5/cost-probe-2026-04-29.jsonl
|
||||
cost_probe_summary: D:/Projects/waggle-os/gepa-phase-5/cost-probe-2026-04-29-summary.md
|
||||
this_manifest: D:/Projects/waggle-os/gepa-phase-5/manifest.yaml
|
||||
|
||||
binding_feedback_rules:
|
||||
- feedback_epsilon_inclusive_boundary # §4.1 #1 ε=1e-9 inclusive
|
||||
- feedback_external_contract_validation # §0.1 registerShape canonical API enforced
|
||||
- feedback_cost_projection_real_anchoring # §0.3 three-element decomposition + probe-validation
|
||||
- feedback_substrate_readiness_gate # §0 grep evidence binding before §2 deployment
|
||||
- feedback_config_inheritance_audit # §0.2 explicit differential block, no implicit defaults
|
||||
- feedback_brief_wall_clock_discipline # wall-clock 2-4 dana labeled "projection NOT trigger"
|
||||
- feedback_production_vs_research_cost_discipline # NEW 2026-04-30: cost amendment regime classification
|
||||
- feedback_waggle_primary_framing # NEW 2026-04-30: cross_stream_primary (Waggle) vs secondary (KVARK)
|
||||
|
||||
# ── Manifest history ─────────────────────────────────────────────────────────
|
||||
|
||||
manifest_history:
|
||||
- version: phase-5-deployment-v1
|
||||
locked_date: 2026-04-29
|
||||
cost_amendment_locked_date: 2026-04-30
|
||||
authoring_session: phase_5_§1_§5_implementation_2026_04_29_post_pm_amendment_post_restart
|
||||
drives:
|
||||
- §2_canary_toggle_implementation
|
||||
- §3_monitoring_infrastructure
|
||||
- §4_exit_criteria_coverage_doc
|
||||
- §5_cross_stream_dependencies_doc
|
||||
audit_chain_terminus: this_file
|
||||
370
gepa-phase-5/preflight-evidence.md
Normal file
370
gepa-phase-5/preflight-evidence.md
Normal file
@@ -0,0 +1,370 @@
|
||||
# Phase 5 §0 Preflight Evidence
|
||||
|
||||
**Date:** 2026-04-29 (CC execution session)
|
||||
**Author:** CC (Claude Opus 4.7)
|
||||
**Branch:** `phase-5-deployment-v2`
|
||||
**HEAD:** `6bc20897d3851072eda34e80070faf39772bee66` (`6bc2089`) — verified `git rev-parse HEAD`
|
||||
**Brief:** `D:/Projects/PM-Waggle-OS/briefs/2026-04-29-phase-5-deployment-brief-v1.md`
|
||||
**Branch architecture:** Opcija C per `decisions/2026-04-30-branch-architecture-opcija-c.md`
|
||||
**Verdict aggregation (initial):** §0.1 **PARTIAL (PM ratification needed)** · §0.2 **PASS** · §0.3 **DEFERRED (probe pending §0.1 ratification)** · §0.4 **PARTIAL (design-stage)**
|
||||
**Halt-and-PM trigger (initial):** YES — §0.1 mutation-validator regression + §0.4 implementation gating
|
||||
|
||||
---
|
||||
|
||||
## PM Round 1 ratification (2026-04-29)
|
||||
|
||||
PM ratified all 3 initial halt-and-PM asks:
|
||||
- **Ask #1 Option 1**: Quarantine `mutation-validator.test.ts` + `registry-injection.test.ts` (extension same session, PM informed) under `benchmarks/gepa/tests/faza-1/__faza1-closed/`. Quarantine commit `50393b1`. Test suite: 6046 (14 failed) → 6019 (0 failed, 1 skipped).
|
||||
- **Ask #2 AUTHORIZE**: 5-request probe per variant via existing LiteLLM proxy (matches Faza 1 runner pattern). Probe executed; results below.
|
||||
- **Ask #3 RATIFY**: §0 is design gate, not build gate. Brief §0.4 #2 functional-stub wording was PM authoring artifact; intent was design-readiness.
|
||||
|
||||
**Verdict aggregation (post-Round-1):** §0.1 **PASS (post-quarantine)** · §0.2 **PASS** · §0.3 **PROBE-COMPLETE — HARD-CAP-EXCEED** · §0.4 **PASS-design-stage**
|
||||
**Halt-and-PM trigger (Round 2):** YES — §0.3 probe ceiling exceeds brief §5.4 hard cap.
|
||||
|
||||
---
|
||||
|
||||
## §0.1 — Substrate readiness grep
|
||||
|
||||
### Verification anchors
|
||||
|
||||
| # | Requirement | Evidence | Verdict |
|
||||
|---|---|---|---|
|
||||
| 1 | `REGISTRY` in `selector.ts` contains base shapes `claude`, `qwen-thinking`, `qwen-non-thinking`, `gpt`, `generic-simple` | `packages/agent/src/prompt-shapes/selector.ts:30-36` | PASS |
|
||||
| 2 | `registerShape` canonical API exported from `selector.ts` AND barrel `index.ts` | `selector.ts:65-76` (export function declaration) + `index.ts:46` (barrel re-export) | PASS |
|
||||
| 3 | `gen1-v1` shape definitions exist for `claude` + `qwen-thinking` | `gepa-evolved/claude-gen1-v1.ts:23` (`claudeGen1V1Shape` const, `name: 'claude-gen1-v1'`) + `gepa-evolved/qwen-thinking-gen1-v1.ts:23` (`qwenThinkingGen1V1Shape` const, `name: 'qwen-thinking-gen1-v1'`) | PASS |
|
||||
| 4 | `git merge-base --is-ancestor 6bc2089 HEAD` exit=0 | Exit code `0` (HEAD itself is `6bc2089`; no post-terminus commits) | PASS |
|
||||
| 5 | No orphaned `gpt::gen1-v2` references in Phase 5 deployment artifacts | `gepa-phase-5/` grep empty. Repo-wide grep finds 2 references both in Faza 1 audit anchors: `packages/agent/src/prompt-shapes/gepa-evolved/gpt-gen1-v2.ts` (variant source code, present but not deployed) + `benchmarks/gepa/scripts/faza-1/run-checkpoint-c.ts` (Faza 1 held-out validation runner — produced FAIL verdict that exposed selection bias). Both allowed per brief §1 + §8 audit anchors. | PASS |
|
||||
|
||||
### Test suite execution
|
||||
|
||||
**Agent workspace** (substrate-relevant subsuite):
|
||||
```
|
||||
Test Files 147 passed (147)
|
||||
Tests 2547 passed (2547)
|
||||
Duration 14.93s
|
||||
```
|
||||
Matches 2026-04-28 S1 handoff baseline (`2547/2547 agent`). PASS.
|
||||
|
||||
**Repo-root suite (vitest run, full):**
|
||||
```
|
||||
Test Files 2 failed | 409 passed | 1 skipped (412)
|
||||
Tests 14 failed | 6031 passed | 1 skipped (6046)
|
||||
Duration 83.55s
|
||||
```
|
||||
|
||||
**Failure scope** — all 14 failures isolated to `benchmarks/gepa/tests/faza-1/mutation-validator.test.ts`:
|
||||
- 8 × `boundary anchor SHAs match substrate at c9bda3d > baseline {types,claude,qwen-thinking,qwen-non-thinking,gpt,generic-simple}.ts SHA matches pinned`
|
||||
- 2 × `validateCandidate — Gen 0 (baseline) acceptance`
|
||||
- 2 × `validateCandidate — accepts valid Gen 1 mutation`
|
||||
- 2 × `Amendment 8 §registry_invariant_test — REGISTRY cross-module-boundary documents H1 failure mode`
|
||||
|
||||
**Root cause analysis (preliminary, no fix attempted):**
|
||||
|
||||
The mutation-validator test pins baseline shape file SHAs against substrate freeze head `c9bda3d` (Phase 4.7 on `feature/c3-v3-wrapper`). Phase 5 branch `phase-5-deployment-v2` (= `6bc2089`) inherits `gepa-faza-1` parent chain via `origin/main` (`5ec069e`), NOT via `c9bda3d`. Per Opcija C decision §3, Phase 5 grana inherits Faza 1 work + base shapes BUT not Phase 4 long-task fixes from `feature/c3-v3-wrapper`. The shape-file content on `phase-5-deployment-v2` therefore reflects `origin/main` lineage, which has different SHAs than `c9bda3d` substrate the test pins to.
|
||||
|
||||
Faza 1 closure §F.4 reports `105/105 anchor invariance checks PASS` during Faza 1 evaluation runs — those checks ran inside `D:/Projects/waggle-os-faza1-wt` worktree pinned at `c9bda3d`. After Faza 1 closure and branch reconstruction (per Opcija C §1 #2 dangling-commit recovery via `git branch gepa-faza-1 6bc2089`), the test substrate context shifted from `c9bda3d` worktree to `6bc2089` repo head, and the SHA pins no longer match the live shape file content.
|
||||
|
||||
**Verdict on test suite:** PARTIAL.
|
||||
- Substrate API surface (REGISTRY, registerShape, gen1-v1 definitions) operational and verified — Phase 5 deployment substrate intact.
|
||||
- Mutation-validator failures appear to be Faza 1 audit-period artifact (test scope = Faza 1 closure-time SHA pinning that did not get quarantined post-closure under Opcija C branch architecture).
|
||||
- 14 failures do not affect Phase 5 deployment substrate API contract; do affect literal "noviji test broj passing" requirement of brief §0.1 #3.
|
||||
|
||||
### §0.1 Verdict: **PARTIAL — PM ratification required**
|
||||
|
||||
5/5 substrate API anchor requirements PASS. Test suite has 14 failures isolated to Faza 1 mutation-validator scope (post-closure SHA pin against `c9bda3d` substrate not reachable from `phase-5-deployment-v2` Opcija C inheritance chain). PM ratification needed on whether these failures are scope-isolated and the gate advances, or whether mutation-validator must be quarantined / re-pinned as remediation before §2 deployment.
|
||||
|
||||
**Halt-and-PM ask for §0.1:**
|
||||
1. Ratify §0.1 PARTIAL → PASS conditional on mutation-validator quarantine (e.g., move under `benchmarks/gepa/tests/faza-1/__faza1-closed/` with explicit "deferred to integration sprint" annotation)
|
||||
2. OR escalate as substrate failure requiring branch surgery before §2 (would defer Phase 5 by integration sprint estimated 2-3 days wall-clock per Opcija C §5)
|
||||
|
||||
CC recommends Option 1 — Faza 1 closure verdict §F.4 already documents 105/105 anchor invariance during evaluation runs; Phase 5 deployment substrate API surface (registerShape canonical API, gen1-v1 shape definitions) is verified independently and operational. Mutation-validator drift is post-closure scope leakage, not Phase 5 substrate failure.
|
||||
|
||||
---
|
||||
|
||||
## §0.2 — Config inheritance audit
|
||||
|
||||
### Differential — Faza 1 manifest v7 vs Phase 5 deployment
|
||||
|
||||
| Field | Faza 1 manifest v7 value | Phase 5 deployment value | Justification |
|
||||
|---|---|---|---|
|
||||
| `temperature` | **agent-side default 0.3**; corpus generation 0.7 (variation), 0.5 (ground truth); **judge primary 1.0 explicit** (claude-opus-4-7) and `omitted` for reasoning models (gpt-5.4, minimax-m27) per pilot runner line 385/387; oracle 1.0 (mutation oracle); `omitted` for Anthropic with `response_format` per Faza 1 mutation oracle line 6000 cap | **Production-default 0.3** for both deployed variants (claude::gen1-v1 + qwen-thinking::gen1-v1) — inherits agent-side default from Faza 1; production traffic uses standard temperature for variance control per brief §5.1 ("production usually lower variance") | Inherits agent-side default. No corpus generation or judging in Phase 5 production deployment (those were Faza 1 scope). Probe-validation (deferred — see §0.3) may surface variant-specific temperature recommendation. |
|
||||
| `max_tokens` | judge 3000, agent default 16000, oracle 8000, mutation-oracle response_format 6000 | **claude::gen1-v1 = 4096** (per shape `metadata.defaultMaxTokens`, file `gepa-evolved/claude-gen1-v1.ts:30`); **qwen-thinking::gen1-v1 = 16000** (per shape `metadata.defaultMaxTokens`, file `gepa-evolved/qwen-thinking-gen1-v1.ts:30`) | Aligns with Phase 4.5 retrieval engagement finding (Qwen 1.33×/task → target ≥1.7 retrievals/task; thinking-mode requires output budget for retrieval-driven synthesis). Claude shape default lower because narrative+XML structure is more output-efficient than minimal-scaffolding qwen-thinking. |
|
||||
| `judge_model_primary` | **trio = claude-opus-4-7 + gpt-5.4 + minimax-m27** with κ=0.7878 anchor (raw-agreement primary metric per Amendment 5 §judge_metric_design) | **NOT APPLICABLE — Phase 5 production deployment has no inline judge.** §3 monitoring metrics replace evaluation judging: Pass II rate (proxy via downstream signal), retrieval engagement (per-request count), latency p50/p95, cost per request, error rate. Judge invocation deferred to Faza 2 re-validation runs (gpt::gen1-v2 N=16) and arxiv §5 evidence integration. | Inherent task-type shift: Faza 1 = evolution + held-out validation (judge required); Phase 5 = production deployment + monitoring (signal-side metrics instead of judge labels). |
|
||||
| `evaluation_corpus_source` | **50-instance NorthLane CFO synthesis corpus** (5 task families F1-F5: strategic synthesis, cross-thread coord, decision support, investor comm, scenario planning) + LoCoMo dataset for non-GEPA stages (held-out 5 instances of slice 8-12 of seed=42 shuffle for Checkpoint C) | **Production live traffic** (real user sessions on Waggle OS) | Inherent task-type shift per brief §5.1. Phase 5 deployment evaluates in-the-wild against actual user prompts, not pre-registered corpus. Production traffic distribution unknown a priori — §3 monitoring + §4 promotion criteria provide acceptance bounds. |
|
||||
| `failure_mode_taxonomy` | Faza 1 Amendment 4 texture audit — **F1-F6 task family classifier + per-family Pass II rate**: F1 strategic synthesis, F2 cross-thread coord, F3 decision support, F4 investor comm, F5 scenario planning, F6 = unsupported-specifics / missed-shallow / conflation (T2 modes per `phase-4-3-rescore-delta-report.md`) | **Production rollback triggers** per brief §4.2: (a) Pass II < baseline -10pp, (b) error rate > baseline +5pp, (c) cost > baseline ×2.0, (d) latency p95 > baseline ×3.0, (e) manual halt-and-PM. **Plus Opcija C-specific addition:** loop_exhausted rate > 5% baseline → halt with "long-task fixes potrebni" rationale (Phase 4 long-task fixes not inherited per Opcija C §3). | New mapping required — production threshold-based detection vs. Faza 1's content-classifier. Additional Opcija C trade-off rationale field surfaces when Phase 4 long-task absence becomes material. |
|
||||
| `cost_per_request_baseline` | $0.27/instance (Amendment 3 probe-corrected from $0.10 generic estimate; Faza 1 actual $43.49 total / 175 substantive evals = ~$0.25/eval) | **Probe-validated Phase 5 (re-probe REQUIRED per brief §0.3)**. Pricing snapshot taken (see §0.3 below); live 5-request probe per variant DEFERRED pending §0.1 PM ratification. Estimate-only ceiling provided. | Cost may shift with production load patterns (cache hit rates, prompt caching strategy, conversation-context size). Brief §0.3 binds probe-validation, not estimate. |
|
||||
|
||||
### Opcija C trade-off acknowledgment
|
||||
|
||||
Per `decisions/2026-04-30-branch-architecture-opcija-c.md` §3:
|
||||
|
||||
| Phase 4 long-task fix not inherited | Mitigation in Phase 5 monitoring |
|
||||
|---|---|
|
||||
| `runRetrievalAgentLoopWithRecovery` (recovery from loop_exhausted) | §3 monitoring `error_rate` metric watches `loop_exhausted` count; threshold alert ≥5% triggers halt-and-PM with explicit "Phase 4 long-task fixes potrebni" rationale + selective cherry-pick option from `feature/c3-v3-wrapper` commits `c9bda3d`, `be8f702`, `e906114`, `4d0542f`, `8b8a940` |
|
||||
| Failure classifier (F-codes for long-task failure attribution) | Phase 5 uses threshold-based classifier (binary pass/fail per metric) instead. F-code attribution deferred to integration sprint post-production-stable |
|
||||
| Reporting module (long-task structured reports) | §3 daily summary `phase-5-daily-summary/<ISO_date>.md` provides simple metric snapshots; structured reporting deferred |
|
||||
| Messages-array compression (long-conversation context budget management) | Phase 5 traffic profile is simple-medium tasks (per Opcija C §3); compression not load-bearing for canary phase. Watch latency p95 — if escalates due to context bloat, halt-and-PM |
|
||||
|
||||
### §0.2 Verdict: **PASS**
|
||||
|
||||
All 6 differential rows have explicit value + justification. Implicit defaults forbidden — none used. Opcija C trade-off documented with explicit monitoring mitigation per row.
|
||||
|
||||
---
|
||||
|
||||
## §0.3 — Cost projection probe
|
||||
|
||||
### Pricing snapshot (2026-04-29)
|
||||
|
||||
| Model | Input $/1M tokens | Output $/1M tokens | Cache Hits $/1M | Source | Snapshot timestamp |
|
||||
|---|---|---|---|---|---|
|
||||
| Claude Opus 4.7 (claude::gen1-v1 deployment) | **$5.00** | **$25.00** | $0.50 | https://platform.claude.com/docs/en/docs/about-claude/pricing | 2026-04-29 (CC fetch this session) |
|
||||
| DashScope Qwen 35B-A3B International thinking mode (qwen-thinking::gen1-v1) | **$0.25** | **$2.00** (thinking mode) | n/a | https://www.alibabacloud.com/help/en/model-studio/billing-for-model-studio | 2026-04-29 (CC fetch this session) |
|
||||
| DashScope Qwen 35B-A3B Global mode (alternative reference) | $0.057 | $0.459 (non-thinking) / $1.835 (thinking) | n/a | same as above | 2026-04-29 |
|
||||
|
||||
**Note on Opus 4.7 pricing:** New tokenizer may use up to 35% more tokens for the same fixed text — affects cost computation upward by up to 1.35× for content-equivalent prompts vs. older tokenizer baselines.
|
||||
|
||||
### Estimate-only cost projection (live probe DEFERRED)
|
||||
|
||||
Without live probe, point estimates use shape-file `defaultMaxTokens` as upper bound for output and conservative input estimate from Faza 1 corpus average (~3000 input tokens per instance). **These are NOT brief-binding numbers — probe-validation per §0.3 is required for canary kick-off.**
|
||||
|
||||
| Variant | Input est. | Output est. | Per-request est. (no cache) | 740-request canary × 1.20 buffer |
|
||||
|---|---|---|---|---|
|
||||
| claude::gen1-v1 | 3000 tok @ $5/M = $0.015 | 4096 tok @ $25/M = $0.102 | **~$0.117** | 740 × $0.117 × 1.20 = **~$103.86** |
|
||||
| qwen-thinking::gen1-v1 | 3000 tok @ $0.25/M = $0.0008 | 16000 tok @ $2/M (thinking) = $0.032 | **~$0.033** | 740 × $0.033 × 1.20 = **~$29.30** |
|
||||
|
||||
`canary_cost_p95_ceiling estimate (max of two)` = **~$103.86** (claude-dominated).
|
||||
|
||||
**Comparison to brief ceiling:**
|
||||
- Brief §5.4: `canary_cost_p95_ceiling ≤ $20` required.
|
||||
- Estimate ceiling **$103.86 >> $20** — naive estimate triggers ceiling validation FAIL.
|
||||
|
||||
**However, this estimate is materially conservative** because:
|
||||
1. **Output tokens at `defaultMaxTokens` upper bound** is worst-case; actual canary p95 likely far lower (Faza 1 evals averaged ~2000 output tokens, not 4096).
|
||||
2. **No prompt caching credit** — Phase 5 production should leverage 5m-cache writes (1.25× input cost, recovers via 0.1× cache reads). With 80% cache hit rate, claude input cost drops from $0.015 → $0.0042 per request.
|
||||
3. **Brief §5.2 day-by-day breakdown** assumes 10 requests/day Day 0-1 (not 740 immediate), so the 740-request total accumulates over 7 days canary, not in one batch.
|
||||
4. **Per Faza 1 actuals** ($43.49 total / 175 evals = ~$0.25/eval), claude judging+running was $0.25/eval — that includes Opus judging at higher token volume than Phase 5 production deployment will incur.
|
||||
|
||||
### §0.3 Verdict (post-PM Round 1 Ask #2 AUTHORIZE): **PROBE-COMPLETE — HARD-CAP-EXCEED**
|
||||
|
||||
Probe harness `gepa-phase-5/scripts/cost-probe.ts` shipped + executed via LiteLLM proxy (matches Faza 1 runner pattern). 5 varying-complexity prompts × 2 variants = 10 requests. Spent: **$0.1628** (under $0.30-$0.50 budget). All 10 requests succeeded (0 errors).
|
||||
|
||||
#### Per-variant probe statistics
|
||||
|
||||
| Variant | Model alias | OK | p50 | p95 | max | mean | total |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| `claude::gen1-v1` | `claude-opus-4-7` | 5/5 | $0.0239 | **$0.0432** | $0.0432 | $0.0250 | $0.1251 |
|
||||
| `qwen-thinking::gen1-v1` | `qwen3.6-35b-a3b-via-dashscope-direct` | 5/5 | $0.0064 | **$0.0176** | $0.0176 | $0.0075 | $0.0377 |
|
||||
|
||||
#### Per-request raw probe data (audit anchor)
|
||||
|
||||
| Variant | Complexity | Input tokens | Output tokens | Cost | Latency |
|
||||
|---|---|---|---|---|---|
|
||||
| claude::gen1-v1 | trivial | 638 | 39 | $0.0042 | 2.4s |
|
||||
| claude::gen1-v1 | medium-1 | 816 | 600 | $0.0191 | 11.4s |
|
||||
| claude::gen1-v1 | medium-2 | 787 | 800 | $0.0239 | 15.7s |
|
||||
| claude::gen1-v1 | complex | 953 | 1200 | $0.0348 | 21.6s |
|
||||
| claude::gen1-v1 | stretch | 1135 | 1500 | $0.0432 | 27.6s |
|
||||
| qwen-thinking::gen1-v1 | trivial | 152 | 950 | $0.0019 | 7.6s |
|
||||
| qwen-thinking::gen1-v1 | medium-1 | 277 | 2278 | $0.0046 | 17.0s |
|
||||
| qwen-thinking::gen1-v1 | medium-2 | 237 | 3189 | $0.0064 | 25.0s |
|
||||
| qwen-thinking::gen1-v1 | complex | 350 | 3496 | $0.0071 | 29.5s |
|
||||
| qwen-thinking::gen1-v1 | stretch | 506 | 8752 | $0.0176 | 66.6s |
|
||||
|
||||
Per-row JSONL: `gepa-phase-5/cost-probe-2026-04-29.jsonl`. Summary: `gepa-phase-5/cost-probe-2026-04-29-summary.md`.
|
||||
|
||||
#### Ceiling validation per brief §5.4
|
||||
|
||||
```
|
||||
canary_cost_p95_ceiling = 740 × max(p95) × 1.20
|
||||
= 740 × $0.0432 × 1.20
|
||||
= $38.34
|
||||
```
|
||||
|
||||
Brief §5.4 thresholds:
|
||||
- `halt_trigger = $20`
|
||||
- `hard_cap = $25`
|
||||
|
||||
**$38.34 > $25 hard cap → HARD-CAP-EXCEED → halt-and-PM mandatory per brief §0.3 #5 ("Ako prelazi, halt-and-PM za scope re-evaluation").**
|
||||
|
||||
#### Mechanism analysis
|
||||
|
||||
The claude::gen1-v1 stretch case dominates the ceiling: 1500 output tokens at $25/M = $0.0375 of the $0.0432 per-request cost. Phase 5 production output volume is the binding constraint. Two readings:
|
||||
|
||||
1. **Brief §5.2 volume estimate may be too aggressive.** The brief assumes 740 requests across 7-day canary phase (Day 0-1: 40, Day 1-3: 100, Day 3-5: 200, Day 5+: 400). For claude::gen1-v1 alone, that's already $32.13 expected (740 × $0.0432). Doubling with qwen-thinking adds modestly given its lower per-request cost.
|
||||
|
||||
2. **Opus 4.7 input pricing reduction did not propagate to output pricing.** Per 2026-04-29 snapshot, Opus 4.7 is $5/$25/M (input/output) — input dropped from $15 (Faza 1 pricing reference for `claude-opus-4-7`) to $5, but output stayed at $25. Output remains the cost driver for variants that produce long syntheses (the stretch case is a 1500-token DCF-comps-LBO walkthrough — the kind of long-form output the variant is designed for).
|
||||
|
||||
#### Mitigation options for PM ratification
|
||||
|
||||
| # | Option | Mechanism | Trade-off |
|
||||
|---|---|---|---|
|
||||
| **A** | **Reduce volume target** | Lower 740 → 386 (= $20 / 0.0432 / 1.2). Canary phase shrinks: maybe Day 0-1 only 5 reqs/day, Day 3-5 only 25 reqs/day, Day 5+ delayed. Brief §2.1 gradient timing extends. | Slower §4.2 promotion criteria sample-floor accumulation (≥30 samples per variant per metric); wall-clock floor `max(7_days, 30_samples)` shifts to dominated by 30-samples constraint. |
|
||||
| **B** | **Cap variant max_tokens** | Phase 5 production caps `max_tokens` at e.g. 1000 (vs claude shape default 4096). Re-probe shows lower p95 since output dominates cost. Stretch case truncated. | Shape default tuned for retrieval-driven synthesis (qwen-thinking has 16k for thinking budget); cap may reduce response quality on long-form analytical tasks (M&A memos, root-cause diagnosis). Need re-probe to verify ceiling lands under $20. |
|
||||
| **C** | **Pivot to qwen-only canary** | Deploy qwen-thinking::gen1-v1 alone (claude::gen1-v1 deferred). qwen p95 = $0.0176; ceiling = 740 × $0.0176 × 1.2 = **$15.63** (under $20). | Scope reduction — claude::gen1-v1 goes to Faza 2 alongside gpt::gen1-v2 even though it has clean validation. Loses the cross-family generalization production-validation per brief §1. KVARK pitch deck §6.3 still anchors on qwen 96% Opus parity but loses claude flagship continuity story. |
|
||||
| **D** | **Split canary cadences** | claude::gen1-v1 at lower canary% (e.g., max 25% never going to 100%); qwen-thinking::gen1-v1 at full gradient. Volume per variant differs. | Asymmetric promotion criteria; complicates §4.1 promotion logic and §3 monitoring threshold normalization. |
|
||||
| **E** | **Brief §5.4 ceiling amendment** | Raise hard_cap to e.g. $50, halt_trigger to $40. Per §4.4 amendment requires PM ratification + LOCKED memo. | Burns headroom for Faza 2 re-validation runs ($71.51 unspent of $115 cap). Cumulative project spend rises. Need to re-justify against Faza 1 closure §F cost discipline. |
|
||||
| **F** | **Prompt-cache leverage** | Phase 5 implementation MUST use Anthropic prompt caching (5m or 1h cache write × 0.1× cache read multiplier). System prompt + persona + materials cached across requests in same workspace/session. Re-probe with cache-active after §1-§2 wiring. | Adds implementation complexity to §1 work. Cache benefit depends on real production access pattern (high cache-hit rate requires repeat conversation context). Probe is one-shot so doesn't reflect cache benefit; need cache-aware re-probe. |
|
||||
|
||||
CC recommendation rationale (advisory, PM decides):
|
||||
|
||||
- **Option F (prompt caching) is the strongest engineering lever** if Phase 5 production traffic has reasonable conversation locality. With 80% cache hit rate, claude per-request input drops from $0.015 to $0.0042; total p95 drops from $0.0432 to ~$0.0420 (output-bound). Limited improvement because output dominates.
|
||||
- **Option A (volume reduction) is the cleanest scope-preserving mitigation.** Lower volume preserves variant scope + ceiling discipline. Wall-clock floor extends but Phase 5 brief §2.2 already binds `max(7_days, 30_samples)` so this is a known tradeoff.
|
||||
- **Option C (qwen-only) preserves the strongest scientific narrative** (Phase 4.5 mechanism CONFIRMED out-of-distribution, KVARK pitch arxiv §5.3 evidence) at cost of dropping claude flagship continuity validation.
|
||||
- **Options B + D add complexity without clean pareto improvement.**
|
||||
- **Option E (amendment) should be last resort** per §4.4 binding.
|
||||
|
||||
**Halt-and-PM ask for §0.3 (Round 2):**
|
||||
1. Ratify mitigation option (A / B / C / D / E / F or combination) for §0.3 ceiling exceed, OR
|
||||
2. Direct CC to investigate alternative reduction (e.g., prompt cache + volume-reduce combo with re-probe), OR
|
||||
3. Direct CC to re-run probe with adjusted parameters (e.g., max_tokens cap, different prompt complexity distribution).
|
||||
|
||||
---
|
||||
|
||||
## §0.4 — Deployment readiness checklist
|
||||
|
||||
### Verification anchors
|
||||
|
||||
| # | Requirement | Evidence | Verdict |
|
||||
|---|---|---|---|
|
||||
| 1 | `phase_5_pre_deployment_sha` pinned via `git rev-parse HEAD` | **`6bc20897d3851072eda34e80070faf39772bee66`** (`6bc2089`) — pinned in this evidence file. Rollback procedure: `git revert <deployment_commit_sha>` or `git reset --hard 6bc2089`. Forbidden in-place file overwrites. | PASS |
|
||||
| 2 | Monitoring infrastructure stubs functional (JSONL emitters: Pass II rate, retrieval engagement, latency p50/p95, cost per request, error rate) | **NOT YET BUILT.** `gepa-phase-5/monitoring/` directory created (this session) but no JSONL emitter code exists. Per brief §3.1 + §3.4 Stage 1: JSONL files + daily markdown summary. Implementation is §1-§2 work; brief §0.4 #2 requires "basic stubs OK ali ne smeju biti no-op" pre canary kick-off. | DEFERRED to §1-§2 |
|
||||
| 3 | Canary toggle mechanism (env var `WAGGLE_PHASE5_CANARY_PCT`) | **NOT YET BUILT.** Repo grep finds no canary toggle infrastructure. Implementation is §1-§2 work. | DEFERRED to §1-§2 |
|
||||
| 4 | Pre-registered exit criteria locked (§4 promotion + rollback thresholds) | **LOCKED in brief §4.1-§4.4** per `briefs/2026-04-29-phase-5-deployment-brief-v1.md`. Brief LOCKED 2026-04-29 ("sve ok idemo dalje"). §4.4 no-revisit-without-amendment binding active. Phase 5 manifest authorization (formal copy of §4 into `gepa-phase-5/manifest.yaml`) is §1 work. | PASS for brief; DEFERRED for manifest copy |
|
||||
| 5 | Halt-and-PM trigger automation wired (auto-emit halt request on §3 threshold breach) | **NOT YET BUILT.** No automation scripts for `phase-5-alerts/<ISO_date>.jsonl` emission. Implementation is §1-§2 work. | DEFERRED to §1-§2 |
|
||||
|
||||
### §0.4 Verdict: **PARTIAL — design-stage**
|
||||
|
||||
Items 1 + 4 (documentation/pinning) PASS. Items 2 + 3 + 5 (functional infrastructure) NOT YET BUILT — that's §1-§2 implementation work. Brief §0.4 wording ("basic stubs OK ali ne smeju biti no-op pre canary kick-off") implies stubs must exist BEFORE canary kick-off (§2), not before §0 PASS. §0 thus verifies design+plan readiness, not built infrastructure.
|
||||
|
||||
**Halt-and-PM ask for §0.4:**
|
||||
1. Confirm interpretation: §0.4 #2/#3/#5 verifies design-stage readiness only; functional stubs are §1-§2 deliverables verified before canary kick-off (not before §0 advancement).
|
||||
2. OR escalate: §0.4 requires functional stubs at §0 → CC builds stubs as part of preflight (estimated 1 day wall-clock) before §0 PASS aggregation.
|
||||
|
||||
CC recommends Option 1 — design intent at §0, build at §1-§2, verify functional pre-canary.
|
||||
|
||||
### Selective cherry-pick option from `feature/c3-v3-wrapper` (Opcija C §3 mitigation)
|
||||
|
||||
Documented for monitoring escalation path. If §3 monitoring fires "long-task fixes potrebni" rationale (loop_exhausted rate > 5% baseline), candidate cherry-pick set:
|
||||
|
||||
| Commit | Subject |
|
||||
|---|---|
|
||||
| `c9bda3d` | (Phase 4.7 head; substrate freeze for Faza 1 — already inherited via gen1-v1 shape baseline pins) |
|
||||
| `be8f702`, `e906114`, `4d0542f`, `8b8a940` | (Phase 4 long-task fix candidates — `runRetrievalAgentLoopWithRecovery`, failure classifier, reporting module, messages-array compression — per Opcija C §3) |
|
||||
|
||||
Cherry-pick procedure (if triggered): branch from `phase-5-deployment-v2`, cherry-pick selected commits, resolve `packages/agent` conflicts, verify agent test suite remains 2547+ passing, merge back to `phase-5-deployment-v2`, document in rollback log.
|
||||
|
||||
---
|
||||
|
||||
## §0 — Aggregate verdict (post-Round-1 + Round-2)
|
||||
|
||||
```
|
||||
§0_verdict_aggregate (Round 2) = §0.1 (PASS post-quarantine) AND §0.2 (PASS) AND §0.3 (HARD-CAP-EXCEED) AND §0.4 (PASS-design-stage)
|
||||
= NOT-PASS — §0.3 ceiling fail
|
||||
```
|
||||
|
||||
**Halt-and-PM trigger fires (Round 2).** CC stops at §0 again; does not self-advance to §1-§2 implementation. §0.1 + §0.2 + §0.4 all clean post-Round-1; only §0.3 remains open due to probe-validated cost ceiling exceed.
|
||||
|
||||
### Round 2 halt-and-PM ratification ask (1)
|
||||
|
||||
1. **§0.3 ceiling mitigation** — ratify Option A/B/C/D/E/F or combination (see §0.3 mitigation table above) to bring `canary_cost_p95_ceiling` from $38.34 to ≤ $20.
|
||||
|
||||
### Cost summary (§0)
|
||||
|
||||
| Item | Spent | Budget |
|
||||
|---|---|---|
|
||||
| Pricing snapshots (web fetches) | $0.00 (free) | n/a |
|
||||
| Probe (5 requests per variant, 10 total) | **$0.1628** | $0.30-$0.50 |
|
||||
| **§0 total spent** | **$0.1628** | $1.00 hard cap |
|
||||
|
||||
Probe budget headroom remaining: $0.14-$0.34 (for re-probe after PM mitigation choice if Option B / D / F selected).
|
||||
|
||||
### Wall-clock summary (§0)
|
||||
|
||||
| Item | Wall-clock |
|
||||
|---|---|
|
||||
| Brief + decisions load | ~5 min |
|
||||
| §0.1 substrate grep + test suite execution | ~20 min |
|
||||
| §0.2 config differential authoring | ~10 min |
|
||||
| §0.3 pricing snapshot fetches (Round 1) | ~3 min |
|
||||
| §0.4 deployment-readiness grep + documentation | ~10 min |
|
||||
| Round 1 evidence aggregation + commit | ~25 min |
|
||||
| Round 1 PM ratification turnaround | (PM-side, ~minutes) |
|
||||
| Quarantine commit (mutation-validator + registry-injection) | ~15 min |
|
||||
| Probe harness construction | ~20 min |
|
||||
| Probe execution | ~4 min (10 requests, mostly serial) |
|
||||
| Round 2 evidence update + commit | ~15 min |
|
||||
| **§0 total wall-clock** | **~127 min** |
|
||||
|
||||
Beyond initial 1-2h estimate due to two halt-and-PM rounds — expected per pre-registration discipline (each halt-and-PM is the design working as intended, not a delay).
|
||||
|
||||
---
|
||||
|
||||
## Audit chain anchors
|
||||
|
||||
| Item | Path / SHA |
|
||||
|---|---|
|
||||
| Brief LOCKED | `D:/Projects/PM-Waggle-OS/briefs/2026-04-29-phase-5-deployment-brief-v1.md` |
|
||||
| Brief LOCKED ratification | `decisions/2026-04-29-phase-5-brief-LOCKED.md` |
|
||||
| Scope LOCKED | `decisions/2026-04-29-phase-5-scope-LOCKED.md` |
|
||||
| Faza 1 closure (terminus) | `decisions/2026-04-29-gepa-faza1-results.md` |
|
||||
| Branch architecture (Opcija C) | `decisions/2026-04-30-branch-architecture-opcija-c.md` |
|
||||
| Phase 5 baseline branch | `phase-5-deployment-v2` (HEAD = `6bc20897d3851072eda34e80070faf39772bee66`) |
|
||||
| Faza 1 archive branch | `gepa-faza-1` (HEAD identical = `6bc2089`) |
|
||||
| Faza 1 manifest v7 | `benchmarks/preregistration/manifest-v7-gepa-faza1.yaml` (substrate_freeze_head `c9bda3d`) |
|
||||
| **THIS EVIDENCE FILE** | `D:/Projects/waggle-os/gepa-phase-5/preflight-evidence.md` |
|
||||
|
||||
---
|
||||
|
||||
## Round 3 — Post-amendment §0 PASS aggregate (2026-04-30)
|
||||
|
||||
PM amended Phase 5 brief §5.4 cost ceiling per `decisions/2026-04-30-phase-5-cost-amendment-LOCKED.md`:
|
||||
|
||||
| Field | Original (v0, brief §5.4 v1) | Amended (v1, LOCKED 2026-04-30) |
|
||||
|---|---|---|
|
||||
| Hard cap | $25 | **$75** |
|
||||
| Halt trigger | $20 | **$60** |
|
||||
| Expected total | $8-13 | **$35-45** |
|
||||
| Buffer above probe-validated $38.34 | (negative; ceiling exceeded) | ~96% (= $75 / $38.34) |
|
||||
|
||||
Marko ratification: "stavi visi slobodno" (2026-04-30).
|
||||
|
||||
### Regime classification
|
||||
|
||||
Per `feedback_production_vs_research_cost_discipline` (NEW memory entry authored alongside this Round 3 update): cost cap discipline differs by regime. Phase 5 is **production deployment**, where the cap is an operational projection that can be amended via PM decision memo + ratification when probe data reveals an underestimate. Amendment does NOT invalidate prior evidence (scope LOCK, manifests, validation runs upstream remain binding). Faza N research evals operate under a stricter no-revisit-without-amendment binding regime; the two were briefly conflated when PM proposed Opcija C qwen-only as "cleaner math" — Marko corrected.
|
||||
|
||||
### §0.3 verdict revision
|
||||
|
||||
`canary_cost_p95_ceiling = $38.34 < halt_trigger $60 < hard_cap $75` → **§0.3 PASS** (post-amendment).
|
||||
|
||||
Probe data unchanged; only the cost ceiling threshold revised. Probe artifacts at `gepa-phase-5/cost-probe-2026-04-29.jsonl` (per-row JSONL) + `gepa-phase-5/cost-probe-2026-04-29-summary.md` (verdict summary) remain authoritative.
|
||||
|
||||
### §0 aggregate verdict revision
|
||||
|
||||
```
|
||||
§0_verdict_aggregate (Round 3) = §0.1 (PASS post-quarantine) AND §0.2 (PASS) AND §0.3 (PASS post-amendment) AND §0.4 (PASS-design-stage)
|
||||
= PASS sva 4 sub-gates
|
||||
```
|
||||
|
||||
**CC unblocked for §1-§5 implementation per Phase 5 brief.** Halt-and-PM cleared. §0.4 deferred items (#2 monitoring stubs, #3 canary toggle, #5 halt-and-PM automation) become §1-§3 deliverables verified before canary kick-off (§7.3 PM ratification gate), not before §0 advancement.
|
||||
|
||||
### Cumulative cost summary (§0)
|
||||
|
||||
| Item | Spent | Budget |
|
||||
|---|---|---|
|
||||
| Pricing snapshots (web fetches) | $0.00 (free) | n/a |
|
||||
| §0.3 probe (5 requests per variant, 10 total) | $0.1628 | $0.30-$0.50 |
|
||||
| **§0 total spent** | **$0.1628** | $1.00 hard cap (§0 alone) |
|
||||
|
||||
§1-§5 implementation budget envelope (per amended cost cap): $74.84 remaining of $75 hard cap. Implementation work itself (§1 manifest authoring, §2 canary toggle code, §3 monitoring stubs, §4 coverage doc, §5 cross-stream doc) burns no LLM cost.
|
||||
|
||||
### Audit anchor for Round 3
|
||||
|
||||
| Item | Path |
|
||||
|---|---|
|
||||
| Cost amendment LOCKED memo | `D:/Projects/PM-Waggle-OS/decisions/2026-04-30-phase-5-cost-amendment-LOCKED.md` |
|
||||
| New memory entry — cost regime | `C:/Users/MarkoMarkovic/.claude/projects/D--Projects-waggle-os/memory/feedback_production_vs_research_cost_discipline.md` |
|
||||
| New memory entry — Waggle primary framing | `C:/Users/MarkoMarkovic/.claude/projects/D--Projects-waggle-os/memory/feedback_waggle_primary_framing.md` |
|
||||
|
||||
---
|
||||
|
||||
**End of §0 preflight evidence. §0 PASS aggregate Round 3. CC implementing §1-§5.**
|
||||
425
gepa-phase-5/scripts/cost-probe.ts
Normal file
425
gepa-phase-5/scripts/cost-probe.ts
Normal file
@@ -0,0 +1,425 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* Phase 5 §0.3 — Cost projection probe.
|
||||
*
|
||||
* Per PM ratification 2026-04-29 Phase 5 §0 preflight Ask #2: AUTHORIZED.
|
||||
* Per brief §0.3 #3 + §5.4 cost ceiling computation.
|
||||
*
|
||||
* Sends 5 prompts of varying complexity to each Phase 5 deployed variant via
|
||||
* the existing LiteLLM proxy (matches Faza 1 runner pattern):
|
||||
* - claude::gen1-v1 model alias = claude-opus-4-7
|
||||
* - qwen-thinking::gen1-v1 model alias = qwen3.6-35b-a3b-via-dashscope-direct
|
||||
*
|
||||
* Records actual input/output tokens × pricing snapshot 2026-04-29.
|
||||
* Emits per-request JSONL + computes p50/p95/max.
|
||||
*
|
||||
* Validates canary_cost_p95_ceiling = 740 × max(p95) × 1.20 ≤ $20.
|
||||
*
|
||||
* Pricing snapshot (2026-04-29 — public docs URL):
|
||||
* - Claude Opus 4.7: $5/M input, $25/M output (NEW Opus 4.7 pricing per
|
||||
* platform.claude.com/docs/en/docs/about-claude/pricing 2026-04-29)
|
||||
* - DashScope Qwen 35B-A3B intl thinking: $0.25/M input, $2/M output
|
||||
* (alibabacloud.com/help/en/model-studio/billing-for-model-studio 2026-04-29)
|
||||
*
|
||||
* NOTE on Faza 1 pricing reference: run-checkpoint-c.ts uses different
|
||||
* historical pricing ($15/$75 for Opus, $0.20/$0.80 for Qwen). Phase 5
|
||||
* uses the 2026-04-29 snapshot above (Opus 4.7 reduced rate; Qwen intl
|
||||
* thinking-mode rate). Probe pricing is the binding source for §0.3.
|
||||
*
|
||||
* Usage: npx tsx gepa-phase-5/scripts/cost-probe.ts
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { claudeGen1V1Shape } from '../../packages/agent/src/prompt-shapes/gepa-evolved/claude-gen1-v1.js';
|
||||
import { qwenThinkingGen1V1Shape } from '../../packages/agent/src/prompt-shapes/gepa-evolved/qwen-thinking-gen1-v1.js';
|
||||
|
||||
// Manual .env load (no dotenv dependency; keep probe self-contained)
|
||||
const envPath = path.resolve(process.cwd(), '.env');
|
||||
if (fs.existsSync(envPath)) {
|
||||
const raw = fs.readFileSync(envPath, 'utf-8');
|
||||
for (const line of raw.split(/\r?\n/)) {
|
||||
const m = /^([A-Z_][A-Z0-9_]*)=(.*)$/.exec(line.trim());
|
||||
if (m && !process.env[m[1]]) {
|
||||
let v = m[2];
|
||||
if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) {
|
||||
v = v.slice(1, -1);
|
||||
}
|
||||
process.env[m[1]] = v;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const REPO_ROOT = path.resolve(__dirname, '../..');
|
||||
const OUT_JSONL = path.join(REPO_ROOT, 'gepa-phase-5/cost-probe-2026-04-29.jsonl');
|
||||
const OUT_SUMMARY = path.join(REPO_ROOT, 'gepa-phase-5/cost-probe-2026-04-29-summary.md');
|
||||
|
||||
const LITELLM_URL = process.env.LITELLM_URL ?? 'http://localhost:4000';
|
||||
const LITELLM_MASTER_KEY = process.env.LITELLM_MASTER_KEY;
|
||||
|
||||
// 2026-04-29 pricing snapshot (Phase 5 binding)
|
||||
const PRICING = {
|
||||
'claude::gen1-v1': { in_per_m: 5.0, out_per_m: 25.0, model_alias: 'claude-opus-4-7' },
|
||||
'qwen-thinking::gen1-v1': {
|
||||
in_per_m: 0.25,
|
||||
out_per_m: 2.0,
|
||||
model_alias: 'qwen3.6-35b-a3b-via-dashscope-direct',
|
||||
},
|
||||
} as const;
|
||||
|
||||
interface ProbePrompt {
|
||||
complexity: 'trivial' | 'medium-1' | 'medium-2' | 'complex' | 'stretch';
|
||||
persona: string;
|
||||
materials: string;
|
||||
question: string;
|
||||
max_tokens: number;
|
||||
}
|
||||
|
||||
const PROMPTS: ProbePrompt[] = [
|
||||
{
|
||||
complexity: 'trivial',
|
||||
persona: 'a concise factual analyst',
|
||||
materials: 'The number two added to the number two equals four. This is elementary arithmetic.',
|
||||
question: 'What is 2+2?',
|
||||
max_tokens: 200,
|
||||
},
|
||||
{
|
||||
complexity: 'medium-1',
|
||||
persona: 'a CFO advisor at a mid-stage SaaS startup',
|
||||
materials: [
|
||||
'Q3 metrics for Acme SaaS (Series B, $12M ARR):',
|
||||
'- ARR growth slowed from 18% QoQ to 9% QoQ.',
|
||||
'- Gross margin held at 78%.',
|
||||
'- Net dollar retention dropped from 121% to 108%.',
|
||||
'- Churn ticked up in SMB segment from 1.4% to 2.1% monthly.',
|
||||
'- Sales cycle elongated 22% in mid-market.',
|
||||
'- Two competitors raised Series C at higher valuations.',
|
||||
].join('\n'),
|
||||
question: 'List the 3 highest-priority risks the CEO should brief the board on, in order.',
|
||||
max_tokens: 600,
|
||||
},
|
||||
{
|
||||
complexity: 'medium-2',
|
||||
persona: 'a software architect advising on legacy modernization',
|
||||
materials: [
|
||||
'Context: 8-year-old Rails monolith. ~400k LOC. Critical revenue path.',
|
||||
'Team: 12 backend engineers, 4 platform engineers. No prior modernization experience.',
|
||||
'Pressure: Sales reports the monolith blocks integration deals worth ~$8M ARR.',
|
||||
'Constraints: 9-month runway-extension clock; CEO wants visible progress every 60 days.',
|
||||
].join('\n'),
|
||||
question: 'Compare strangler-fig vs big-bang refactor for this team. Recommend one with explicit risk acknowledgment.',
|
||||
max_tokens: 800,
|
||||
},
|
||||
{
|
||||
complexity: 'complex',
|
||||
persona: 'a strategic operations consultant',
|
||||
materials: [
|
||||
'NorthLane Retail (mid-market apparel chain):',
|
||||
'- Q3 SSS missed plan by 8% (-3.2% vs +5% planned).',
|
||||
'- Macro: discretionary spend down 4% sector-wide; 30y yield up 80bps in quarter.',
|
||||
'- Competitive: Aritzia opened 4 new doors in core trade areas; Old Navy ran 30% friends-and-family promo for 6 weeks.',
|
||||
'- Internal: New POS system rollout caused 6% transaction-error rate for 3 weeks; e-commerce site speed regressed (LCP 2.4s -> 4.1s).',
|
||||
'- Brand: Latest TikTok creator partnership underperformed engagement targets by 60%; Brand consideration score flat.',
|
||||
'- Inventory: $42M aged 90+ days; merch margin compressed 180bps to fund clearance.',
|
||||
].join('\n'),
|
||||
question: 'Diagnose root causes across 4 dimensions (macro / competitive / internal execution / brand). For each, attribute approximate share of the 8% miss + propose one targeted Q4 intervention.',
|
||||
max_tokens: 1200,
|
||||
},
|
||||
{
|
||||
complexity: 'stretch',
|
||||
persona: 'an M&A analyst preparing an investment memo for the deal committee',
|
||||
materials: [
|
||||
'Target: Cobalt Cloud (Series D enterprise SaaS, governance + audit logging).',
|
||||
'- ARR: $50M trailing; +52% YoY; 110% NDR.',
|
||||
'- Customers: 320 logos, top-10 = 28% of ARR. Net new logos +18% YoY.',
|
||||
'- Gross margin: 76% (best-in-class for category 73-78%).',
|
||||
'- Sales efficiency (Magic Number): 1.1 (LTM); CAC payback 16 months.',
|
||||
'- Burn: $4M/quarter, 18 months runway.',
|
||||
'- Last round: $750M post on $35M ARR (21x), 2.5 years ago.',
|
||||
'- Comps: Drata (~25x ARR), Vanta (~22x ARR), Hyperproof (~14x ARR), AuditBoard (public, 8x).',
|
||||
'- Recent context: 2 strategic offers verbal at 12-15x ARR. Founder open to strategic at 15x+.',
|
||||
].join('\n'),
|
||||
question: [
|
||||
'Walk through three valuation methods for Cobalt Cloud:',
|
||||
'1) DCF — state assumptions explicit (revenue growth fade, margin trajectory, discount rate, terminal multiple).',
|
||||
'2) Trading comps — apply weighted multiple from 4 comps to next-twelve-months revenue.',
|
||||
'3) LBO — assume Sponsor offer at $750M EV, 6.5x leverage, 5y hold, exit at 12x EBITDA. State minimum revenue CAGR for 25% IRR.',
|
||||
'Reconcile any gap > 25% between the three methods + recommend offer band.',
|
||||
].join('\n'),
|
||||
max_tokens: 1500,
|
||||
},
|
||||
];
|
||||
|
||||
interface ProbeResult {
|
||||
variant: 'claude::gen1-v1' | 'qwen-thinking::gen1-v1';
|
||||
complexity: ProbePrompt['complexity'];
|
||||
model_alias: string;
|
||||
input_tokens: number;
|
||||
output_tokens: number;
|
||||
cost_usd: number;
|
||||
latency_ms: number;
|
||||
ts: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
function appendJsonl(file: string, row: ProbeResult) {
|
||||
fs.appendFileSync(file, JSON.stringify(row) + '\n');
|
||||
}
|
||||
|
||||
async function callViaLitellm(model: string, system: string, user: string, max_tokens: number) {
|
||||
if (!LITELLM_MASTER_KEY) throw new Error('LITELLM_MASTER_KEY not set');
|
||||
const payload = {
|
||||
model,
|
||||
messages: [
|
||||
{ role: 'system', content: system },
|
||||
{ role: 'user', content: user },
|
||||
],
|
||||
max_tokens,
|
||||
};
|
||||
const t0 = Date.now();
|
||||
const resp = await fetch(`${LITELLM_URL}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${LITELLM_MASTER_KEY}`,
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const latency_ms = Date.now() - t0;
|
||||
const data = (await resp.json()) as Record<string, unknown>;
|
||||
if (!resp.ok || 'error' in data) {
|
||||
const errMsg =
|
||||
(data as { error?: { message?: string } }).error?.message ??
|
||||
`HTTP ${resp.status}: ${JSON.stringify(data).slice(0, 200)}`;
|
||||
return { ok: false as const, error: String(errMsg), latency_ms };
|
||||
}
|
||||
const choices = data.choices as Array<{ message: { content: string } }> | undefined;
|
||||
const usage = data.usage as { prompt_tokens?: number; completion_tokens?: number } | undefined;
|
||||
return {
|
||||
ok: true as const,
|
||||
content: choices?.[0]?.message?.content ?? '',
|
||||
input_tokens: usage?.prompt_tokens ?? 0,
|
||||
output_tokens: usage?.completion_tokens ?? 0,
|
||||
latency_ms,
|
||||
};
|
||||
}
|
||||
|
||||
async function probe(
|
||||
variant: 'claude::gen1-v1' | 'qwen-thinking::gen1-v1',
|
||||
prompt: ProbePrompt,
|
||||
): Promise<ProbeResult> {
|
||||
const shape = variant === 'claude::gen1-v1' ? claudeGen1V1Shape : qwenThinkingGen1V1Shape;
|
||||
const config = PRICING[variant];
|
||||
|
||||
const sysPrompt = shape.systemPrompt({
|
||||
persona: prompt.persona,
|
||||
question: prompt.question,
|
||||
isMultiStep: false,
|
||||
});
|
||||
const userPrompt = shape.soloUserPrompt({
|
||||
persona: prompt.persona,
|
||||
materials: prompt.materials,
|
||||
question: prompt.question,
|
||||
});
|
||||
|
||||
const result = await callViaLitellm(config.model_alias, sysPrompt, userPrompt, prompt.max_tokens);
|
||||
|
||||
if (!result.ok) {
|
||||
return {
|
||||
variant,
|
||||
complexity: prompt.complexity,
|
||||
model_alias: config.model_alias,
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
cost_usd: 0,
|
||||
latency_ms: result.latency_ms,
|
||||
ts: new Date().toISOString(),
|
||||
error: result.error,
|
||||
};
|
||||
}
|
||||
|
||||
const cost_usd =
|
||||
(result.input_tokens * config.in_per_m) / 1e6 +
|
||||
(result.output_tokens * config.out_per_m) / 1e6;
|
||||
return {
|
||||
variant,
|
||||
complexity: prompt.complexity,
|
||||
model_alias: config.model_alias,
|
||||
input_tokens: result.input_tokens,
|
||||
output_tokens: result.output_tokens,
|
||||
cost_usd,
|
||||
latency_ms: result.latency_ms,
|
||||
ts: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function percentile(values: number[], p: number): number {
|
||||
if (values.length === 0) return 0;
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1));
|
||||
return sorted[idx];
|
||||
}
|
||||
|
||||
interface VariantSummary {
|
||||
variant: string;
|
||||
count: number;
|
||||
errors: number;
|
||||
p50: number;
|
||||
p95: number;
|
||||
max: number;
|
||||
mean: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
function summarize(results: ProbeResult[], variant: string): VariantSummary {
|
||||
const ok = results.filter((r) => r.variant === variant && !r.error);
|
||||
const errors = results.filter((r) => r.variant === variant && r.error).length;
|
||||
if (ok.length === 0) {
|
||||
return {
|
||||
variant,
|
||||
count: 0,
|
||||
errors,
|
||||
p50: NaN,
|
||||
p95: NaN,
|
||||
max: NaN,
|
||||
mean: NaN,
|
||||
total: 0,
|
||||
};
|
||||
}
|
||||
const costs = ok.map((r) => r.cost_usd);
|
||||
const total = costs.reduce((a, b) => a + b, 0);
|
||||
return {
|
||||
variant,
|
||||
count: ok.length,
|
||||
errors,
|
||||
p50: percentile(costs, 50),
|
||||
p95: percentile(costs, 95),
|
||||
max: Math.max(...costs),
|
||||
mean: total / ok.length,
|
||||
total,
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
fs.writeFileSync(OUT_JSONL, '');
|
||||
|
||||
console.log('Phase 5 §0.3 cost probe — start');
|
||||
console.log(`LiteLLM: ${LITELLM_URL}`);
|
||||
console.log(`Output JSONL: ${OUT_JSONL}`);
|
||||
|
||||
const results: ProbeResult[] = [];
|
||||
|
||||
for (const variant of ['claude::gen1-v1', 'qwen-thinking::gen1-v1'] as const) {
|
||||
for (const prompt of PROMPTS) {
|
||||
console.log(` ${variant} <- ${prompt.complexity}`);
|
||||
const r = await probe(variant, prompt);
|
||||
results.push(r);
|
||||
appendJsonl(OUT_JSONL, r);
|
||||
if (r.error) console.log(` ERROR: ${r.error}`);
|
||||
else
|
||||
console.log(
|
||||
` in=${r.input_tokens} out=${r.output_tokens} cost=$${r.cost_usd.toFixed(4)} latency=${r.latency_ms}ms`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const claudeSummary = summarize(results, 'claude::gen1-v1');
|
||||
const qwenSummary = summarize(results, 'qwen-thinking::gen1-v1');
|
||||
|
||||
const validP95s = [claudeSummary.p95, qwenSummary.p95].filter(
|
||||
(v) => Number.isFinite(v) && v > 0,
|
||||
);
|
||||
const max_p95 = validP95s.length > 0 ? Math.max(...validP95s) : NaN;
|
||||
const VOLUME = 740;
|
||||
const BUFFER = 1.2;
|
||||
const canary_cost_p95_ceiling = Number.isFinite(max_p95) ? VOLUME * max_p95 * BUFFER : NaN;
|
||||
const HALT = 20;
|
||||
const HARD_CAP = 25;
|
||||
|
||||
const lines: string[] = [];
|
||||
lines.push('# Phase 5 §0.3 Cost Probe Summary');
|
||||
lines.push('');
|
||||
lines.push(`**Date:** ${new Date().toISOString()}`);
|
||||
lines.push(`**Pricing snapshot:** 2026-04-29`);
|
||||
lines.push(`**Branch:** phase-5-deployment-v2`);
|
||||
lines.push(`**Endpoint:** ${LITELLM_URL} (LiteLLM proxy, matches Faza 1 runner pattern)`);
|
||||
lines.push('');
|
||||
lines.push('## Per-variant statistics');
|
||||
lines.push('');
|
||||
lines.push('| Variant | Model alias | OK | Errors | p50 | p95 | max | mean | total |');
|
||||
lines.push('|---|---|---|---|---|---|---|---|---|');
|
||||
for (const s of [claudeSummary, qwenSummary]) {
|
||||
const config = PRICING[s.variant as keyof typeof PRICING];
|
||||
if (s.count === 0) {
|
||||
lines.push(`| ${s.variant} | ${config.model_alias} | 0 | ${s.errors} | n/a | n/a | n/a | n/a | n/a |`);
|
||||
} else {
|
||||
lines.push(
|
||||
`| ${s.variant} | ${config.model_alias} | ${s.count} | ${s.errors} | $${s.p50.toFixed(4)} | $${s.p95.toFixed(4)} | $${s.max.toFixed(4)} | $${s.mean.toFixed(4)} | $${s.total.toFixed(4)} |`,
|
||||
);
|
||||
}
|
||||
}
|
||||
lines.push('');
|
||||
lines.push('## Canary cost ceiling (per brief §5.4)');
|
||||
lines.push('');
|
||||
lines.push(
|
||||
`Formula: \`canary_cost_p95_ceiling = ${VOLUME} requests × max(p95) × ${BUFFER} (buffer)\``,
|
||||
);
|
||||
lines.push('');
|
||||
if (Number.isFinite(max_p95)) {
|
||||
lines.push(`max(p95) = $${max_p95.toFixed(4)}`);
|
||||
lines.push(
|
||||
`canary_cost_p95_ceiling = ${VOLUME} × $${max_p95.toFixed(4)} × ${BUFFER} = **$${canary_cost_p95_ceiling.toFixed(2)}**`,
|
||||
);
|
||||
} else {
|
||||
lines.push(`max(p95) = NaN (no successful samples)`);
|
||||
lines.push(`canary_cost_p95_ceiling = INDETERMINATE`);
|
||||
}
|
||||
lines.push('');
|
||||
lines.push('## Verdict');
|
||||
lines.push('');
|
||||
if (Number.isFinite(canary_cost_p95_ceiling)) {
|
||||
if (canary_cost_p95_ceiling <= HALT) {
|
||||
lines.push(
|
||||
`**PASS** — ceiling $${canary_cost_p95_ceiling.toFixed(2)} <= halt trigger $${HALT}.`,
|
||||
);
|
||||
} else if (canary_cost_p95_ceiling <= HARD_CAP) {
|
||||
lines.push(
|
||||
`**HALT-AND-PM** — ceiling $${canary_cost_p95_ceiling.toFixed(2)} > halt trigger $${HALT}, but <= hard cap $${HARD_CAP}.`,
|
||||
);
|
||||
} else {
|
||||
lines.push(
|
||||
`**HARD-CAP-EXCEED** — ceiling $${canary_cost_p95_ceiling.toFixed(2)} > hard cap $${HARD_CAP}.`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
lines.push(`**INDETERMINATE** — no valid p95.`);
|
||||
}
|
||||
lines.push('');
|
||||
lines.push('## Probe spend (this script)');
|
||||
const totalSpend = (claudeSummary.total ?? 0) + (qwenSummary.total ?? 0);
|
||||
lines.push('');
|
||||
lines.push(`Total: **$${totalSpend.toFixed(4)}**`);
|
||||
lines.push(`Brief §5.4 probe budget: $0.30-$0.50.`);
|
||||
lines.push('');
|
||||
lines.push('## JSONL anchor');
|
||||
lines.push('');
|
||||
lines.push('Per-request rows: `gepa-phase-5/cost-probe-2026-04-29.jsonl`');
|
||||
|
||||
const summaryText = lines.join('\n');
|
||||
fs.writeFileSync(OUT_SUMMARY, summaryText);
|
||||
|
||||
console.log('');
|
||||
console.log('==========================================');
|
||||
console.log(summaryText);
|
||||
console.log('==========================================');
|
||||
console.log(`Summary: ${OUT_SUMMARY}`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('FATAL:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
244
gepa-phase-5/scripts/phase-5-daily-summary.ts
Normal file
244
gepa-phase-5/scripts/phase-5-daily-summary.ts
Normal file
@@ -0,0 +1,244 @@
|
||||
#!/usr/bin/env tsx
|
||||
/**
|
||||
* Phase 5 §3.3 — Daily monitoring summary aggregator.
|
||||
*
|
||||
* Reads JSONL emit files from `gepa-phase-5/monitoring/<ISO_date>/<variant>.jsonl`
|
||||
* and alert log from `gepa-phase-5/phase-5-alerts/<ISO_date>.jsonl`, then
|
||||
* writes a markdown summary to `gepa-phase-5/phase-5-daily-summary/<ISO_date>.md`.
|
||||
*
|
||||
* Per brief §3.3 reading cadence: PM-side reading is 1×/day during canary
|
||||
* Day 0-5, then 2×/week. This script is intended to run once per UTC day.
|
||||
*
|
||||
* Per brief §3.4 Stage 1 deliverable: JSONL files + daily markdown summary
|
||||
* (no UI). Stage 2 dashboard deferred post-launch.
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx gepa-phase-5/scripts/phase-5-daily-summary.ts # default = today UTC
|
||||
* npx tsx gepa-phase-5/scripts/phase-5-daily-summary.ts 2026-04-30
|
||||
*
|
||||
* Audit: gepa-phase-5/manifest.yaml § halt_and_pm_triggers, § rollback_triggers.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
// gepa-phase-5/scripts/phase-5-daily-summary.ts → repo root
|
||||
const REPO_ROOT = path.resolve(path.dirname(__filename), '../..');
|
||||
const MONITORING_DIR = path.join(REPO_ROOT, 'gepa-phase-5', 'monitoring');
|
||||
const ALERTS_DIR = path.join(REPO_ROOT, 'gepa-phase-5', 'phase-5-alerts');
|
||||
const SUMMARY_DIR = path.join(REPO_ROOT, 'gepa-phase-5', 'phase-5-daily-summary');
|
||||
|
||||
interface MetricEntry {
|
||||
ts: string;
|
||||
variant: string;
|
||||
request_id: string;
|
||||
metric_name: string;
|
||||
metric_value: number;
|
||||
baseline_comparison?: { baseline_value: number; delta?: number };
|
||||
error_type?: string;
|
||||
}
|
||||
|
||||
interface AlertEntry {
|
||||
ts: string;
|
||||
trigger_id: string;
|
||||
variant: string;
|
||||
metric_name: string;
|
||||
observed_value: number;
|
||||
threshold: number;
|
||||
is_rollback_trigger: boolean;
|
||||
diagnostic?: string;
|
||||
}
|
||||
|
||||
function todayIsoUtc(): string {
|
||||
return new Date().toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function readJsonlSafe<T>(filePath: string): T[] {
|
||||
if (!fs.existsSync(filePath)) return [];
|
||||
const raw = fs.readFileSync(filePath, 'utf-8');
|
||||
const lines = raw.split('\n').filter((l) => l.trim().length > 0);
|
||||
return lines.map((l) => JSON.parse(l) as T);
|
||||
}
|
||||
|
||||
function p50(values: number[]): number | null {
|
||||
if (values.length === 0) return null;
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
return sorted[Math.floor(sorted.length / 2)] ?? null;
|
||||
}
|
||||
|
||||
function p95(values: number[]): number | null {
|
||||
if (values.length === 0) return null;
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
return sorted[Math.floor(sorted.length * 0.95)] ?? null;
|
||||
}
|
||||
|
||||
function mean(values: number[]): number | null {
|
||||
if (values.length === 0) return null;
|
||||
return values.reduce((a, b) => a + b, 0) / values.length;
|
||||
}
|
||||
|
||||
function ensureDir(dir: string): void {
|
||||
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
interface VariantSummary {
|
||||
variant: string;
|
||||
totalEvents: number;
|
||||
perMetric: Record<
|
||||
string,
|
||||
{
|
||||
n: number;
|
||||
mean?: number;
|
||||
p50?: number;
|
||||
p95?: number;
|
||||
}
|
||||
>;
|
||||
errorBreakdown?: Record<string, number>;
|
||||
}
|
||||
|
||||
function summarizeVariant(variant: string, entries: MetricEntry[]): VariantSummary {
|
||||
const summary: VariantSummary = {
|
||||
variant,
|
||||
totalEvents: entries.length,
|
||||
perMetric: {},
|
||||
};
|
||||
const byMetric = new Map<string, number[]>();
|
||||
const errorTypes = new Map<string, number>();
|
||||
for (const e of entries) {
|
||||
if (!byMetric.has(e.metric_name)) byMetric.set(e.metric_name, []);
|
||||
byMetric.get(e.metric_name)!.push(e.metric_value);
|
||||
if (e.metric_name === 'error' && e.error_type) {
|
||||
errorTypes.set(e.error_type, (errorTypes.get(e.error_type) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
for (const [metric, vals] of byMetric.entries()) {
|
||||
summary.perMetric[metric] = {
|
||||
n: vals.length,
|
||||
mean: mean(vals) ?? undefined,
|
||||
p50: p50(vals) ?? undefined,
|
||||
p95: p95(vals) ?? undefined,
|
||||
};
|
||||
}
|
||||
if (errorTypes.size > 0) {
|
||||
summary.errorBreakdown = Object.fromEntries(errorTypes.entries());
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
function fmtNum(v: number | undefined, decimals = 4): string {
|
||||
if (v === undefined || !Number.isFinite(v)) return '-';
|
||||
return v.toFixed(decimals);
|
||||
}
|
||||
|
||||
function formatMarkdown(date: string, summaries: VariantSummary[], alerts: AlertEntry[]): string {
|
||||
const lines: string[] = [];
|
||||
lines.push(`# Phase 5 Daily Summary — ${date}`);
|
||||
lines.push('');
|
||||
lines.push(`**Generated:** ${new Date().toISOString()}`);
|
||||
lines.push(`**Source:** \`gepa-phase-5/monitoring/${date}/*.jsonl\` + \`gepa-phase-5/phase-5-alerts/${date}.jsonl\``);
|
||||
lines.push(`**Manifest:** \`gepa-phase-5/manifest.yaml\``);
|
||||
lines.push('');
|
||||
|
||||
// Alerts first (top of summary so PM sees rollback triggers immediately).
|
||||
lines.push('## Alerts');
|
||||
lines.push('');
|
||||
if (alerts.length === 0) {
|
||||
lines.push('_No alerts._');
|
||||
} else {
|
||||
const rollbackAlerts = alerts.filter((a) => a.is_rollback_trigger);
|
||||
const informational = alerts.filter((a) => !a.is_rollback_trigger);
|
||||
if (rollbackAlerts.length > 0) {
|
||||
lines.push(`### 🚨 ROLLBACK TRIGGERS (${rollbackAlerts.length})`);
|
||||
lines.push('');
|
||||
for (const a of rollbackAlerts) {
|
||||
lines.push(`- **${a.trigger_id}** — ${a.variant} — \`${a.metric_name}\` observed=${fmtNum(a.observed_value)} threshold=${fmtNum(a.threshold)}`);
|
||||
if (a.diagnostic) lines.push(` - ${a.diagnostic}`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
if (informational.length > 0) {
|
||||
lines.push(`### Informational alerts (${informational.length})`);
|
||||
lines.push('');
|
||||
for (const a of informational) {
|
||||
lines.push(`- ${a.trigger_id} — ${a.variant} — observed=${fmtNum(a.observed_value)}`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
// Per-variant aggregates.
|
||||
lines.push('## Per-variant metrics');
|
||||
lines.push('');
|
||||
if (summaries.length === 0) {
|
||||
lines.push('_No metric events recorded._');
|
||||
} else {
|
||||
for (const s of summaries) {
|
||||
lines.push(`### ${s.variant} (events: ${s.totalEvents})`);
|
||||
lines.push('');
|
||||
lines.push('| Metric | n | mean | p50 | p95 |');
|
||||
lines.push('|---|---|---|---|---|');
|
||||
for (const [metric, agg] of Object.entries(s.perMetric)) {
|
||||
lines.push(
|
||||
`| ${metric} | ${agg.n} | ${fmtNum(agg.mean)} | ${fmtNum(agg.p50)} | ${fmtNum(agg.p95)} |`,
|
||||
);
|
||||
}
|
||||
if (s.errorBreakdown) {
|
||||
lines.push('');
|
||||
lines.push('**Error breakdown:**');
|
||||
for (const [type, count] of Object.entries(s.errorBreakdown)) {
|
||||
lines.push(`- ${type}: ${count}`);
|
||||
}
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
}
|
||||
|
||||
lines.push('---');
|
||||
lines.push('');
|
||||
lines.push('**Cross-references:**');
|
||||
lines.push('- Manifest: `gepa-phase-5/manifest.yaml` (LOCKED scope + thresholds)');
|
||||
lines.push('- Brief: `D:/Projects/PM-Waggle-OS/briefs/2026-04-29-phase-5-deployment-brief-v1.md` §3 monitoring + §4 exit criteria');
|
||||
lines.push('- Cost amendment: `D:/Projects/PM-Waggle-OS/decisions/2026-04-30-phase-5-cost-amendment-LOCKED.md`');
|
||||
lines.push('');
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const date = process.argv[2] ?? todayIsoUtc();
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) {
|
||||
process.stderr.write(`Invalid date format: ${date}. Expected YYYY-MM-DD.\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const monitoringDateDir = path.join(MONITORING_DIR, date);
|
||||
const summaries: VariantSummary[] = [];
|
||||
if (fs.existsSync(monitoringDateDir)) {
|
||||
const files = fs.readdirSync(monitoringDateDir).filter((f) => f.endsWith('.jsonl'));
|
||||
for (const file of files) {
|
||||
const variantSafe = file.replace(/\.jsonl$/, '');
|
||||
const entries = readJsonlSafe<MetricEntry>(path.join(monitoringDateDir, file));
|
||||
// Recover real variant name from entries[0] (sanitize is one-way).
|
||||
const realVariant = entries[0]?.variant ?? variantSafe;
|
||||
summaries.push(summarizeVariant(realVariant, entries));
|
||||
}
|
||||
}
|
||||
|
||||
const alertsFile = path.join(ALERTS_DIR, `${date}.jsonl`);
|
||||
const alerts = readJsonlSafe<AlertEntry>(alertsFile);
|
||||
|
||||
const markdown = formatMarkdown(date, summaries, alerts);
|
||||
|
||||
ensureDir(SUMMARY_DIR);
|
||||
const outFile = path.join(SUMMARY_DIR, `${date}.md`);
|
||||
fs.writeFileSync(outFile, markdown, 'utf-8');
|
||||
|
||||
process.stdout.write(`Wrote: ${outFile}\n`);
|
||||
process.stdout.write(`Variants summarized: ${summaries.length}\n`);
|
||||
process.stdout.write(`Alerts: ${alerts.length} (rollback triggers: ${alerts.filter((a) => a.is_rollback_trigger).length})\n`);
|
||||
}
|
||||
|
||||
main();
|
||||
Reference in New Issue
Block a user