moving

packages/agent/tests/eval/scenarios-prompt-assembler-v5.ts

@@ -0,0 +1,241 @@
+/**
+ * PromptAssembler v5 eval scenarios — see docs/specs/PROMPT-ASSEMBLER-V4.md
+ * and the v5 brief §11.4.
+ *
+ * v5 changes relative to v4 (scenarios-prompt-assembler.ts): ONLY the
+ * primingTurns are revised. Names, shapes, languages, test turns,
+ * memoryVerificationSubstrings, and rubricHints are preserved verbatim so
+ * v4 and v5 eval results are directly comparable.
+ *
+ * Root cause of v4's priming misses: autoSaveFromExchange's decision-
+ * extractor pulls the sentence matching a decision-trigger regex
+ * ("we decided", "we'll use", "going with"). Facts in adjacent sentences
+ * did not save. v5 co-locates each target fact with its trigger in the
+ * SAME sentence.
+ */
+
+// v5 re-uses v4's type contracts — no shape change, just revised primings.
+export type { ScenarioLanguage, PrimingTurn, PromptAssemblerScenario } from './scenarios-prompt-assembler.js';
+import type { PromptAssemblerScenario } from './scenarios-prompt-assembler.js';
+
+export const SCENARIOS_V5: PromptAssemblerScenario[] = [
+  // ── Scenario 1 — Analysis/Decide, Serbian ──────────────────────────
+  // v5 note: v4 already passed all three substrings. Priming minor-
+  // strengthened only — "for all three initial customers" + "Data
+  // residency is the non-negotiable driver" pulls the three signals
+  // (on-prem, H200, data residency) together.
+  {
+    name: 'sovereignty-deployment',
+    shape: 'decide',
+    language: 'sr',
+    primingTurns: [
+      {
+        user:
+          'Imamo novi projekat. Tri početna enterprise klijenta — banke i telco iz ' +
+          'regiona. Svi imaju regulatorne zahteve za data residency u Srbiji, to je ' +
+          'tvrdo ograničenje.',
+      },
+      {
+        user:
+          'We decided to go with on-prem deployment on our H200 x8 hardware for all ' +
+          'three initial customers. Suverenitet je core value proposition — klijenti ' +
+          'ne žele hyperscaler cloud. Data residency is the non-negotiable driver.',
+      },
+    ],
+    testTurn: {
+      query:
+        'Sumiraj naš deployment pristup za prva tri klijenta i obrazloži zašto smo tako odlučili.',
+    },
+    memoryVerificationSubstrings: ['data residency', 'on-prem', 'H200'],
+    rubricHints:
+      'Should cite on-prem decision, reference data-residency constraints, mention ' +
+      'sovereignty positioning, acknowledge H200 hardware. Serbian response expected. ' +
+      'Classifier confidence likely low on Serbian query — scaffold likely not applied. ' +
+      "That's acceptable.",
+  },
+
+  // ── Scenario 2 — Compare, English ──────────────────────────────────
+  // v5 fix: "24-agent" wasn't saved in v4 because the MECE decision
+  // frame was about simple workflows, not the 24-agent case. Second
+  // turn now contains "I decided we need to pick between MECE or BPMN
+  // specifically for this 24-agent case" — one sentence, decision
+  // trigger + the 24-agent substring co-located.
+  {
+    name: 'decomposition-choice',
+    shape: 'compare',
+    language: 'en',
+    primingTurns: [
+      {
+        user:
+          'I ran a decomposition experiment last week. Finding: MECE is the ' +
+          'cost-efficient winner — same IC% as BPMN at 2-4x lower token cost. ' +
+          "BPMN wins on gate complexity: 14 LLM calls vs MECE's 8 for equivalent " +
+          'gate logic. I decided MECE is our default for simple workflows.',
+      },
+      {
+        user:
+          'Now a new challenge. An energy client wants a 24-agent workflow with ' +
+          'complex cross-agent dependencies throughout: orchestration, approvals, ' +
+          'compensation, rollback. I decided we need to pick between MECE or BPMN ' +
+          'specifically for this 24-agent case — help me choose.',
+      },
+    ],
+    testTurn: {
+      query:
+        'Compare MECE vs BPMN for this 24-agent workflow. Which method should we use and why?',
+    },
+    memoryVerificationSubstrings: ['MECE', 'BPMN', '24-agent'],
+    rubricHints:
+      'Should recommend BPMN for the complex gates despite higher cost; acknowledge ' +
+      'MECE as simpler-default; state trade-off explicitly. Expected scaffold: ' +
+      'analysis (assumption → trade-offs → recommendation).',
+  },
+
+  // ── Scenario 3 — Plan-execute, English ─────────────────────────────
+  // v5 fix: v4 saved only 2 frames; "$29" and "Stripe" and "workspace
+  // mind" didn't land. Split into 3 priming turns, each landing a
+  // decision + fact pair in the same sentence. First turn: pricing
+  // numbers. Second turn: Stripe/M2-2 blocker. Third turn: architecture
+  // + migration path.
+  {
+    name: 'migration-plan',
+    shape: 'plan-execute',
+    language: 'en',
+    primingTurns: [
+      {
+        user:
+          'Our pricing model is decided. I want you to remember these exact figures: ' +
+          'Solo is free, Teams is $29 per user per month, Business is $79 per user per ' +
+          'month. These numbers matter for any migration math.',
+      },
+      {
+        user:
+          'Technical dependency: Teams tier requires Stripe integration for billing ' +
+          '— the cloud webhook, pending as M2-2 in our sprint. We decided Teams cannot ' +
+          'ship to customers until Stripe is wired.',
+      },
+      {
+        user:
+          'Data architecture decision: Solo uses local SQLite per user, Teams adds a ' +
+          'shared workspace mind on top with team sync. Personal minds stay local. ' +
+          "Migration path we decided: user's local SQLite frames replicate to the " +
+          'workspace mind on first Teams login.',
+      },
+    ],
+    testTurn: {
+      query:
+        'Create a plan to migrate a 10-person design firm from Waggle Solo to Waggle Teams. ' +
+        'Break it down into concrete steps including any blockers.',
+    },
+    memoryVerificationSubstrings: ['$29', 'Teams', 'Stripe', 'workspace mind'],
+    rubricHints:
+      'Numbered plan ~5-7 steps, Stripe/M2-2 as blocker, data migration ' +
+      '(local → workspace mind), total cost ($290/mo). Expected scaffold: ' +
+      'execution (confirm inputs → plan → execute → report).',
+  },
+
+  // ── Scenario 4 — Research, English ─────────────────────────────────
+  // v5 fix: v4 missed "license boundary" and "non-negotiable" as
+  // substrings. Second turn now includes "the KVARK license boundary
+  // in this deal is deployment-only" and "We decided the license
+  // boundary is a hard non-negotiable constraint" — both phrases
+  // present in decision sentences.
+  {
+    name: 'license-boundary',
+    shape: 'research',
+    language: 'en',
+    primingTurns: [
+      {
+        user:
+          "We're preparing a proposal for Yettel Serbia — AI and MLOps platform " +
+          'based on our KVARK core plus custom connectors for their telco systems.',
+      },
+      {
+        user:
+          'Critical decision — and this is non-negotiable: the KVARK license boundary ' +
+          'in this deal is deployment-only, we do not license source code. KVARK remains ' +
+          'Egzakta property. We decided the license boundary is a hard non-negotiable ' +
+          'constraint, because it protects our IP so we can reuse KVARK for other clients.',
+      },
+    ],
+    testTurn: {
+      query:
+        'What is the KVARK license boundary in the Yettel proposal, and why is it non-negotiable?',
+    },
+    memoryVerificationSubstrings: ['KVARK', 'license boundary', 'non-negotiable'],
+    rubricHints:
+      'Cite the specific fact (boundary non-negotiable) and the reason ' +
+      '(IP separation, KVARK stays Egzakta). Direct answer, no hedging. ' +
+      'Expected scaffold: retrieval (cite frame → quote → answer).',
+  },
+
+  // ── Scenario 5 — Research, Serbian ─────────────────────────────────
+  // v5 fix: "Clipperton" missed in v4 — the NDA-signing decision frame
+  // cut off before the name. Second turn restructured so the English
+  // decision trigger "we decided to move forward with Clipperton
+  // Finance" appears in the same sentence as the name.
+  {
+    name: 'investor-status',
+    shape: 'research',
+    language: 'sr',
+    primingTurns: [
+      {
+        user:
+          'Radimo rundu investicije. Cilj nam je EUR 20M, pre-money procena između ' +
+          '70 i 80 miliona evra.',
+      },
+      {
+        user:
+          'Active investor contact: we decided to move forward with Clipperton ' +
+          'Finance, partner Dr. Nikolas Westphal. NDA is signed, pitch deck je ' +
+          'poslat. Trenutno su u fazi dubinske analize, čekamo povratnu ' +
+          'informaciju sa Clipperton strane.',
+      },
+    ],
+    testTurn: {
+      query: 'Ko su aktivni investitori za našu rundu i u kojoj fazi smo sa njima?',
+    },
+    memoryVerificationSubstrings: ['Clipperton', 'Westphal', '20M'],
+    rubricHints:
+      'Should name Clipperton Finance and Dr. Nikolas Westphal, state status ' +
+      '(NDA signed, deck sent, due diligence). Serbian response. Low classifier ' +
+      'confidence likely → no scaffold. Tests whether mid-Serbian-context ' +
+      'bilingual priming saved the facts.',
+  },
+
+  // ── Scenario 6 — Draft, English ────────────────────────────────────
+  // v5 fix: "Mistral" missed in v4 — consortium-partner mention was
+  // narrative, not a decision. Second turn now contains "Decision on
+  // consortium partner: we're going with Mistral AI" with both
+  // "decision" and "going with" in the same sentence as "Mistral".
+  {
+    name: 'floodtwin-summary',
+    shape: 'draft',
+    language: 'en',
+    primingTurns: [
+      {
+        user:
+          "We're drafting FloodTwin-WB — a concept for the EU Horizon 2026 call. " +
+          'Flood digital twin for the Western Balkans. Deadline April 2026.',
+      },
+      {
+        user:
+          'Scope we decided on: Serbia plus five Western Balkan countries. Existing ' +
+          "hydro models are siloed per country. We'll use a cross-border digital twin " +
+          'with real-time sensor fusion to unify them. Decision on consortium ' +
+          "partner: we're going with Mistral AI because the sovereignty narrative " +
+          'strengthens the EU angle.',
+      },
+    ],
+    testTurn: {
+      query: 'Draft a 150-word executive summary for the FloodTwin-WB proposal.',
+    },
+    memoryVerificationSubstrings: ['Western Balkans', 'Mistral', 'cross-border'],
+    rubricHints:
+      'Creative task — judge on coherence and inclusion of key elements ' +
+      '(Western Balkans, Mistral, cross-border unification, EU sovereignty). ' +
+      "NO scaffold should apply — `draft` shape maps to creation category, " +
+      'no scaffold at any tier. If an expansion-style C2 condition emits a ' +
+      "scaffold here, it's a classification bug.",
+  },
+];