moving

packages/agent/tests/eval/scenarios-prompt-assembler.ts

@@ -0,0 +1,225 @@
+/**
+ * PromptAssembler eval scenarios — see docs/specs/PROMPT-ASSEMBLER-V4.md §13.
+ *
+ * Each scenario is a mini-conversation:
+ *  1. primingTurns run via Sonnet 4.6 to populate memory organically through
+ *     the real save_memory / cognify path.
+ *  2. memoryVerificationSubstrings confirm priming actually saved frames.
+ *  3. testTurn runs under each of the 6 primary + 4 secondary conditions.
+ *
+ * Every priming user message includes at least one English save-trigger phrase
+ * ("decided", "we'll use", "I prefer", "going with") so autoSaveFromExchange
+ * fires even on Serbian-dominant content.
+ */
+
+import type { TaskShape } from '../../src/task-shape.js';
+
+export type ScenarioLanguage = 'en' | 'sr';
+
+export interface PrimingTurn {
+  user: string;
+}
+
+export interface PromptAssemblerScenario {
+  /** Stable scenario id (slug, no spaces) */
+  name: string;
+  /** Expected task shape — also used to verify the classifier landed correctly */
+  shape: TaskShape['type'];
+  /** Dominant language of the priming + test content */
+  language: ScenarioLanguage;
+  /** 2 priming turns run via Sonnet 4.6 */
+  primingTurns: PrimingTurn[];
+  /** The actual test question asked after priming */
+  testTurn: { query: string };
+  /** Substrings expected to appear in saved memory frames (verification gate) */
+  memoryVerificationSubstrings: string[];
+  /** Judge-rubric hints — what a good answer looks like */
+  rubricHints: string;
+}
+
+export const SCENARIOS: PromptAssemblerScenario[] = [
+  // ── Scenario 1 — Analysis/Decide, Serbian ──────────────────────────
+  {
+    name: 'sovereignty-deployment',
+    shape: 'decide',
+    language: 'sr',
+    primingTurns: [
+      {
+        user:
+          'Imamo novi projekat. Tri početna enterprise klijenta — banke i telco iz ' +
+          'regiona. Svi imaju regulatorne zahteve za data residency u Srbiji, to je ' +
+          'tvrdo ograničenje.',
+      },
+      {
+        user:
+          'We decided to go with on-prem deployment on our H200 x8 hardware. ' +
+          'Suverenitet je core value proposition — klijenti ne žele hyperscaler cloud. ' +
+          "We'll use our own stack for all three initial customers.",
+      },
+    ],
+    testTurn: {
+      query:
+        'Sumiraj naš deployment pristup za prva tri klijenta i obrazloži zašto smo tako odlučili.',
+    },
+    memoryVerificationSubstrings: ['data residency', 'on-prem', 'H200'],
+    rubricHints:
+      'Should cite on-prem decision, reference data-residency constraints, mention ' +
+      'sovereignty positioning, acknowledge H200 hardware. Serbian response expected. ' +
+      'Classifier confidence likely low on Serbian query — scaffold likely not applied. ' +
+      "That's acceptable.",
+  },
+
+  // ── Scenario 2 — Compare, English ──────────────────────────────────
+  {
+    name: 'decomposition-choice',
+    shape: 'compare',
+    language: 'en',
+    primingTurns: [
+      {
+        user:
+          'I ran a decomposition experiment last week. Finding: MECE is the ' +
+          'cost-efficient winner — same IC% as BPMN at 2-4x lower token cost. ' +
+          "BPMN wins on gate complexity: 14 LLM calls vs MECE's 8 for equivalent " +
+          'gate logic. I decided MECE is our default for simple workflows.',
+      },
+      {
+        user:
+          'New challenge. An energy client wants a 24-agent workflow with complex ' +
+          'cross-agent dependencies — orchestration, approvals, compensation, ' +
+          "rollback. We'll use one of the two methods for this.",
+      },
+    ],
+    testTurn: {
+      query:
+        'Compare MECE vs BPMN for this 24-agent workflow. Which method should we use and why?',
+    },
+    memoryVerificationSubstrings: ['MECE', 'BPMN', '24-agent'],
+    rubricHints:
+      'Should recommend BPMN for the complex gates despite higher cost; acknowledge ' +
+      'MECE as simpler-default; state trade-off explicitly. Expected scaffold: ' +
+      'analysis (assumption → trade-offs → recommendation).',
+  },
+
+  // ── Scenario 3 — Plan-execute, English ─────────────────────────────
+  {
+    name: 'migration-plan',
+    shape: 'plan-execute',
+    language: 'en',
+    primingTurns: [
+      {
+        user:
+          'Our product has three tiers I want you to remember. Solo is free. ' +
+          'Teams is $29/month per user. Business is $79/month. We decided Teams ' +
+          'requires cloud webhook for billing — Stripe integration, still pending as M2-2.',
+      },
+      {
+        user:
+          'Technical architecture for tiers: Solo uses local SQLite per user — fully ' +
+          'offline. Teams adds a shared workspace mind with team sync on top, but ' +
+          "personal minds remain local. Data migration path: user's local SQLite " +
+          'frames get replicated to the workspace mind on first Teams login.',
+      },
+    ],
+    testTurn: {
+      query:
+        'Create a plan to migrate a 10-person design firm from Waggle Solo to Waggle Teams. ' +
+        'Break it down into concrete steps including any blockers.',
+    },
+    memoryVerificationSubstrings: ['$29', 'Teams', 'Stripe', 'workspace mind'],
+    rubricHints:
+      'Numbered plan ~5-7 steps, Stripe/M2-2 as blocker, data migration ' +
+      '(local → workspace mind), total cost ($290/mo). Expected scaffold: ' +
+      'execution (confirm inputs → plan → execute → report).',
+  },
+
+  // ── Scenario 4 — Research, English ─────────────────────────────────
+  {
+    name: 'license-boundary',
+    shape: 'research',
+    language: 'en',
+    primingTurns: [
+      {
+        user:
+          "We're preparing a proposal for Yettel Serbia — AI and MLOps platform " +
+          'based on our KVARK core plus custom connectors for their telco systems.',
+      },
+      {
+        user:
+          'Critical clause we decided on: the KVARK license boundary must be ' +
+          'non-negotiable in this deal. We license a deployment, not the source. ' +
+          'That protects our IP — KVARK remains Egzakta property and we can use ' +
+          'it for other clients. I want you to remember this as a hard constraint.',
+      },
+    ],
+    testTurn: {
+      query:
+        'What is the KVARK license boundary in the Yettel proposal, and why is it non-negotiable?',
+    },
+    memoryVerificationSubstrings: ['KVARK', 'license boundary', 'non-negotiable'],
+    rubricHints:
+      'Cite the specific fact (boundary non-negotiable) and the reason ' +
+      '(IP separation, KVARK stays Egzakta). Direct answer, no hedging. ' +
+      'Expected scaffold: retrieval (cite frame → quote → answer).',
+  },
+
+  // ── Scenario 5 — Research, Serbian ─────────────────────────────────
+  {
+    name: 'investor-status',
+    shape: 'research',
+    language: 'sr',
+    primingTurns: [
+      {
+        user:
+          'Radimo rundu investicije. Cilj nam je EUR 20M, pre-money procena između ' +
+          '70 i 80 miliona evra.',
+      },
+      {
+        user:
+          'Aktivan kontakt je Clipperton Finance, partner Dr. Nikolas Westphal. ' +
+          'We decided to sign the NDA, pitch deck je poslat. Trenutno su u fazi ' +
+          'dubinske analize, čekamo povratnu informaciju.',
+      },
+    ],
+    testTurn: {
+      query: 'Ko su aktivni investitori za našu rundu i u kojoj fazi smo sa njima?',
+    },
+    memoryVerificationSubstrings: ['Clipperton', 'Westphal', '20M'],
+    rubricHints:
+      'Should name Clipperton Finance and Dr. Nikolas Westphal, state status ' +
+      '(NDA signed, deck sent, due diligence). Serbian response. Low classifier ' +
+      'confidence likely → no scaffold. Tests whether mid-Serbian-context ' +
+      'bilingual priming saved the facts.',
+  },
+
+  // ── Scenario 6 — Draft, English ────────────────────────────────────
+  {
+    name: 'floodtwin-summary',
+    shape: 'draft',
+    language: 'en',
+    primingTurns: [
+      {
+        user:
+          "We're drafting FloodTwin-WB — a concept for the EU Horizon 2026 call. " +
+          'Flood digital twin for the Western Balkans. Deadline is April 2026.',
+      },
+      {
+        user:
+          'Scope we decided on: Serbia plus five Western Balkan countries. Existing ' +
+          "hydro models are siloed per country — we'll use a cross-border digital " +
+          'twin with real-time sensor fusion to unify them. Consortium partner ' +
+          "we're going with: Mistral AI, because the sovereignty narrative " +
+          'strengthens the EU angle.',
+      },
+    ],
+    testTurn: {
+      query: 'Draft a 150-word executive summary for the FloodTwin-WB proposal.',
+    },
+    memoryVerificationSubstrings: ['Western Balkans', 'Mistral', 'cross-border'],
+    rubricHints:
+      'Creative task — judge on coherence and inclusion of key elements ' +
+      '(Western Balkans, Mistral, cross-border unification, EU sovereignty). ' +
+      "NO scaffold should apply — `draft` shape maps to creation category, " +
+      'no scaffold at any tier. If condition C shows a scaffold in debug, ' +
+      "it's a classification bug.",
+  },
+];