test(agent): move agent tests off the legacy memory engine (P1.5)

- prompt assembler harnesses (v4 and v5): prime an in-memory contract store,
  snapshot its memories and replay them into a fresh store per condition
  instead of copying a MindDB file; the identity card is created per
  orchestrator. They had also drifted from the current Orchestrator API
  (config takes memory, buildSystemPrompt is async).
- prompt assembler unit test: build RecallFrame fixtures, the type the
  assembler consumes, instead of legacy MemoryFrame rows.
- M3c integration: drop the knowledge-graph Ontology case; the knowledge
  graph is not part of v1 and core no longer exports it.
This commit is contained in:
shumeiko.it
2026-10-24 17:07:18 +02:00
parent ad7b90bebd
commit f94331c202
5 changed files with 134 additions and 189 deletions

View File

@@ -23,7 +23,13 @@
* EVAL-RESULTS.md — human-readable summary (committed)
*/
import { MindDB, VaultStore, type Embedder } from '@waggle/core';
import { VaultStore } from '@waggle/core';
import {
createInMemoryMemoryStore,
PERSONAL_SCOPE,
type MemoryRecord,
type MemoryStore,
} from '@waggle/memory-contract';
import { Orchestrator } from '../../src/orchestrator.js';
import { type ModelTier } from '../../src/model-tier.js';
import { detectTaskShape } from '../../src/task-shape.js';
@@ -233,47 +239,45 @@ async function callModel(
return callOpenRouter(model, systemPrompt, userMsg, opts);
}
// ── Stub embedder ───────────────────────────────────────────────────
class StubEmbedder implements Embedder {
private dim = 384;
async embed(_text: string): Promise<Float32Array> {
return new Float32Array(this.dim).fill(0);
}
async embedBatch(texts: string[]): Promise<Float32Array[]> {
return Promise.all(texts.map(t => this.embed(t)));
}
getDimension(): number { return this.dim; }
}
// ── Scenario pipeline ───────────────────────────────────────────────
interface ScenarioSetup {
tempDir: string;
dbPath: string;
snapshotPath: string;
/** Memories left by priming. Every condition starts from a fresh store holding a copy. */
type MemorySnapshot = readonly MemoryRecord[];
function createOrchestrator(memory: MemoryStore, model: string): Orchestrator {
const orch = new Orchestrator({ memory, model });
orch.getIdentity().create({
name: 'Marko',
role: 'CEO',
department: 'Egzakta Group',
personality: 'Direct, pragmatic, sovereignty-focused',
capabilities: 'Strategic decisions, technical oversight',
system_prompt: '',
});
return orch;
}
function setupCleanScenario(scenarioName: string): ScenarioSetup {
const tempDir = path.join(os.tmpdir(), `waggle-eval-${Date.now()}-${scenarioName}`);
if (fs.existsSync(tempDir)) {
fs.rmSync(tempDir, { recursive: true, force: true });
async function snapshotMemory(memory: MemoryStore): Promise<MemorySnapshot> {
const filter = { scopes: [PERSONAL_SCOPE] };
return memory.list({ ...filter, limit: await memory.count(filter) });
}
async function restoreMemory(snapshot: MemorySnapshot): Promise<MemoryStore> {
const memory = createInMemoryMemoryStore();
// list() returns newest first; replay oldest first.
for (const record of [...snapshot].reverse()) {
await memory.remember({
scope: record.scope,
text: record.text,
kind: record.kind,
importance: record.importance,
source: record.source,
origin: record.origin ?? undefined,
sessionId: record.sessionId ?? undefined,
metadata: record.metadata,
});
}
fs.mkdirSync(tempDir, { recursive: true });
const dbPath = path.join(tempDir, 'mind.db');
const snapshotPath = path.join(tempDir, 'snapshot.db');
const db = new MindDB(dbPath);
const raw = db.getDatabase();
const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
if (count !== 0) {
db.close();
throw new Error(`CLEAN SLATE VIOLATION for ${scenarioName}: ${count} frames in fresh DB`);
}
db.close();
return { tempDir, dbPath, snapshotPath };
return memory;
}
async function runPriming(
@@ -281,7 +285,7 @@ async function runPriming(
scenario: PromptAssemblerScenario,
): Promise<void> {
for (const turn of scenario.primingTurns) {
const systemPrompt = orch.buildSystemPrompt();
const systemPrompt = await orch.buildSystemPrompt();
const assistantMsg = await callAnthropic(
PRIMING_MODEL,
systemPrompt,
@@ -293,36 +297,23 @@ async function runPriming(
}
function verifyMemory(
db: MindDB,
snapshot: MemorySnapshot,
scenario: PromptAssemblerScenario,
): { count: number; matches: Record<string, boolean> } {
const raw = db.getDatabase();
const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
const matches: Record<string, boolean> = {};
for (const sub of scenario.memoryVerificationSubstrings) {
const row = raw.prepare('SELECT 1 FROM memory_frames WHERE content LIKE ? LIMIT 1').get(`%${sub}%`);
matches[sub] = !!row;
matches[sub] = snapshot.some(record => record.text.toLowerCase().includes(sub.toLowerCase()));
}
return { count, matches };
return { count: snapshot.length, matches };
}
async function runCondition(
snapshotPath: string,
snapshot: MemorySnapshot,
condition: ConditionSpec,
scenario: PromptAssemblerScenario,
workDir: string,
seed: number,
): Promise<ConditionRun> {
const safeCode = condition.code.replace(/[^\w]/g, '_');
const workDbPath = path.join(workDir, `work-${safeCode}-seed${seed}.db`);
fs.copyFileSync(snapshotPath, workDbPath);
const db = new MindDB(workDbPath);
const orch = new Orchestrator({
db,
embedder: new StubEmbedder(),
model: condition.model,
});
const orch = createOrchestrator(await restoreMemory(snapshot), condition.model);
let systemPrompt: string;
let debug: ConditionRun['debug'] = undefined;
@@ -345,7 +336,7 @@ async function runCondition(
};
} else {
delete process.env.WAGGLE_PROMPT_ASSEMBLER;
systemPrompt = orch.buildSystemPrompt();
systemPrompt = await orch.buildSystemPrompt();
}
const output = await callModel(
@@ -355,10 +346,8 @@ async function runCondition(
scenario.testTurn.query,
{ temperature: TEMPERATURE_GEN, seed },
);
db.close();
return { seed, output, durationMs: Date.now() - start, debug };
} catch (err) {
db.close();
return {
seed,
output: '',
@@ -640,23 +629,10 @@ async function main(): Promise<void> {
for (const [idx, scenario] of SCENARIOS.entries()) {
console.log(`\n[${idx + 1}/${SCENARIOS.length}] === Scenario: ${scenario.name} (${scenario.language}, ${scenario.shape}) ===`);
const setup = setupCleanScenario(scenario.name);
const primingStart = Date.now();
const primingDb = new MindDB(setup.dbPath);
const primingOrch = new Orchestrator({
db: primingDb,
embedder: new StubEmbedder(),
model: PRIMING_MODEL,
});
primingOrch.getIdentity().create({
name: 'Marko',
role: 'CEO',
department: 'Egzakta Group',
personality: 'Direct, pragmatic, sovereignty-focused',
capabilities: 'Strategic decisions, technical oversight',
system_prompt: '',
});
const primingMemory = createInMemoryMemoryStore();
const primingOrch = createOrchestrator(primingMemory, PRIMING_MODEL);
console.log(` [priming] ${scenario.primingTurns.length} turns via ${PRIMING_MODEL}...`);
try {
@@ -665,12 +641,10 @@ async function main(): Promise<void> {
console.error(` [priming] failed: ${err instanceof Error ? err.message : String(err)}`);
}
const verification = verifyMemory(primingDb, scenario);
const snapshot = await snapshotMemory(primingMemory);
const verification = verifyMemory(snapshot, scenario);
const primingFailed = verification.count < 2 || !Object.values(verification.matches).some(v => v);
console.log(` [priming] frames=${verification.count}, matches=${JSON.stringify(verification.matches)}, failed=${primingFailed}`);
primingDb.close();
fs.copyFileSync(setup.dbPath, setup.snapshotPath);
console.log(` [priming] memories=${verification.count}, matches=${JSON.stringify(verification.matches)}, failed=${primingFailed}`);
const scenarioResult: ScenarioResult = {
scenario: scenario.name,
@@ -687,7 +661,7 @@ async function main(): Promise<void> {
scenarioResult.conditions[condition.code] = [];
for (let seed = 0; seed < SEEDS; seed++) {
console.log(` [run] ${condition.code} (${condition.label}), seed=${seed}...`);
const run = await runCondition(setup.snapshotPath, condition, scenario, setup.tempDir, seed);
const run = await runCondition(snapshot, condition, scenario, seed);
scenarioResult.conditions[condition.code].push(run);
if (run.error) console.log(` ERROR: ${run.error.slice(0, 200)}`);
else console.log(` OK (${run.durationMs}ms, ${run.output.length} chars)`);
@@ -727,7 +701,6 @@ async function main(): Promise<void> {
result.scenarios.push(scenarioResult);
fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
fs.rmSync(setup.tempDir, { recursive: true, force: true });
}
result.durationMs = Date.now() - startTime;

View File

@@ -32,7 +32,13 @@
* judges are reachable, the eval halts and reports partial state.
*/
import { MindDB, VaultStore, type Embedder } from '@waggle/core';
import { VaultStore } from '@waggle/core';
import {
createInMemoryMemoryStore,
PERSONAL_SCOPE,
type MemoryRecord,
type MemoryStore,
} from '@waggle/memory-contract';
import { Orchestrator } from '../../src/orchestrator.js';
import { type ModelTier } from '../../src/model-tier.js';
import { detectTaskShape } from '../../src/task-shape.js';
@@ -607,43 +613,50 @@ const SECONDARY_QWEN_CONDITIONS: ConditionSpec[] = [
{ code: 'H', label: 'Qwen3-30B thinking · PA (compression)', model: QWEN_THINKING_MODEL, provider: 'openrouter', usesPromptAssembler: true, scaffoldStyle: 'compression', suite: 'secondary-qwen' },
];
// ── Stub embedder (reuse v4 pattern) ────────────────────────────────
class StubEmbedder implements Embedder {
private dim = 384;
async embed(_text: string): Promise<Float32Array> { return new Float32Array(this.dim).fill(0); }
async embedBatch(texts: string[]): Promise<Float32Array[]> { return Promise.all(texts.map(t => this.embed(t))); }
getDimension(): number { return this.dim; }
}
// ── Scenario pipeline ───────────────────────────────────────────────
interface ScenarioSetup {
tempDir: string;
dbPath: string;
snapshotPath: string;
/** Memories left by priming. Every condition starts from a fresh store holding a copy. */
type MemorySnapshot = readonly MemoryRecord[];
function createOrchestrator(memory: MemoryStore, model: string): Orchestrator {
const orch = new Orchestrator({ memory, model });
orch.getIdentity().create({
name: 'Marko',
role: 'CEO',
department: 'Egzakta Group',
personality: 'Direct, pragmatic, sovereignty-focused',
capabilities: 'Strategic decisions, technical oversight',
system_prompt: '',
});
return orch;
}
function setupCleanScenario(scenarioName: string): ScenarioSetup {
const tempDir = path.join(os.tmpdir(), `waggle-v5eval-${Date.now()}-${scenarioName}`);
if (fs.existsSync(tempDir)) fs.rmSync(tempDir, { recursive: true, force: true });
fs.mkdirSync(tempDir, { recursive: true });
const dbPath = path.join(tempDir, 'mind.db');
const snapshotPath = path.join(tempDir, 'snapshot.db');
const db = new MindDB(dbPath);
const raw = db.getDatabase();
const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
if (count !== 0) {
db.close();
throw new Error(`CLEAN SLATE VIOLATION for ${scenarioName}: ${count} frames in fresh DB`);
async function snapshotMemory(memory: MemoryStore): Promise<MemorySnapshot> {
const filter = { scopes: [PERSONAL_SCOPE] };
return memory.list({ ...filter, limit: await memory.count(filter) });
}
async function restoreMemory(snapshot: MemorySnapshot): Promise<MemoryStore> {
const memory = createInMemoryMemoryStore();
// list() returns newest first; replay oldest first.
for (const record of [...snapshot].reverse()) {
await memory.remember({
scope: record.scope,
text: record.text,
kind: record.kind,
importance: record.importance,
source: record.source,
origin: record.origin ?? undefined,
sessionId: record.sessionId ?? undefined,
metadata: record.metadata,
});
}
db.close();
return { tempDir, dbPath, snapshotPath };
return memory;
}
async function runPriming(orch: Orchestrator, scenario: PromptAssemblerScenario): Promise<void> {
for (const turn of scenario.primingTurns) {
const systemPrompt = orch.buildSystemPrompt();
const systemPrompt = await orch.buildSystemPrompt();
const assistantMsg = await callAnthropic(PRIMING_MODEL, systemPrompt, turn.user, {
temperature: 0,
maxTokens: MAX_TOKENS_GEN,
@@ -652,30 +665,24 @@ async function runPriming(orch: Orchestrator, scenario: PromptAssemblerScenario)
}
}
function verifyMemory(db: MindDB, scenario: PromptAssemblerScenario): { count: number; matches: Record<string, boolean> } {
const raw = db.getDatabase();
const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
function verifyMemory(
snapshot: MemorySnapshot,
scenario: PromptAssemblerScenario,
): { count: number; matches: Record<string, boolean> } {
const matches: Record<string, boolean> = {};
for (const sub of scenario.memoryVerificationSubstrings) {
const row = raw.prepare('SELECT 1 FROM memory_frames WHERE content LIKE ? LIMIT 1').get(`%${sub}%`);
matches[sub] = !!row;
matches[sub] = snapshot.some(record => record.text.toLowerCase().includes(sub.toLowerCase()));
}
return { count, matches };
return { count: snapshot.length, matches };
}
async function runCondition(
snapshotPath: string,
snapshot: MemorySnapshot,
condition: ConditionSpec,
scenario: PromptAssemblerScenario,
workDir: string,
seed: number,
): Promise<ConditionRun> {
const safeCode = condition.code.replace(/[^\w]/g, '_');
const workDbPath = path.join(workDir, `work-${safeCode}-seed${seed}.db`);
fs.copyFileSync(snapshotPath, workDbPath);
const db = new MindDB(workDbPath);
const orch = new Orchestrator({ db, embedder: new StubEmbedder(), model: condition.model });
const orch = createOrchestrator(await restoreMemory(snapshot), condition.model);
let systemPrompt: string;
let debug: ConditionRun['debug'] = undefined;
@@ -702,7 +709,7 @@ async function runCondition(
};
} else {
delete process.env.WAGGLE_PROMPT_ASSEMBLER;
systemPrompt = orch.buildSystemPrompt();
systemPrompt = await orch.buildSystemPrompt();
}
const output = await callGenerationModel(
@@ -712,10 +719,8 @@ async function runCondition(
scenario.testTurn.query,
{ temperature: TEMPERATURE_GEN, seed },
);
db.close();
return { seed, output, durationMs: Date.now() - start, debug };
} catch (err) {
db.close();
return {
seed,
output: '',
@@ -1111,23 +1116,10 @@ async function main(): Promise<void> {
for (const [idx, scenario] of scenariosToRun.entries()) {
console.log(`\n[${idx + 1}/${scenariosToRun.length}] === ${scenario.name} (${scenario.language}, ${scenario.shape}) ===`);
const setup = setupCleanScenario(scenario.name);
const primingStart = Date.now();
const primingDb = new MindDB(setup.dbPath);
const primingOrch = new Orchestrator({
db: primingDb,
embedder: new StubEmbedder(),
model: PRIMING_MODEL,
});
primingOrch.getIdentity().create({
name: 'Marko',
role: 'CEO',
department: 'Egzakta Group',
personality: 'Direct, pragmatic, sovereignty-focused',
capabilities: 'Strategic decisions, technical oversight',
system_prompt: '',
});
const primingMemory = createInMemoryMemoryStore();
const primingOrch = createOrchestrator(primingMemory, PRIMING_MODEL);
const effectiveTier = process.env.WAGGLE_EVAL_MODE === '1' ? 'bypassed (enterprise-equivalent)' : 'default (tier-enforced)';
console.log(` [priming] ${scenario.primingTurns.length} turns via ${PRIMING_MODEL} | tier: ${effectiveTier}`);
@@ -1138,13 +1130,11 @@ async function main(): Promise<void> {
console.error(` [priming] failed: ${err instanceof Error ? err.message : String(err)}`);
}
const verification = verifyMemory(primingDb, scenario);
const snapshot = await snapshotMemory(primingMemory);
const verification = verifyMemory(snapshot, scenario);
const primingFailed = verification.count < 2 || !Object.values(verification.matches).some(v => v);
const primingPartial = !primingFailed && Object.values(verification.matches).some(v => !v);
console.log(` [priming] frames=${verification.count}, matches=${JSON.stringify(verification.matches)}, failed=${primingFailed}, partial=${primingPartial}`);
primingDb.close();
fs.copyFileSync(setup.dbPath, setup.snapshotPath);
console.log(` [priming] memories=${verification.count}, matches=${JSON.stringify(verification.matches)}, failed=${primingFailed}, partial=${primingPartial}`);
const scenarioResult: ScenarioResult = {
scenario: scenario.name,
@@ -1167,7 +1157,7 @@ async function main(): Promise<void> {
// Initial 3 seeds.
for (let seed = 0; seed < BASE_SEEDS; seed++) {
process.stdout.write(` [run] ${condition.code.padEnd(4)} seed=${seed}... `);
const run = await runCondition(setup.snapshotPath, condition, scenario, setup.tempDir, seed);
const run = await runCondition(snapshot, condition, scenario, seed);
scenarioResult.conditions[condition.code].push(run);
if (run.error) console.log(`ERROR: ${run.error.slice(0, 120)}`);
else console.log(`OK (${run.durationMs}ms, ${run.output.length}ch)`);
@@ -1209,7 +1199,7 @@ async function main(): Promise<void> {
scenarioResult.retriedConditions.push(condition.code);
for (let seed = BASE_SEEDS; seed < BASE_SEEDS + RETRY_EXTRA_SEEDS; seed++) {
process.stdout.write(` [run] ${condition.code.padEnd(4)} seed=${seed}... `);
const run = await runCondition(setup.snapshotPath, condition, scenario, setup.tempDir, seed);
const run = await runCondition(snapshot, condition, scenario, seed);
scenarioResult.conditions[condition.code].push(run);
if (run.error) {
console.log(`ERROR: ${run.error.slice(0, 120)}`);
@@ -1229,7 +1219,6 @@ async function main(): Promise<void> {
result.scenarios.push(scenarioResult);
fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
fs.rmSync(setup.tempDir, { recursive: true, force: true });
}
result.durationMs = Date.now() - startTime;