test(agent): move agent tests off the legacy memory engine (P1.5)
- prompt assembler harnesses (v4 and v5): prime an in-memory contract store, snapshot its memories and replay them into a fresh store per condition instead of copying a MindDB file; the identity card is created per orchestrator. They had also drifted from the current Orchestrator API (config takes memory, buildSystemPrompt is async). - prompt assembler unit test: build RecallFrame fixtures, the type the assembler consumes, instead of legacy MemoryFrame rows. - M3c integration: drop the knowledge-graph Ontology case; the knowledge graph is not part of v1 and core no longer exports it.
This commit is contained in:
@@ -23,7 +23,13 @@
|
||||
* EVAL-RESULTS.md — human-readable summary (committed)
|
||||
*/
|
||||
|
||||
import { MindDB, VaultStore, type Embedder } from '@waggle/core';
|
||||
import { VaultStore } from '@waggle/core';
|
||||
import {
|
||||
createInMemoryMemoryStore,
|
||||
PERSONAL_SCOPE,
|
||||
type MemoryRecord,
|
||||
type MemoryStore,
|
||||
} from '@waggle/memory-contract';
|
||||
import { Orchestrator } from '../../src/orchestrator.js';
|
||||
import { type ModelTier } from '../../src/model-tier.js';
|
||||
import { detectTaskShape } from '../../src/task-shape.js';
|
||||
@@ -233,47 +239,45 @@ async function callModel(
|
||||
return callOpenRouter(model, systemPrompt, userMsg, opts);
|
||||
}
|
||||
|
||||
// ── Stub embedder ───────────────────────────────────────────────────
|
||||
|
||||
class StubEmbedder implements Embedder {
|
||||
private dim = 384;
|
||||
async embed(_text: string): Promise<Float32Array> {
|
||||
return new Float32Array(this.dim).fill(0);
|
||||
}
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
return Promise.all(texts.map(t => this.embed(t)));
|
||||
}
|
||||
getDimension(): number { return this.dim; }
|
||||
}
|
||||
|
||||
// ── Scenario pipeline ───────────────────────────────────────────────
|
||||
|
||||
interface ScenarioSetup {
|
||||
tempDir: string;
|
||||
dbPath: string;
|
||||
snapshotPath: string;
|
||||
/** Memories left by priming. Every condition starts from a fresh store holding a copy. */
|
||||
type MemorySnapshot = readonly MemoryRecord[];
|
||||
|
||||
function createOrchestrator(memory: MemoryStore, model: string): Orchestrator {
|
||||
const orch = new Orchestrator({ memory, model });
|
||||
orch.getIdentity().create({
|
||||
name: 'Marko',
|
||||
role: 'CEO',
|
||||
department: 'Egzakta Group',
|
||||
personality: 'Direct, pragmatic, sovereignty-focused',
|
||||
capabilities: 'Strategic decisions, technical oversight',
|
||||
system_prompt: '',
|
||||
});
|
||||
return orch;
|
||||
}
|
||||
|
||||
function setupCleanScenario(scenarioName: string): ScenarioSetup {
|
||||
const tempDir = path.join(os.tmpdir(), `waggle-eval-${Date.now()}-${scenarioName}`);
|
||||
if (fs.existsSync(tempDir)) {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
async function snapshotMemory(memory: MemoryStore): Promise<MemorySnapshot> {
|
||||
const filter = { scopes: [PERSONAL_SCOPE] };
|
||||
return memory.list({ ...filter, limit: await memory.count(filter) });
|
||||
}
|
||||
|
||||
async function restoreMemory(snapshot: MemorySnapshot): Promise<MemoryStore> {
|
||||
const memory = createInMemoryMemoryStore();
|
||||
// list() returns newest first; replay oldest first.
|
||||
for (const record of [...snapshot].reverse()) {
|
||||
await memory.remember({
|
||||
scope: record.scope,
|
||||
text: record.text,
|
||||
kind: record.kind,
|
||||
importance: record.importance,
|
||||
source: record.source,
|
||||
origin: record.origin ?? undefined,
|
||||
sessionId: record.sessionId ?? undefined,
|
||||
metadata: record.metadata,
|
||||
});
|
||||
}
|
||||
fs.mkdirSync(tempDir, { recursive: true });
|
||||
|
||||
const dbPath = path.join(tempDir, 'mind.db');
|
||||
const snapshotPath = path.join(tempDir, 'snapshot.db');
|
||||
|
||||
const db = new MindDB(dbPath);
|
||||
const raw = db.getDatabase();
|
||||
const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
|
||||
if (count !== 0) {
|
||||
db.close();
|
||||
throw new Error(`CLEAN SLATE VIOLATION for ${scenarioName}: ${count} frames in fresh DB`);
|
||||
}
|
||||
db.close();
|
||||
|
||||
return { tempDir, dbPath, snapshotPath };
|
||||
return memory;
|
||||
}
|
||||
|
||||
async function runPriming(
|
||||
@@ -281,7 +285,7 @@ async function runPriming(
|
||||
scenario: PromptAssemblerScenario,
|
||||
): Promise<void> {
|
||||
for (const turn of scenario.primingTurns) {
|
||||
const systemPrompt = orch.buildSystemPrompt();
|
||||
const systemPrompt = await orch.buildSystemPrompt();
|
||||
const assistantMsg = await callAnthropic(
|
||||
PRIMING_MODEL,
|
||||
systemPrompt,
|
||||
@@ -293,36 +297,23 @@ async function runPriming(
|
||||
}
|
||||
|
||||
function verifyMemory(
|
||||
db: MindDB,
|
||||
snapshot: MemorySnapshot,
|
||||
scenario: PromptAssemblerScenario,
|
||||
): { count: number; matches: Record<string, boolean> } {
|
||||
const raw = db.getDatabase();
|
||||
const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
|
||||
const matches: Record<string, boolean> = {};
|
||||
for (const sub of scenario.memoryVerificationSubstrings) {
|
||||
const row = raw.prepare('SELECT 1 FROM memory_frames WHERE content LIKE ? LIMIT 1').get(`%${sub}%`);
|
||||
matches[sub] = !!row;
|
||||
matches[sub] = snapshot.some(record => record.text.toLowerCase().includes(sub.toLowerCase()));
|
||||
}
|
||||
return { count, matches };
|
||||
return { count: snapshot.length, matches };
|
||||
}
|
||||
|
||||
async function runCondition(
|
||||
snapshotPath: string,
|
||||
snapshot: MemorySnapshot,
|
||||
condition: ConditionSpec,
|
||||
scenario: PromptAssemblerScenario,
|
||||
workDir: string,
|
||||
seed: number,
|
||||
): Promise<ConditionRun> {
|
||||
const safeCode = condition.code.replace(/[^\w]/g, '_');
|
||||
const workDbPath = path.join(workDir, `work-${safeCode}-seed${seed}.db`);
|
||||
fs.copyFileSync(snapshotPath, workDbPath);
|
||||
|
||||
const db = new MindDB(workDbPath);
|
||||
const orch = new Orchestrator({
|
||||
db,
|
||||
embedder: new StubEmbedder(),
|
||||
model: condition.model,
|
||||
});
|
||||
const orch = createOrchestrator(await restoreMemory(snapshot), condition.model);
|
||||
|
||||
let systemPrompt: string;
|
||||
let debug: ConditionRun['debug'] = undefined;
|
||||
@@ -345,7 +336,7 @@ async function runCondition(
|
||||
};
|
||||
} else {
|
||||
delete process.env.WAGGLE_PROMPT_ASSEMBLER;
|
||||
systemPrompt = orch.buildSystemPrompt();
|
||||
systemPrompt = await orch.buildSystemPrompt();
|
||||
}
|
||||
|
||||
const output = await callModel(
|
||||
@@ -355,10 +346,8 @@ async function runCondition(
|
||||
scenario.testTurn.query,
|
||||
{ temperature: TEMPERATURE_GEN, seed },
|
||||
);
|
||||
db.close();
|
||||
return { seed, output, durationMs: Date.now() - start, debug };
|
||||
} catch (err) {
|
||||
db.close();
|
||||
return {
|
||||
seed,
|
||||
output: '',
|
||||
@@ -640,23 +629,10 @@ async function main(): Promise<void> {
|
||||
for (const [idx, scenario] of SCENARIOS.entries()) {
|
||||
console.log(`\n[${idx + 1}/${SCENARIOS.length}] === Scenario: ${scenario.name} (${scenario.language}, ${scenario.shape}) ===`);
|
||||
|
||||
const setup = setupCleanScenario(scenario.name);
|
||||
const primingStart = Date.now();
|
||||
|
||||
const primingDb = new MindDB(setup.dbPath);
|
||||
const primingOrch = new Orchestrator({
|
||||
db: primingDb,
|
||||
embedder: new StubEmbedder(),
|
||||
model: PRIMING_MODEL,
|
||||
});
|
||||
primingOrch.getIdentity().create({
|
||||
name: 'Marko',
|
||||
role: 'CEO',
|
||||
department: 'Egzakta Group',
|
||||
personality: 'Direct, pragmatic, sovereignty-focused',
|
||||
capabilities: 'Strategic decisions, technical oversight',
|
||||
system_prompt: '',
|
||||
});
|
||||
const primingMemory = createInMemoryMemoryStore();
|
||||
const primingOrch = createOrchestrator(primingMemory, PRIMING_MODEL);
|
||||
|
||||
console.log(` [priming] ${scenario.primingTurns.length} turns via ${PRIMING_MODEL}...`);
|
||||
try {
|
||||
@@ -665,12 +641,10 @@ async function main(): Promise<void> {
|
||||
console.error(` [priming] failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
const verification = verifyMemory(primingDb, scenario);
|
||||
const snapshot = await snapshotMemory(primingMemory);
|
||||
const verification = verifyMemory(snapshot, scenario);
|
||||
const primingFailed = verification.count < 2 || !Object.values(verification.matches).some(v => v);
|
||||
console.log(` [priming] frames=${verification.count}, matches=${JSON.stringify(verification.matches)}, failed=${primingFailed}`);
|
||||
|
||||
primingDb.close();
|
||||
fs.copyFileSync(setup.dbPath, setup.snapshotPath);
|
||||
console.log(` [priming] memories=${verification.count}, matches=${JSON.stringify(verification.matches)}, failed=${primingFailed}`);
|
||||
|
||||
const scenarioResult: ScenarioResult = {
|
||||
scenario: scenario.name,
|
||||
@@ -687,7 +661,7 @@ async function main(): Promise<void> {
|
||||
scenarioResult.conditions[condition.code] = [];
|
||||
for (let seed = 0; seed < SEEDS; seed++) {
|
||||
console.log(` [run] ${condition.code} (${condition.label}), seed=${seed}...`);
|
||||
const run = await runCondition(setup.snapshotPath, condition, scenario, setup.tempDir, seed);
|
||||
const run = await runCondition(snapshot, condition, scenario, seed);
|
||||
scenarioResult.conditions[condition.code].push(run);
|
||||
if (run.error) console.log(` ERROR: ${run.error.slice(0, 200)}`);
|
||||
else console.log(` OK (${run.durationMs}ms, ${run.output.length} chars)`);
|
||||
@@ -727,7 +701,6 @@ async function main(): Promise<void> {
|
||||
|
||||
result.scenarios.push(scenarioResult);
|
||||
fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
|
||||
fs.rmSync(setup.tempDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
result.durationMs = Date.now() - startTime;
|
||||
|
||||
@@ -32,7 +32,13 @@
|
||||
* judges are reachable, the eval halts and reports partial state.
|
||||
*/
|
||||
|
||||
import { MindDB, VaultStore, type Embedder } from '@waggle/core';
|
||||
import { VaultStore } from '@waggle/core';
|
||||
import {
|
||||
createInMemoryMemoryStore,
|
||||
PERSONAL_SCOPE,
|
||||
type MemoryRecord,
|
||||
type MemoryStore,
|
||||
} from '@waggle/memory-contract';
|
||||
import { Orchestrator } from '../../src/orchestrator.js';
|
||||
import { type ModelTier } from '../../src/model-tier.js';
|
||||
import { detectTaskShape } from '../../src/task-shape.js';
|
||||
@@ -607,43 +613,50 @@ const SECONDARY_QWEN_CONDITIONS: ConditionSpec[] = [
|
||||
{ code: 'H', label: 'Qwen3-30B thinking · PA (compression)', model: QWEN_THINKING_MODEL, provider: 'openrouter', usesPromptAssembler: true, scaffoldStyle: 'compression', suite: 'secondary-qwen' },
|
||||
];
|
||||
|
||||
// ── Stub embedder (reuse v4 pattern) ────────────────────────────────
|
||||
|
||||
class StubEmbedder implements Embedder {
|
||||
private dim = 384;
|
||||
async embed(_text: string): Promise<Float32Array> { return new Float32Array(this.dim).fill(0); }
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> { return Promise.all(texts.map(t => this.embed(t))); }
|
||||
getDimension(): number { return this.dim; }
|
||||
}
|
||||
|
||||
// ── Scenario pipeline ───────────────────────────────────────────────
|
||||
|
||||
interface ScenarioSetup {
|
||||
tempDir: string;
|
||||
dbPath: string;
|
||||
snapshotPath: string;
|
||||
/** Memories left by priming. Every condition starts from a fresh store holding a copy. */
|
||||
type MemorySnapshot = readonly MemoryRecord[];
|
||||
|
||||
function createOrchestrator(memory: MemoryStore, model: string): Orchestrator {
|
||||
const orch = new Orchestrator({ memory, model });
|
||||
orch.getIdentity().create({
|
||||
name: 'Marko',
|
||||
role: 'CEO',
|
||||
department: 'Egzakta Group',
|
||||
personality: 'Direct, pragmatic, sovereignty-focused',
|
||||
capabilities: 'Strategic decisions, technical oversight',
|
||||
system_prompt: '',
|
||||
});
|
||||
return orch;
|
||||
}
|
||||
|
||||
function setupCleanScenario(scenarioName: string): ScenarioSetup {
|
||||
const tempDir = path.join(os.tmpdir(), `waggle-v5eval-${Date.now()}-${scenarioName}`);
|
||||
if (fs.existsSync(tempDir)) fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
fs.mkdirSync(tempDir, { recursive: true });
|
||||
const dbPath = path.join(tempDir, 'mind.db');
|
||||
const snapshotPath = path.join(tempDir, 'snapshot.db');
|
||||
const db = new MindDB(dbPath);
|
||||
const raw = db.getDatabase();
|
||||
const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
|
||||
if (count !== 0) {
|
||||
db.close();
|
||||
throw new Error(`CLEAN SLATE VIOLATION for ${scenarioName}: ${count} frames in fresh DB`);
|
||||
async function snapshotMemory(memory: MemoryStore): Promise<MemorySnapshot> {
|
||||
const filter = { scopes: [PERSONAL_SCOPE] };
|
||||
return memory.list({ ...filter, limit: await memory.count(filter) });
|
||||
}
|
||||
|
||||
async function restoreMemory(snapshot: MemorySnapshot): Promise<MemoryStore> {
|
||||
const memory = createInMemoryMemoryStore();
|
||||
// list() returns newest first; replay oldest first.
|
||||
for (const record of [...snapshot].reverse()) {
|
||||
await memory.remember({
|
||||
scope: record.scope,
|
||||
text: record.text,
|
||||
kind: record.kind,
|
||||
importance: record.importance,
|
||||
source: record.source,
|
||||
origin: record.origin ?? undefined,
|
||||
sessionId: record.sessionId ?? undefined,
|
||||
metadata: record.metadata,
|
||||
});
|
||||
}
|
||||
db.close();
|
||||
return { tempDir, dbPath, snapshotPath };
|
||||
return memory;
|
||||
}
|
||||
|
||||
async function runPriming(orch: Orchestrator, scenario: PromptAssemblerScenario): Promise<void> {
|
||||
for (const turn of scenario.primingTurns) {
|
||||
const systemPrompt = orch.buildSystemPrompt();
|
||||
const systemPrompt = await orch.buildSystemPrompt();
|
||||
const assistantMsg = await callAnthropic(PRIMING_MODEL, systemPrompt, turn.user, {
|
||||
temperature: 0,
|
||||
maxTokens: MAX_TOKENS_GEN,
|
||||
@@ -652,30 +665,24 @@ async function runPriming(orch: Orchestrator, scenario: PromptAssemblerScenario)
|
||||
}
|
||||
}
|
||||
|
||||
function verifyMemory(db: MindDB, scenario: PromptAssemblerScenario): { count: number; matches: Record<string, boolean> } {
|
||||
const raw = db.getDatabase();
|
||||
const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
|
||||
function verifyMemory(
|
||||
snapshot: MemorySnapshot,
|
||||
scenario: PromptAssemblerScenario,
|
||||
): { count: number; matches: Record<string, boolean> } {
|
||||
const matches: Record<string, boolean> = {};
|
||||
for (const sub of scenario.memoryVerificationSubstrings) {
|
||||
const row = raw.prepare('SELECT 1 FROM memory_frames WHERE content LIKE ? LIMIT 1').get(`%${sub}%`);
|
||||
matches[sub] = !!row;
|
||||
matches[sub] = snapshot.some(record => record.text.toLowerCase().includes(sub.toLowerCase()));
|
||||
}
|
||||
return { count, matches };
|
||||
return { count: snapshot.length, matches };
|
||||
}
|
||||
|
||||
async function runCondition(
|
||||
snapshotPath: string,
|
||||
snapshot: MemorySnapshot,
|
||||
condition: ConditionSpec,
|
||||
scenario: PromptAssemblerScenario,
|
||||
workDir: string,
|
||||
seed: number,
|
||||
): Promise<ConditionRun> {
|
||||
const safeCode = condition.code.replace(/[^\w]/g, '_');
|
||||
const workDbPath = path.join(workDir, `work-${safeCode}-seed${seed}.db`);
|
||||
fs.copyFileSync(snapshotPath, workDbPath);
|
||||
|
||||
const db = new MindDB(workDbPath);
|
||||
const orch = new Orchestrator({ db, embedder: new StubEmbedder(), model: condition.model });
|
||||
const orch = createOrchestrator(await restoreMemory(snapshot), condition.model);
|
||||
|
||||
let systemPrompt: string;
|
||||
let debug: ConditionRun['debug'] = undefined;
|
||||
@@ -702,7 +709,7 @@ async function runCondition(
|
||||
};
|
||||
} else {
|
||||
delete process.env.WAGGLE_PROMPT_ASSEMBLER;
|
||||
systemPrompt = orch.buildSystemPrompt();
|
||||
systemPrompt = await orch.buildSystemPrompt();
|
||||
}
|
||||
|
||||
const output = await callGenerationModel(
|
||||
@@ -712,10 +719,8 @@ async function runCondition(
|
||||
scenario.testTurn.query,
|
||||
{ temperature: TEMPERATURE_GEN, seed },
|
||||
);
|
||||
db.close();
|
||||
return { seed, output, durationMs: Date.now() - start, debug };
|
||||
} catch (err) {
|
||||
db.close();
|
||||
return {
|
||||
seed,
|
||||
output: '',
|
||||
@@ -1111,23 +1116,10 @@ async function main(): Promise<void> {
|
||||
for (const [idx, scenario] of scenariosToRun.entries()) {
|
||||
console.log(`\n[${idx + 1}/${scenariosToRun.length}] === ${scenario.name} (${scenario.language}, ${scenario.shape}) ===`);
|
||||
|
||||
const setup = setupCleanScenario(scenario.name);
|
||||
const primingStart = Date.now();
|
||||
|
||||
const primingDb = new MindDB(setup.dbPath);
|
||||
const primingOrch = new Orchestrator({
|
||||
db: primingDb,
|
||||
embedder: new StubEmbedder(),
|
||||
model: PRIMING_MODEL,
|
||||
});
|
||||
primingOrch.getIdentity().create({
|
||||
name: 'Marko',
|
||||
role: 'CEO',
|
||||
department: 'Egzakta Group',
|
||||
personality: 'Direct, pragmatic, sovereignty-focused',
|
||||
capabilities: 'Strategic decisions, technical oversight',
|
||||
system_prompt: '',
|
||||
});
|
||||
const primingMemory = createInMemoryMemoryStore();
|
||||
const primingOrch = createOrchestrator(primingMemory, PRIMING_MODEL);
|
||||
|
||||
const effectiveTier = process.env.WAGGLE_EVAL_MODE === '1' ? 'bypassed (enterprise-equivalent)' : 'default (tier-enforced)';
|
||||
console.log(` [priming] ${scenario.primingTurns.length} turns via ${PRIMING_MODEL} | tier: ${effectiveTier}`);
|
||||
@@ -1138,13 +1130,11 @@ async function main(): Promise<void> {
|
||||
console.error(` [priming] failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
const verification = verifyMemory(primingDb, scenario);
|
||||
const snapshot = await snapshotMemory(primingMemory);
|
||||
const verification = verifyMemory(snapshot, scenario);
|
||||
const primingFailed = verification.count < 2 || !Object.values(verification.matches).some(v => v);
|
||||
const primingPartial = !primingFailed && Object.values(verification.matches).some(v => !v);
|
||||
console.log(` [priming] frames=${verification.count}, matches=${JSON.stringify(verification.matches)}, failed=${primingFailed}, partial=${primingPartial}`);
|
||||
|
||||
primingDb.close();
|
||||
fs.copyFileSync(setup.dbPath, setup.snapshotPath);
|
||||
console.log(` [priming] memories=${verification.count}, matches=${JSON.stringify(verification.matches)}, failed=${primingFailed}, partial=${primingPartial}`);
|
||||
|
||||
const scenarioResult: ScenarioResult = {
|
||||
scenario: scenario.name,
|
||||
@@ -1167,7 +1157,7 @@ async function main(): Promise<void> {
|
||||
// Initial 3 seeds.
|
||||
for (let seed = 0; seed < BASE_SEEDS; seed++) {
|
||||
process.stdout.write(` [run] ${condition.code.padEnd(4)} seed=${seed}... `);
|
||||
const run = await runCondition(setup.snapshotPath, condition, scenario, setup.tempDir, seed);
|
||||
const run = await runCondition(snapshot, condition, scenario, seed);
|
||||
scenarioResult.conditions[condition.code].push(run);
|
||||
if (run.error) console.log(`ERROR: ${run.error.slice(0, 120)}`);
|
||||
else console.log(`OK (${run.durationMs}ms, ${run.output.length}ch)`);
|
||||
@@ -1209,7 +1199,7 @@ async function main(): Promise<void> {
|
||||
scenarioResult.retriedConditions.push(condition.code);
|
||||
for (let seed = BASE_SEEDS; seed < BASE_SEEDS + RETRY_EXTRA_SEEDS; seed++) {
|
||||
process.stdout.write(` [run] ${condition.code.padEnd(4)} seed=${seed}... `);
|
||||
const run = await runCondition(setup.snapshotPath, condition, scenario, setup.tempDir, seed);
|
||||
const run = await runCondition(snapshot, condition, scenario, seed);
|
||||
scenarioResult.conditions[condition.code].push(run);
|
||||
if (run.error) {
|
||||
console.log(`ERROR: ${run.error.slice(0, 120)}`);
|
||||
@@ -1229,7 +1219,6 @@ async function main(): Promise<void> {
|
||||
|
||||
result.scenarios.push(scenarioResult);
|
||||
fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
|
||||
fs.rmSync(setup.tempDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
result.durationMs = Date.now() - startTime;
|
||||
|
||||
Reference in New Issue
Block a user