test(agent): move agent tests off the legacy memory engine (P1.5)
- prompt assembler harnesses (v4 and v5): prime an in-memory contract store, snapshot its memories and replay them into a fresh store per condition instead of copying a MindDB file; the identity card is created per orchestrator. They had also drifted from the current Orchestrator API (config takes memory, buildSystemPrompt is async). - prompt assembler unit test: build RecallFrame fixtures, the type the assembler consumes, instead of legacy MemoryFrame rows. - M3c integration: drop the knowledge-graph Ontology case; the knowledge graph is not part of v1 and core no longer exports it.
This commit is contained in:
@@ -2,7 +2,7 @@
|
|||||||
* Combined Retrieval — merge engine tests.
|
* Combined Retrieval — merge engine tests.
|
||||||
*
|
*
|
||||||
* Tests the merge engine in isolation. MemorySearchLike (workspace + personal
|
* Tests the merge engine in isolation. MemorySearchLike (workspace + personal
|
||||||
* HybridSearch) is mocked. No server, no vault, no DB, no wiring.
|
* memory search) is mocked. No server, no vault, no DB, no wiring.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { describe, it, expect, vi } from 'vitest';
|
import { describe, it, expect, vi } from 'vitest';
|
||||||
|
|||||||
@@ -23,7 +23,13 @@
|
|||||||
* EVAL-RESULTS.md — human-readable summary (committed)
|
* EVAL-RESULTS.md — human-readable summary (committed)
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { MindDB, VaultStore, type Embedder } from '@waggle/core';
|
import { VaultStore } from '@waggle/core';
|
||||||
|
import {
|
||||||
|
createInMemoryMemoryStore,
|
||||||
|
PERSONAL_SCOPE,
|
||||||
|
type MemoryRecord,
|
||||||
|
type MemoryStore,
|
||||||
|
} from '@waggle/memory-contract';
|
||||||
import { Orchestrator } from '../../src/orchestrator.js';
|
import { Orchestrator } from '../../src/orchestrator.js';
|
||||||
import { type ModelTier } from '../../src/model-tier.js';
|
import { type ModelTier } from '../../src/model-tier.js';
|
||||||
import { detectTaskShape } from '../../src/task-shape.js';
|
import { detectTaskShape } from '../../src/task-shape.js';
|
||||||
@@ -233,47 +239,45 @@ async function callModel(
|
|||||||
return callOpenRouter(model, systemPrompt, userMsg, opts);
|
return callOpenRouter(model, systemPrompt, userMsg, opts);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Stub embedder ───────────────────────────────────────────────────
|
|
||||||
|
|
||||||
class StubEmbedder implements Embedder {
|
|
||||||
private dim = 384;
|
|
||||||
async embed(_text: string): Promise<Float32Array> {
|
|
||||||
return new Float32Array(this.dim).fill(0);
|
|
||||||
}
|
|
||||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
|
||||||
return Promise.all(texts.map(t => this.embed(t)));
|
|
||||||
}
|
|
||||||
getDimension(): number { return this.dim; }
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Scenario pipeline ───────────────────────────────────────────────
|
// ── Scenario pipeline ───────────────────────────────────────────────
|
||||||
|
|
||||||
interface ScenarioSetup {
|
/** Memories left by priming. Every condition starts from a fresh store holding a copy. */
|
||||||
tempDir: string;
|
type MemorySnapshot = readonly MemoryRecord[];
|
||||||
dbPath: string;
|
|
||||||
snapshotPath: string;
|
function createOrchestrator(memory: MemoryStore, model: string): Orchestrator {
|
||||||
|
const orch = new Orchestrator({ memory, model });
|
||||||
|
orch.getIdentity().create({
|
||||||
|
name: 'Marko',
|
||||||
|
role: 'CEO',
|
||||||
|
department: 'Egzakta Group',
|
||||||
|
personality: 'Direct, pragmatic, sovereignty-focused',
|
||||||
|
capabilities: 'Strategic decisions, technical oversight',
|
||||||
|
system_prompt: '',
|
||||||
|
});
|
||||||
|
return orch;
|
||||||
}
|
}
|
||||||
|
|
||||||
function setupCleanScenario(scenarioName: string): ScenarioSetup {
|
async function snapshotMemory(memory: MemoryStore): Promise<MemorySnapshot> {
|
||||||
const tempDir = path.join(os.tmpdir(), `waggle-eval-${Date.now()}-${scenarioName}`);
|
const filter = { scopes: [PERSONAL_SCOPE] };
|
||||||
if (fs.existsSync(tempDir)) {
|
return memory.list({ ...filter, limit: await memory.count(filter) });
|
||||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
||||||
}
|
}
|
||||||
fs.mkdirSync(tempDir, { recursive: true });
|
|
||||||
|
|
||||||
const dbPath = path.join(tempDir, 'mind.db');
|
async function restoreMemory(snapshot: MemorySnapshot): Promise<MemoryStore> {
|
||||||
const snapshotPath = path.join(tempDir, 'snapshot.db');
|
const memory = createInMemoryMemoryStore();
|
||||||
|
// list() returns newest first; replay oldest first.
|
||||||
const db = new MindDB(dbPath);
|
for (const record of [...snapshot].reverse()) {
|
||||||
const raw = db.getDatabase();
|
await memory.remember({
|
||||||
const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
|
scope: record.scope,
|
||||||
if (count !== 0) {
|
text: record.text,
|
||||||
db.close();
|
kind: record.kind,
|
||||||
throw new Error(`CLEAN SLATE VIOLATION for ${scenarioName}: ${count} frames in fresh DB`);
|
importance: record.importance,
|
||||||
|
source: record.source,
|
||||||
|
origin: record.origin ?? undefined,
|
||||||
|
sessionId: record.sessionId ?? undefined,
|
||||||
|
metadata: record.metadata,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
db.close();
|
return memory;
|
||||||
|
|
||||||
return { tempDir, dbPath, snapshotPath };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function runPriming(
|
async function runPriming(
|
||||||
@@ -281,7 +285,7 @@ async function runPriming(
|
|||||||
scenario: PromptAssemblerScenario,
|
scenario: PromptAssemblerScenario,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
for (const turn of scenario.primingTurns) {
|
for (const turn of scenario.primingTurns) {
|
||||||
const systemPrompt = orch.buildSystemPrompt();
|
const systemPrompt = await orch.buildSystemPrompt();
|
||||||
const assistantMsg = await callAnthropic(
|
const assistantMsg = await callAnthropic(
|
||||||
PRIMING_MODEL,
|
PRIMING_MODEL,
|
||||||
systemPrompt,
|
systemPrompt,
|
||||||
@@ -293,36 +297,23 @@ async function runPriming(
|
|||||||
}
|
}
|
||||||
|
|
||||||
function verifyMemory(
|
function verifyMemory(
|
||||||
db: MindDB,
|
snapshot: MemorySnapshot,
|
||||||
scenario: PromptAssemblerScenario,
|
scenario: PromptAssemblerScenario,
|
||||||
): { count: number; matches: Record<string, boolean> } {
|
): { count: number; matches: Record<string, boolean> } {
|
||||||
const raw = db.getDatabase();
|
|
||||||
const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
|
|
||||||
const matches: Record<string, boolean> = {};
|
const matches: Record<string, boolean> = {};
|
||||||
for (const sub of scenario.memoryVerificationSubstrings) {
|
for (const sub of scenario.memoryVerificationSubstrings) {
|
||||||
const row = raw.prepare('SELECT 1 FROM memory_frames WHERE content LIKE ? LIMIT 1').get(`%${sub}%`);
|
matches[sub] = snapshot.some(record => record.text.toLowerCase().includes(sub.toLowerCase()));
|
||||||
matches[sub] = !!row;
|
|
||||||
}
|
}
|
||||||
return { count, matches };
|
return { count: snapshot.length, matches };
|
||||||
}
|
}
|
||||||
|
|
||||||
async function runCondition(
|
async function runCondition(
|
||||||
snapshotPath: string,
|
snapshot: MemorySnapshot,
|
||||||
condition: ConditionSpec,
|
condition: ConditionSpec,
|
||||||
scenario: PromptAssemblerScenario,
|
scenario: PromptAssemblerScenario,
|
||||||
workDir: string,
|
|
||||||
seed: number,
|
seed: number,
|
||||||
): Promise<ConditionRun> {
|
): Promise<ConditionRun> {
|
||||||
const safeCode = condition.code.replace(/[^\w]/g, '_');
|
const orch = createOrchestrator(await restoreMemory(snapshot), condition.model);
|
||||||
const workDbPath = path.join(workDir, `work-${safeCode}-seed${seed}.db`);
|
|
||||||
fs.copyFileSync(snapshotPath, workDbPath);
|
|
||||||
|
|
||||||
const db = new MindDB(workDbPath);
|
|
||||||
const orch = new Orchestrator({
|
|
||||||
db,
|
|
||||||
embedder: new StubEmbedder(),
|
|
||||||
model: condition.model,
|
|
||||||
});
|
|
||||||
|
|
||||||
let systemPrompt: string;
|
let systemPrompt: string;
|
||||||
let debug: ConditionRun['debug'] = undefined;
|
let debug: ConditionRun['debug'] = undefined;
|
||||||
@@ -345,7 +336,7 @@ async function runCondition(
|
|||||||
};
|
};
|
||||||
} else {
|
} else {
|
||||||
delete process.env.WAGGLE_PROMPT_ASSEMBLER;
|
delete process.env.WAGGLE_PROMPT_ASSEMBLER;
|
||||||
systemPrompt = orch.buildSystemPrompt();
|
systemPrompt = await orch.buildSystemPrompt();
|
||||||
}
|
}
|
||||||
|
|
||||||
const output = await callModel(
|
const output = await callModel(
|
||||||
@@ -355,10 +346,8 @@ async function runCondition(
|
|||||||
scenario.testTurn.query,
|
scenario.testTurn.query,
|
||||||
{ temperature: TEMPERATURE_GEN, seed },
|
{ temperature: TEMPERATURE_GEN, seed },
|
||||||
);
|
);
|
||||||
db.close();
|
|
||||||
return { seed, output, durationMs: Date.now() - start, debug };
|
return { seed, output, durationMs: Date.now() - start, debug };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
db.close();
|
|
||||||
return {
|
return {
|
||||||
seed,
|
seed,
|
||||||
output: '',
|
output: '',
|
||||||
@@ -640,23 +629,10 @@ async function main(): Promise<void> {
|
|||||||
for (const [idx, scenario] of SCENARIOS.entries()) {
|
for (const [idx, scenario] of SCENARIOS.entries()) {
|
||||||
console.log(`\n[${idx + 1}/${SCENARIOS.length}] === Scenario: ${scenario.name} (${scenario.language}, ${scenario.shape}) ===`);
|
console.log(`\n[${idx + 1}/${SCENARIOS.length}] === Scenario: ${scenario.name} (${scenario.language}, ${scenario.shape}) ===`);
|
||||||
|
|
||||||
const setup = setupCleanScenario(scenario.name);
|
|
||||||
const primingStart = Date.now();
|
const primingStart = Date.now();
|
||||||
|
|
||||||
const primingDb = new MindDB(setup.dbPath);
|
const primingMemory = createInMemoryMemoryStore();
|
||||||
const primingOrch = new Orchestrator({
|
const primingOrch = createOrchestrator(primingMemory, PRIMING_MODEL);
|
||||||
db: primingDb,
|
|
||||||
embedder: new StubEmbedder(),
|
|
||||||
model: PRIMING_MODEL,
|
|
||||||
});
|
|
||||||
primingOrch.getIdentity().create({
|
|
||||||
name: 'Marko',
|
|
||||||
role: 'CEO',
|
|
||||||
department: 'Egzakta Group',
|
|
||||||
personality: 'Direct, pragmatic, sovereignty-focused',
|
|
||||||
capabilities: 'Strategic decisions, technical oversight',
|
|
||||||
system_prompt: '',
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log(` [priming] ${scenario.primingTurns.length} turns via ${PRIMING_MODEL}...`);
|
console.log(` [priming] ${scenario.primingTurns.length} turns via ${PRIMING_MODEL}...`);
|
||||||
try {
|
try {
|
||||||
@@ -665,12 +641,10 @@ async function main(): Promise<void> {
|
|||||||
console.error(` [priming] failed: ${err instanceof Error ? err.message : String(err)}`);
|
console.error(` [priming] failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const verification = verifyMemory(primingDb, scenario);
|
const snapshot = await snapshotMemory(primingMemory);
|
||||||
|
const verification = verifyMemory(snapshot, scenario);
|
||||||
const primingFailed = verification.count < 2 || !Object.values(verification.matches).some(v => v);
|
const primingFailed = verification.count < 2 || !Object.values(verification.matches).some(v => v);
|
||||||
console.log(` [priming] frames=${verification.count}, matches=${JSON.stringify(verification.matches)}, failed=${primingFailed}`);
|
console.log(` [priming] memories=${verification.count}, matches=${JSON.stringify(verification.matches)}, failed=${primingFailed}`);
|
||||||
|
|
||||||
primingDb.close();
|
|
||||||
fs.copyFileSync(setup.dbPath, setup.snapshotPath);
|
|
||||||
|
|
||||||
const scenarioResult: ScenarioResult = {
|
const scenarioResult: ScenarioResult = {
|
||||||
scenario: scenario.name,
|
scenario: scenario.name,
|
||||||
@@ -687,7 +661,7 @@ async function main(): Promise<void> {
|
|||||||
scenarioResult.conditions[condition.code] = [];
|
scenarioResult.conditions[condition.code] = [];
|
||||||
for (let seed = 0; seed < SEEDS; seed++) {
|
for (let seed = 0; seed < SEEDS; seed++) {
|
||||||
console.log(` [run] ${condition.code} (${condition.label}), seed=${seed}...`);
|
console.log(` [run] ${condition.code} (${condition.label}), seed=${seed}...`);
|
||||||
const run = await runCondition(setup.snapshotPath, condition, scenario, setup.tempDir, seed);
|
const run = await runCondition(snapshot, condition, scenario, seed);
|
||||||
scenarioResult.conditions[condition.code].push(run);
|
scenarioResult.conditions[condition.code].push(run);
|
||||||
if (run.error) console.log(` ERROR: ${run.error.slice(0, 200)}`);
|
if (run.error) console.log(` ERROR: ${run.error.slice(0, 200)}`);
|
||||||
else console.log(` OK (${run.durationMs}ms, ${run.output.length} chars)`);
|
else console.log(` OK (${run.durationMs}ms, ${run.output.length} chars)`);
|
||||||
@@ -727,7 +701,6 @@ async function main(): Promise<void> {
|
|||||||
|
|
||||||
result.scenarios.push(scenarioResult);
|
result.scenarios.push(scenarioResult);
|
||||||
fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
|
fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
|
||||||
fs.rmSync(setup.tempDir, { recursive: true, force: true });
|
|
||||||
}
|
}
|
||||||
|
|
||||||
result.durationMs = Date.now() - startTime;
|
result.durationMs = Date.now() - startTime;
|
||||||
|
|||||||
@@ -32,7 +32,13 @@
|
|||||||
* judges are reachable, the eval halts and reports partial state.
|
* judges are reachable, the eval halts and reports partial state.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { MindDB, VaultStore, type Embedder } from '@waggle/core';
|
import { VaultStore } from '@waggle/core';
|
||||||
|
import {
|
||||||
|
createInMemoryMemoryStore,
|
||||||
|
PERSONAL_SCOPE,
|
||||||
|
type MemoryRecord,
|
||||||
|
type MemoryStore,
|
||||||
|
} from '@waggle/memory-contract';
|
||||||
import { Orchestrator } from '../../src/orchestrator.js';
|
import { Orchestrator } from '../../src/orchestrator.js';
|
||||||
import { type ModelTier } from '../../src/model-tier.js';
|
import { type ModelTier } from '../../src/model-tier.js';
|
||||||
import { detectTaskShape } from '../../src/task-shape.js';
|
import { detectTaskShape } from '../../src/task-shape.js';
|
||||||
@@ -607,43 +613,50 @@ const SECONDARY_QWEN_CONDITIONS: ConditionSpec[] = [
|
|||||||
{ code: 'H', label: 'Qwen3-30B thinking · PA (compression)', model: QWEN_THINKING_MODEL, provider: 'openrouter', usesPromptAssembler: true, scaffoldStyle: 'compression', suite: 'secondary-qwen' },
|
{ code: 'H', label: 'Qwen3-30B thinking · PA (compression)', model: QWEN_THINKING_MODEL, provider: 'openrouter', usesPromptAssembler: true, scaffoldStyle: 'compression', suite: 'secondary-qwen' },
|
||||||
];
|
];
|
||||||
|
|
||||||
// ── Stub embedder (reuse v4 pattern) ────────────────────────────────
|
|
||||||
|
|
||||||
class StubEmbedder implements Embedder {
|
|
||||||
private dim = 384;
|
|
||||||
async embed(_text: string): Promise<Float32Array> { return new Float32Array(this.dim).fill(0); }
|
|
||||||
async embedBatch(texts: string[]): Promise<Float32Array[]> { return Promise.all(texts.map(t => this.embed(t))); }
|
|
||||||
getDimension(): number { return this.dim; }
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── Scenario pipeline ───────────────────────────────────────────────
|
// ── Scenario pipeline ───────────────────────────────────────────────
|
||||||
|
|
||||||
interface ScenarioSetup {
|
/** Memories left by priming. Every condition starts from a fresh store holding a copy. */
|
||||||
tempDir: string;
|
type MemorySnapshot = readonly MemoryRecord[];
|
||||||
dbPath: string;
|
|
||||||
snapshotPath: string;
|
function createOrchestrator(memory: MemoryStore, model: string): Orchestrator {
|
||||||
|
const orch = new Orchestrator({ memory, model });
|
||||||
|
orch.getIdentity().create({
|
||||||
|
name: 'Marko',
|
||||||
|
role: 'CEO',
|
||||||
|
department: 'Egzakta Group',
|
||||||
|
personality: 'Direct, pragmatic, sovereignty-focused',
|
||||||
|
capabilities: 'Strategic decisions, technical oversight',
|
||||||
|
system_prompt: '',
|
||||||
|
});
|
||||||
|
return orch;
|
||||||
}
|
}
|
||||||
|
|
||||||
function setupCleanScenario(scenarioName: string): ScenarioSetup {
|
async function snapshotMemory(memory: MemoryStore): Promise<MemorySnapshot> {
|
||||||
const tempDir = path.join(os.tmpdir(), `waggle-v5eval-${Date.now()}-${scenarioName}`);
|
const filter = { scopes: [PERSONAL_SCOPE] };
|
||||||
if (fs.existsSync(tempDir)) fs.rmSync(tempDir, { recursive: true, force: true });
|
return memory.list({ ...filter, limit: await memory.count(filter) });
|
||||||
fs.mkdirSync(tempDir, { recursive: true });
|
|
||||||
const dbPath = path.join(tempDir, 'mind.db');
|
|
||||||
const snapshotPath = path.join(tempDir, 'snapshot.db');
|
|
||||||
const db = new MindDB(dbPath);
|
|
||||||
const raw = db.getDatabase();
|
|
||||||
const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
|
|
||||||
if (count !== 0) {
|
|
||||||
db.close();
|
|
||||||
throw new Error(`CLEAN SLATE VIOLATION for ${scenarioName}: ${count} frames in fresh DB`);
|
|
||||||
}
|
}
|
||||||
db.close();
|
|
||||||
return { tempDir, dbPath, snapshotPath };
|
async function restoreMemory(snapshot: MemorySnapshot): Promise<MemoryStore> {
|
||||||
|
const memory = createInMemoryMemoryStore();
|
||||||
|
// list() returns newest first; replay oldest first.
|
||||||
|
for (const record of [...snapshot].reverse()) {
|
||||||
|
await memory.remember({
|
||||||
|
scope: record.scope,
|
||||||
|
text: record.text,
|
||||||
|
kind: record.kind,
|
||||||
|
importance: record.importance,
|
||||||
|
source: record.source,
|
||||||
|
origin: record.origin ?? undefined,
|
||||||
|
sessionId: record.sessionId ?? undefined,
|
||||||
|
metadata: record.metadata,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return memory;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function runPriming(orch: Orchestrator, scenario: PromptAssemblerScenario): Promise<void> {
|
async function runPriming(orch: Orchestrator, scenario: PromptAssemblerScenario): Promise<void> {
|
||||||
for (const turn of scenario.primingTurns) {
|
for (const turn of scenario.primingTurns) {
|
||||||
const systemPrompt = orch.buildSystemPrompt();
|
const systemPrompt = await orch.buildSystemPrompt();
|
||||||
const assistantMsg = await callAnthropic(PRIMING_MODEL, systemPrompt, turn.user, {
|
const assistantMsg = await callAnthropic(PRIMING_MODEL, systemPrompt, turn.user, {
|
||||||
temperature: 0,
|
temperature: 0,
|
||||||
maxTokens: MAX_TOKENS_GEN,
|
maxTokens: MAX_TOKENS_GEN,
|
||||||
@@ -652,30 +665,24 @@ async function runPriming(orch: Orchestrator, scenario: PromptAssemblerScenario)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function verifyMemory(db: MindDB, scenario: PromptAssemblerScenario): { count: number; matches: Record<string, boolean> } {
|
function verifyMemory(
|
||||||
const raw = db.getDatabase();
|
snapshot: MemorySnapshot,
|
||||||
const count = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
|
scenario: PromptAssemblerScenario,
|
||||||
|
): { count: number; matches: Record<string, boolean> } {
|
||||||
const matches: Record<string, boolean> = {};
|
const matches: Record<string, boolean> = {};
|
||||||
for (const sub of scenario.memoryVerificationSubstrings) {
|
for (const sub of scenario.memoryVerificationSubstrings) {
|
||||||
const row = raw.prepare('SELECT 1 FROM memory_frames WHERE content LIKE ? LIMIT 1').get(`%${sub}%`);
|
matches[sub] = snapshot.some(record => record.text.toLowerCase().includes(sub.toLowerCase()));
|
||||||
matches[sub] = !!row;
|
|
||||||
}
|
}
|
||||||
return { count, matches };
|
return { count: snapshot.length, matches };
|
||||||
}
|
}
|
||||||
|
|
||||||
async function runCondition(
|
async function runCondition(
|
||||||
snapshotPath: string,
|
snapshot: MemorySnapshot,
|
||||||
condition: ConditionSpec,
|
condition: ConditionSpec,
|
||||||
scenario: PromptAssemblerScenario,
|
scenario: PromptAssemblerScenario,
|
||||||
workDir: string,
|
|
||||||
seed: number,
|
seed: number,
|
||||||
): Promise<ConditionRun> {
|
): Promise<ConditionRun> {
|
||||||
const safeCode = condition.code.replace(/[^\w]/g, '_');
|
const orch = createOrchestrator(await restoreMemory(snapshot), condition.model);
|
||||||
const workDbPath = path.join(workDir, `work-${safeCode}-seed${seed}.db`);
|
|
||||||
fs.copyFileSync(snapshotPath, workDbPath);
|
|
||||||
|
|
||||||
const db = new MindDB(workDbPath);
|
|
||||||
const orch = new Orchestrator({ db, embedder: new StubEmbedder(), model: condition.model });
|
|
||||||
|
|
||||||
let systemPrompt: string;
|
let systemPrompt: string;
|
||||||
let debug: ConditionRun['debug'] = undefined;
|
let debug: ConditionRun['debug'] = undefined;
|
||||||
@@ -702,7 +709,7 @@ async function runCondition(
|
|||||||
};
|
};
|
||||||
} else {
|
} else {
|
||||||
delete process.env.WAGGLE_PROMPT_ASSEMBLER;
|
delete process.env.WAGGLE_PROMPT_ASSEMBLER;
|
||||||
systemPrompt = orch.buildSystemPrompt();
|
systemPrompt = await orch.buildSystemPrompt();
|
||||||
}
|
}
|
||||||
|
|
||||||
const output = await callGenerationModel(
|
const output = await callGenerationModel(
|
||||||
@@ -712,10 +719,8 @@ async function runCondition(
|
|||||||
scenario.testTurn.query,
|
scenario.testTurn.query,
|
||||||
{ temperature: TEMPERATURE_GEN, seed },
|
{ temperature: TEMPERATURE_GEN, seed },
|
||||||
);
|
);
|
||||||
db.close();
|
|
||||||
return { seed, output, durationMs: Date.now() - start, debug };
|
return { seed, output, durationMs: Date.now() - start, debug };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
db.close();
|
|
||||||
return {
|
return {
|
||||||
seed,
|
seed,
|
||||||
output: '',
|
output: '',
|
||||||
@@ -1111,23 +1116,10 @@ async function main(): Promise<void> {
|
|||||||
for (const [idx, scenario] of scenariosToRun.entries()) {
|
for (const [idx, scenario] of scenariosToRun.entries()) {
|
||||||
console.log(`\n[${idx + 1}/${scenariosToRun.length}] === ${scenario.name} (${scenario.language}, ${scenario.shape}) ===`);
|
console.log(`\n[${idx + 1}/${scenariosToRun.length}] === ${scenario.name} (${scenario.language}, ${scenario.shape}) ===`);
|
||||||
|
|
||||||
const setup = setupCleanScenario(scenario.name);
|
|
||||||
const primingStart = Date.now();
|
const primingStart = Date.now();
|
||||||
|
|
||||||
const primingDb = new MindDB(setup.dbPath);
|
const primingMemory = createInMemoryMemoryStore();
|
||||||
const primingOrch = new Orchestrator({
|
const primingOrch = createOrchestrator(primingMemory, PRIMING_MODEL);
|
||||||
db: primingDb,
|
|
||||||
embedder: new StubEmbedder(),
|
|
||||||
model: PRIMING_MODEL,
|
|
||||||
});
|
|
||||||
primingOrch.getIdentity().create({
|
|
||||||
name: 'Marko',
|
|
||||||
role: 'CEO',
|
|
||||||
department: 'Egzakta Group',
|
|
||||||
personality: 'Direct, pragmatic, sovereignty-focused',
|
|
||||||
capabilities: 'Strategic decisions, technical oversight',
|
|
||||||
system_prompt: '',
|
|
||||||
});
|
|
||||||
|
|
||||||
const effectiveTier = process.env.WAGGLE_EVAL_MODE === '1' ? 'bypassed (enterprise-equivalent)' : 'default (tier-enforced)';
|
const effectiveTier = process.env.WAGGLE_EVAL_MODE === '1' ? 'bypassed (enterprise-equivalent)' : 'default (tier-enforced)';
|
||||||
console.log(` [priming] ${scenario.primingTurns.length} turns via ${PRIMING_MODEL} | tier: ${effectiveTier}`);
|
console.log(` [priming] ${scenario.primingTurns.length} turns via ${PRIMING_MODEL} | tier: ${effectiveTier}`);
|
||||||
@@ -1138,13 +1130,11 @@ async function main(): Promise<void> {
|
|||||||
console.error(` [priming] failed: ${err instanceof Error ? err.message : String(err)}`);
|
console.error(` [priming] failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const verification = verifyMemory(primingDb, scenario);
|
const snapshot = await snapshotMemory(primingMemory);
|
||||||
|
const verification = verifyMemory(snapshot, scenario);
|
||||||
const primingFailed = verification.count < 2 || !Object.values(verification.matches).some(v => v);
|
const primingFailed = verification.count < 2 || !Object.values(verification.matches).some(v => v);
|
||||||
const primingPartial = !primingFailed && Object.values(verification.matches).some(v => !v);
|
const primingPartial = !primingFailed && Object.values(verification.matches).some(v => !v);
|
||||||
console.log(` [priming] frames=${verification.count}, matches=${JSON.stringify(verification.matches)}, failed=${primingFailed}, partial=${primingPartial}`);
|
console.log(` [priming] memories=${verification.count}, matches=${JSON.stringify(verification.matches)}, failed=${primingFailed}, partial=${primingPartial}`);
|
||||||
|
|
||||||
primingDb.close();
|
|
||||||
fs.copyFileSync(setup.dbPath, setup.snapshotPath);
|
|
||||||
|
|
||||||
const scenarioResult: ScenarioResult = {
|
const scenarioResult: ScenarioResult = {
|
||||||
scenario: scenario.name,
|
scenario: scenario.name,
|
||||||
@@ -1167,7 +1157,7 @@ async function main(): Promise<void> {
|
|||||||
// Initial 3 seeds.
|
// Initial 3 seeds.
|
||||||
for (let seed = 0; seed < BASE_SEEDS; seed++) {
|
for (let seed = 0; seed < BASE_SEEDS; seed++) {
|
||||||
process.stdout.write(` [run] ${condition.code.padEnd(4)} seed=${seed}... `);
|
process.stdout.write(` [run] ${condition.code.padEnd(4)} seed=${seed}... `);
|
||||||
const run = await runCondition(setup.snapshotPath, condition, scenario, setup.tempDir, seed);
|
const run = await runCondition(snapshot, condition, scenario, seed);
|
||||||
scenarioResult.conditions[condition.code].push(run);
|
scenarioResult.conditions[condition.code].push(run);
|
||||||
if (run.error) console.log(`ERROR: ${run.error.slice(0, 120)}`);
|
if (run.error) console.log(`ERROR: ${run.error.slice(0, 120)}`);
|
||||||
else console.log(`OK (${run.durationMs}ms, ${run.output.length}ch)`);
|
else console.log(`OK (${run.durationMs}ms, ${run.output.length}ch)`);
|
||||||
@@ -1209,7 +1199,7 @@ async function main(): Promise<void> {
|
|||||||
scenarioResult.retriedConditions.push(condition.code);
|
scenarioResult.retriedConditions.push(condition.code);
|
||||||
for (let seed = BASE_SEEDS; seed < BASE_SEEDS + RETRY_EXTRA_SEEDS; seed++) {
|
for (let seed = BASE_SEEDS; seed < BASE_SEEDS + RETRY_EXTRA_SEEDS; seed++) {
|
||||||
process.stdout.write(` [run] ${condition.code.padEnd(4)} seed=${seed}... `);
|
process.stdout.write(` [run] ${condition.code.padEnd(4)} seed=${seed}... `);
|
||||||
const run = await runCondition(setup.snapshotPath, condition, scenario, setup.tempDir, seed);
|
const run = await runCondition(snapshot, condition, scenario, seed);
|
||||||
scenarioResult.conditions[condition.code].push(run);
|
scenarioResult.conditions[condition.code].push(run);
|
||||||
if (run.error) {
|
if (run.error) {
|
||||||
console.log(`ERROR: ${run.error.slice(0, 120)}`);
|
console.log(`ERROR: ${run.error.slice(0, 120)}`);
|
||||||
@@ -1229,7 +1219,6 @@ async function main(): Promise<void> {
|
|||||||
|
|
||||||
result.scenarios.push(scenarioResult);
|
result.scenarios.push(scenarioResult);
|
||||||
fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
|
fs.writeFileSync(RESULTS_JSON, JSON.stringify(result, null, 2));
|
||||||
fs.rmSync(setup.tempDir, { recursive: true, force: true });
|
|
||||||
}
|
}
|
||||||
|
|
||||||
result.durationMs = Date.now() - startTime;
|
result.durationMs = Date.now() - startTime;
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ import {
|
|||||||
ConfirmationGate,
|
ConfirmationGate,
|
||||||
MemoryLinker,
|
MemoryLinker,
|
||||||
} from '@waggle/agent';
|
} from '@waggle/agent';
|
||||||
import { Ontology, validateEntity } from '@waggle/core';
|
|
||||||
|
|
||||||
describe('M3c Integration', () => {
|
describe('M3c Integration', () => {
|
||||||
it('hook registry registers and fires', async () => {
|
it('hook registry registers and fires', async () => {
|
||||||
@@ -65,25 +64,12 @@ describe('M3c Integration', () => {
|
|||||||
expect(needsConfirmation('git_commit')).toBe(true);
|
expect(needsConfirmation('git_commit')).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('ontology validates entities', () => {
|
|
||||||
const ontology = new Ontology();
|
|
||||||
ontology.define('person', { required: ['name'], optional: ['email'] });
|
|
||||||
|
|
||||||
const valid = validateEntity(ontology, { type: 'person', properties: { name: 'Alice' } });
|
|
||||||
expect(valid.valid).toBe(true);
|
|
||||||
|
|
||||||
const invalid = validateEntity(ontology, { type: 'person', properties: { email: 'a@b.com' } });
|
|
||||||
expect(invalid.valid).toBe(false);
|
|
||||||
expect(invalid.issues).toContain('Missing required property: name');
|
|
||||||
});
|
|
||||||
|
|
||||||
it('all M3c modules integrate without conflicts', async () => {
|
it('all M3c modules integrate without conflicts', async () => {
|
||||||
// Create all components — verify no import/construction errors
|
// Create all components — verify no import/construction errors
|
||||||
const hooks = new HookRegistry();
|
const hooks = new HookRegistry();
|
||||||
const plan = new Plan();
|
const plan = new Plan();
|
||||||
const permissions = PermissionManager.sandbox();
|
const permissions = PermissionManager.sandbox();
|
||||||
const gate = new ConfirmationGate({ interactive: false });
|
const gate = new ConfirmationGate({ interactive: false });
|
||||||
const ontology = new Ontology();
|
|
||||||
|
|
||||||
// Wire hooks into permission check
|
// Wire hooks into permission check
|
||||||
hooks.on('pre:tool', async (ctx) => {
|
hooks.on('pre:tool', async (ctx) => {
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
import { describe, it, expect } from 'vitest';
|
import { describe, it, expect } from 'vitest';
|
||||||
import type { MemoryFrame, Importance, FrameType, FrameSource } from '@waggle/core';
|
import type { MemoryImportance, MemorySource } from '@waggle/memory-contract';
|
||||||
import type { AgentPersona } from '../src/personas.js';
|
import type { AgentPersona } from '../src/personas.js';
|
||||||
import type { TaskShape } from '../src/task-shape.js';
|
import type { TaskShape } from '../src/task-shape.js';
|
||||||
import type { ContextFrames } from '../src/orchestrator.js';
|
import type { ContextFrames } from '../src/orchestrator.js';
|
||||||
|
import type { RecallFrame } from '../src/recall-frame.js';
|
||||||
import {
|
import {
|
||||||
PromptAssembler,
|
PromptAssembler,
|
||||||
type AssembleInput,
|
type AssembleInput,
|
||||||
@@ -14,20 +15,16 @@ import {
|
|||||||
let nextFrameId = 1;
|
let nextFrameId = 1;
|
||||||
function frame(
|
function frame(
|
||||||
content: string,
|
content: string,
|
||||||
opts: { type?: FrameType; importance?: Importance; source?: FrameSource } = {},
|
opts: { importance?: MemoryImportance; source?: MemorySource } = {},
|
||||||
): MemoryFrame {
|
): RecallFrame {
|
||||||
return {
|
return {
|
||||||
id: nextFrameId++,
|
id: String(nextFrameId++),
|
||||||
frame_type: opts.type ?? 'I',
|
|
||||||
gop_id: 'gop-1',
|
|
||||||
t: 0,
|
|
||||||
base_frame_id: null,
|
|
||||||
content,
|
content,
|
||||||
|
kind: 'fact',
|
||||||
importance: opts.importance ?? 'normal',
|
importance: opts.importance ?? 'normal',
|
||||||
source: opts.source ?? 'user_stated',
|
source: opts.source ?? 'user_stated',
|
||||||
access_count: 0,
|
|
||||||
created_at: '2026-04-17T10:00:00Z',
|
created_at: '2026-04-17T10:00:00Z',
|
||||||
last_accessed: '2026-04-17T10:00:00Z',
|
score: 1,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -113,7 +110,7 @@ describe('PromptAssembler.assemble', () => {
|
|||||||
it('small tier caps State frames at 3', () => {
|
it('small tier caps State frames at 3', () => {
|
||||||
const frames: MemoryFrame[] = [];
|
const frames: MemoryFrame[] = [];
|
||||||
for (let i = 0; i < 10; i++) {
|
for (let i = 0; i < 10; i++) {
|
||||||
frames.push(frame(`State frame ${i}`, { type: 'I', importance: 'important' }));
|
frames.push(frame(`State frame ${i}`, { importance: 'important' }));
|
||||||
}
|
}
|
||||||
const out = assembler.assemble(
|
const out = assembler.assemble(
|
||||||
baseInput({
|
baseInput({
|
||||||
@@ -216,7 +213,7 @@ describe('PromptAssembler.assemble', () => {
|
|||||||
context: {
|
context: {
|
||||||
...emptyContext(),
|
...emptyContext(),
|
||||||
stateFrames: [frame('State says shipping now is safe.')],
|
stateFrames: [frame('State says shipping now is safe.')],
|
||||||
recentChanges: [frame('Recent changes say all gaps are closed.', { type: 'P' })],
|
recentChanges: [frame('Recent changes say all gaps are closed.')],
|
||||||
activeWork: [{ category: 'task', content: 'Ship immediately.', priority: 1 }],
|
activeWork: [{ category: 'task', content: 'Ship immediately.', priority: 1 }],
|
||||||
},
|
},
|
||||||
recalled: {
|
recalled: {
|
||||||
@@ -331,8 +328,8 @@ describe('PromptAssembler.assemble', () => {
|
|||||||
it('maxSystemChars exceeded: Recent changes trimmed first, then Active work, then State', () => {
|
it('maxSystemChars exceeded: Recent changes trimmed first, then Active work, then State', () => {
|
||||||
const big = 'x'.repeat(2000);
|
const big = 'x'.repeat(2000);
|
||||||
const ctx: ContextFrames = {
|
const ctx: ContextFrames = {
|
||||||
stateFrames: [frame(big, { type: 'I', importance: 'critical' })],
|
stateFrames: [frame(big, { importance: 'critical' })],
|
||||||
recentChanges: [frame(big, { type: 'P', importance: 'normal' })],
|
recentChanges: [frame(big, { importance: 'normal' })],
|
||||||
activeWork: [{ category: 'task', content: big, priority: 1 }],
|
activeWork: [{ category: 'task', content: big, priority: 1 }],
|
||||||
keyEntities: [],
|
keyEntities: [],
|
||||||
personalPreferences: [],
|
personalPreferences: [],
|
||||||
@@ -370,8 +367,8 @@ describe('PromptAssembler.assemble', () => {
|
|||||||
const out = assembler.assemble(
|
const out = assembler.assemble(
|
||||||
baseInput({
|
baseInput({
|
||||||
recalled: {
|
recalled: {
|
||||||
workspace: [frame('Poisoned content with secret data', { type: 'P' })],
|
workspace: [frame('Poisoned content with secret data')],
|
||||||
personal: [frame('Another recalled frame', { type: 'P' })],
|
personal: [frame('Another recalled frame')],
|
||||||
scanSafe: false,
|
scanSafe: false,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
@@ -385,8 +382,8 @@ describe('PromptAssembler.assemble', () => {
|
|||||||
const out = assembler.assemble(
|
const out = assembler.assemble(
|
||||||
baseInput({
|
baseInput({
|
||||||
recalled: {
|
recalled: {
|
||||||
workspace: [frame('Workspace memory item', { type: 'P' })],
|
workspace: [frame('Workspace memory item')],
|
||||||
personal: [frame('Personal memory item', { type: 'P' })],
|
personal: [frame('Personal memory item')],
|
||||||
scanSafe: true,
|
scanSafe: true,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
@@ -398,8 +395,8 @@ describe('PromptAssembler.assemble', () => {
|
|||||||
|
|
||||||
it('debug.framesUsed counts rendered frames across sections', () => {
|
it('debug.framesUsed counts rendered frames across sections', () => {
|
||||||
const ctx: ContextFrames = {
|
const ctx: ContextFrames = {
|
||||||
stateFrames: [frame('State 1', { type: 'I' }), frame('State 2', { type: 'I' })],
|
stateFrames: [frame('State 1'), frame('State 2')],
|
||||||
recentChanges: [frame('Change 1', { type: 'P' })],
|
recentChanges: [frame('Change 1')],
|
||||||
activeWork: [{ category: 'task', content: 'Active', priority: 1 }],
|
activeWork: [{ category: 'task', content: 'Active', priority: 1 }],
|
||||||
keyEntities: [],
|
keyEntities: [],
|
||||||
personalPreferences: [],
|
personalPreferences: [],
|
||||||
@@ -424,9 +421,9 @@ describe('PromptAssembler.assemble', () => {
|
|||||||
it('ranks I-frames by importance then recency', () => {
|
it('ranks I-frames by importance then recency', () => {
|
||||||
const ctx: ContextFrames = {
|
const ctx: ContextFrames = {
|
||||||
stateFrames: [
|
stateFrames: [
|
||||||
frame('Old normal', { type: 'I', importance: 'normal' }),
|
frame('Old normal', { importance: 'normal' }),
|
||||||
frame('New critical', { type: 'I', importance: 'critical' }),
|
frame('New critical', { importance: 'critical' }),
|
||||||
frame('Middle important', { type: 'I', importance: 'important' }),
|
frame('Middle important', { importance: 'important' }),
|
||||||
],
|
],
|
||||||
recentChanges: [],
|
recentChanges: [],
|
||||||
activeWork: [],
|
activeWork: [],
|
||||||
@@ -441,7 +438,7 @@ describe('PromptAssembler.assemble', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('dedupes frames by id', () => {
|
it('dedupes frames by id', () => {
|
||||||
const f = frame('Duplicate', { type: 'I' });
|
const f = frame('Duplicate');
|
||||||
const ctx: ContextFrames = {
|
const ctx: ContextFrames = {
|
||||||
stateFrames: [f, f, f],
|
stateFrames: [f, f, f],
|
||||||
recentChanges: [],
|
recentChanges: [],
|
||||||
|
|||||||
Reference in New Issue
Block a user