1705 lines
75 KiB
TypeScript
1705 lines
75 KiB
TypeScript
import fs from 'node:fs';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import type { FastifyInstance } from 'fastify';
|
|
import { MindDB, WaggleConfig } from '@waggle/core';
|
|
import type { AgentLoopConfig, AgentResponse } from '@waggle/agent';
|
|
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { PERSONA_CASES } from '../../../../tests/vision/persona-cases.js';
|
|
import {
|
|
CANONICAL_VERIFIER_REPORT,
|
|
renderVerifierReportEnvelope,
|
|
} from '../../../../tests/vision/verifier-contract.js';
|
|
import { buildLocalServer } from '../../src/local/index.js';
|
|
import { closeAuditDb, getAuditDb } from '../../src/local/routes/events.js';
|
|
import { chatSessionStateKey, persistMessage } from '../../src/local/routes/chat-persistence.js';
|
|
import { injectWithAuth, resetRateLimiter } from '../test-utils.js';
|
|
|
|
const testState = vi.hoisted(() => {
|
|
const previousPromptAssembler = process.env.WAGGLE_PROMPT_ASSEMBLER;
|
|
process.env.WAGGLE_PROMPT_ASSEMBLER = '1';
|
|
return {
|
|
previousPromptAssembler,
|
|
optimizerExpand: vi.fn(),
|
|
runAgentLoop: vi.fn(),
|
|
};
|
|
});
|
|
|
|
vi.mock('@waggle/agent', async (importOriginal) => {
|
|
const actual = await importOriginal<typeof import('@waggle/agent')>();
|
|
return {
|
|
...actual,
|
|
runAgentLoop: testState.runAgentLoop,
|
|
};
|
|
});
|
|
|
|
vi.mock('../../src/local/model-availability.js', async (importOriginal) => {
|
|
const actual = await importOriginal<typeof import('../../src/local/model-availability.js')>();
|
|
return {
|
|
...actual,
|
|
resolveUsableModel: async (_server: FastifyInstance, requestedModel: string) => requestedModel,
|
|
};
|
|
});
|
|
|
|
vi.mock('../../src/local/services/optimizer-service.js', () => ({
|
|
getOptimizerService: async () => ({
|
|
expandWithChoices: testState.optimizerExpand,
|
|
}),
|
|
}));
|
|
|
|
function parseSse(raw: string): Array<{ event: string; data: Record<string, unknown> }> {
|
|
return raw.split(/\n\n/)
|
|
.filter(Boolean)
|
|
.map((block) => {
|
|
const lines = block.split('\n');
|
|
const event = lines.find(line => line.startsWith('event: '))?.slice(7) ?? '';
|
|
const data = lines.find(line => line.startsWith('data: '))?.slice(6) ?? '{}';
|
|
return { event, data: JSON.parse(data) as Record<string, unknown> };
|
|
});
|
|
}
|
|
|
|
const SYNTHETIC_PROVIDER_PROTOCOL_OVERHEAD_CHARS = 2_048;
|
|
const PERSISTED_IDENTITY_SENTINEL = 'Persisted Identity Sentinel';
|
|
const PERSISTED_PROFILE_SENTINEL = 'Persisted Profile Sentinel';
|
|
const PERSISTED_MEMORY_SENTINEL = 'Persisted Memory Sentinel launch decision';
|
|
const PERSISTED_SKILL_SENTINEL = 'persisted-skill-sentinel';
|
|
|
|
/**
|
|
* Test-only static bound: two prompt characters per synthetic token plus a
|
|
* fixed wire/protocol allowance. This is deliberately stricter than the
|
|
* production chars/4 estimate, but it is not a Sonnet tokenizer measurement.
|
|
* The mocked usage value verifies done-metric plumbing only; a paid canary is
|
|
* still the sole proof of the real provider input-token budget.
|
|
*/
|
|
function conservativeSyntheticInputTokenUpperBound(config: AgentLoopConfig): number {
|
|
const messageChars = config.messages.reduce(
|
|
(total, message) => total + message.role.length + message.content.length + 16,
|
|
0,
|
|
);
|
|
const toolSchemaChars = config.tools.length === 0 ? 0 : JSON.stringify(config.tools).length;
|
|
return Math.ceil((
|
|
config.systemPrompt.length
|
|
+ messageChars
|
|
+ toolSchemaChars
|
|
+ SYNTHETIC_PROVIDER_PROTOCOL_OVERHEAD_CHARS
|
|
) / 2);
|
|
}
|
|
|
|
describe('persona acceptance prompt budget', () => {
|
|
let server: FastifyInstance;
|
|
let tmpDir: string;
|
|
let collaborationWorkspaceId: string;
|
|
let capturedConfig: AgentLoopConfig | null = null;
|
|
let capturedSyntheticInputUpperBound = 0;
|
|
|
|
const defaultRunAgentLoop = async (config: AgentLoopConfig): Promise<AgentResponse> => {
|
|
capturedConfig = config;
|
|
capturedSyntheticInputUpperBound = conservativeSyntheticInputTokenUpperBound(config);
|
|
const response = renderVerifierReportEnvelope(CANONICAL_VERIFIER_REPORT);
|
|
config.onToken?.(response);
|
|
return {
|
|
content: response,
|
|
toolsUsed: [],
|
|
usage: {
|
|
// Synthetic on purpose: the assertion below verifies SSE plumbing,
|
|
// not a real provider tokenizer or billing receipt.
|
|
inputTokens: capturedSyntheticInputUpperBound,
|
|
outputTokens: Math.ceil(response.length / 4),
|
|
},
|
|
};
|
|
};
|
|
|
|
beforeAll(async () => {
|
|
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-persona-budget-'));
|
|
const mind = new MindDB(path.join(tmpDir, 'personal.mind'));
|
|
mind.close();
|
|
fs.writeFileSync(
|
|
path.join(tmpDir, 'profile.json'),
|
|
JSON.stringify({ name: PERSISTED_PROFILE_SENTINEL }),
|
|
);
|
|
|
|
testState.runAgentLoop.mockImplementation(defaultRunAgentLoop);
|
|
|
|
server = await buildLocalServer({ dataDir: tmpDir });
|
|
collaborationWorkspaceId = server.workspaceManager.create({
|
|
name: 'Memory boundary collaboration',
|
|
group: 'Test',
|
|
}).id;
|
|
server.agentState.skills.push({
|
|
name: PERSISTED_SKILL_SENTINEL,
|
|
content: 'Private persisted skill guidance must stay outside a memory-denied turn.',
|
|
});
|
|
server.agentState.orchestrator.getIdentity().update({
|
|
name: PERSISTED_IDENTITY_SENTINEL,
|
|
role: 'Read-boundary test identity',
|
|
department: '',
|
|
personality: 'Private',
|
|
capabilities: 'Persistent context',
|
|
system_prompt: 'Never expose this when persisted memory is disabled.',
|
|
});
|
|
await server.agentState.orchestrator.executeTool('save_memory', {
|
|
content: PERSISTED_MEMORY_SENTINEL,
|
|
importance: 'high',
|
|
});
|
|
server.agentState.llmProvider = {
|
|
provider: 'litellm',
|
|
health: 'healthy',
|
|
detail: 'non-paid test double',
|
|
checkedAt: new Date().toISOString(),
|
|
};
|
|
});
|
|
|
|
beforeEach(() => {
|
|
resetRateLimiter(server);
|
|
testState.optimizerExpand.mockReset().mockResolvedValue({
|
|
expanded: null,
|
|
clarifyingQuestions: null,
|
|
intent: 'request',
|
|
isVague: false,
|
|
});
|
|
});
|
|
|
|
afterAll(async () => {
|
|
await server.close();
|
|
closeAuditDb();
|
|
await new Promise(resolve => setTimeout(resolve, 100));
|
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
if (testState.previousPromptAssembler === undefined) {
|
|
delete process.env.WAGGLE_PROMPT_ASSEMBLER;
|
|
} else {
|
|
process.env.WAGGLE_PROMPT_ASSEMBLER = testState.previousPromptAssembler;
|
|
}
|
|
});
|
|
|
|
async function capturePersonaTurn(personaId: 'coder' | 'data-engineer' | 'coordinator') {
|
|
const persona = PERSONA_CASES.find(item => item.id === personaId)!;
|
|
capturedConfig = null;
|
|
capturedSyntheticInputUpperBound = 0;
|
|
const response = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: persona.prompt,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: persona.id,
|
|
session: `persona-acceptance-${personaId}-budget`,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
expect(response.statusCode).toBe(200);
|
|
expect(capturedConfig).not.toBeNull();
|
|
return {
|
|
persona,
|
|
config: capturedConfig!,
|
|
events: parseSse(response.body),
|
|
syntheticInputUpperBound: capturedSyntheticInputUpperBound,
|
|
};
|
|
}
|
|
|
|
it('supports Coder search then read within a lean three-dispatch envelope', async () => {
|
|
const { persona, config, events, syntheticInputUpperBound } = await capturePersonaTurn('coder');
|
|
const selectedNames = config.tools.map(tool => tool.name);
|
|
|
|
expect(selectedNames).toContain('search_files');
|
|
expect(selectedNames.every(name => ['read_file', 'search_files', 'search_content'].includes(name))).toBe(true);
|
|
expect(config.systemPrompt).toMatch(/equivalent glob retries add no evidence/i);
|
|
expect(config.maxTurns).toBe(3);
|
|
expect(config.maxToolRounds).toBe(2);
|
|
expect(config.maxOutputTokens).toBeLessThanOrEqual(persona.maxOutputTokens);
|
|
expect(config.reasoning).toBeUndefined();
|
|
expect(syntheticInputUpperBound * 3).toBeLessThan(persona.maxInputTokens);
|
|
|
|
const metrics = events.find(event => event.event === 'done')?.data.contextMetrics as Record<string, unknown>;
|
|
expect(metrics.toolSelectedCount).toBeGreaterThan(0);
|
|
});
|
|
|
|
it('keeps the exact Data Engineer advisory turn tool-free, recall-free, compact, and completion-bounded', async () => {
|
|
const { persona, config, events, syntheticInputUpperBound } = await capturePersonaTurn('data-engineer');
|
|
|
|
expect(config.tools).toEqual([]);
|
|
expect(config.messages).toEqual([{ role: 'user', content: persona.prompt }]);
|
|
expect(config.maxOutputTokens).toBe(persona.maxOutputTokens);
|
|
expect(config.reasoning).toEqual({ enabled: true, effort: 'low' });
|
|
expect(syntheticInputUpperBound).toBeLessThan(persona.maxInputTokens);
|
|
expect(config.systemPrompt).toContain('# SELF-CONTAINED ADVISORY TURN');
|
|
expect(config.systemPrompt).toContain(
|
|
'unless the user named them or explicitly asked you to identify, recommend, or compare them',
|
|
);
|
|
expect(config.systemPrompt).not.toContain('# Context From Your Memory');
|
|
expect(config.systemPrompt).not.toContain('# Recalled Memories');
|
|
expect(events.some(event => event.data.name === 'auto_recall')).toBe(false);
|
|
|
|
const metrics = events.find(event => event.event === 'done')?.data.contextMetrics as Record<string, unknown>;
|
|
expect(metrics).toMatchObject({ packageMode: 'compact', toolSelectedCount: 0 });
|
|
});
|
|
|
|
it('keeps the exact Coordinator decomposition tool-free, recall-free, compact, and bounded', async () => {
|
|
const { persona, config, events, syntheticInputUpperBound } = await capturePersonaTurn('coordinator');
|
|
|
|
expect(config.tools).toEqual([]);
|
|
expect(config.messages).toEqual([{ role: 'user', content: persona.prompt }]);
|
|
expect(config.maxOutputTokens).toBeLessThanOrEqual(persona.maxOutputTokens);
|
|
expect(config.reasoning).toEqual({ enabled: true, effort: 'low' });
|
|
expect(syntheticInputUpperBound).toBeLessThan(persona.maxInputTokens);
|
|
expect(config.systemPrompt).toContain('# SELF-CONTAINED ADVISORY TURN');
|
|
expect(config.systemPrompt).not.toContain('# Context From Your Memory');
|
|
expect(config.systemPrompt).not.toContain('# Recalled Memories');
|
|
expect(config.onSkillDistillationFire).toBeUndefined();
|
|
expect(events.some(event => event.data.name === 'auto_recall')).toBe(false);
|
|
|
|
const metrics = events.find(event => event.event === 'done')?.data.contextMetrics as Record<string, unknown>;
|
|
expect(metrics).toMatchObject({ packageMode: 'compact', toolSelectedCount: 0 });
|
|
});
|
|
|
|
it('clears the Sonnet advisory reasoning policy when retrying on a non-Sonnet fallback', async () => {
|
|
const pilotConfig = new WaggleConfig(tmpDir);
|
|
pilotConfig.setFallbackModel('openrouter/openai/gpt-5.4');
|
|
pilotConfig.save();
|
|
const previousImplementation = testState.runAgentLoop.getMockImplementation();
|
|
const attempts: AgentLoopConfig[] = [];
|
|
|
|
testState.runAgentLoop.mockImplementation(async (config: AgentLoopConfig): Promise<AgentResponse> => {
|
|
attempts.push(config);
|
|
if (attempts.length === 1) {
|
|
throw new Error('Could not reach the model endpoint after 3 attempts (fetch failed).');
|
|
}
|
|
const response = renderVerifierReportEnvelope(CANONICAL_VERIFIER_REPORT);
|
|
config.onToken?.(response);
|
|
return {
|
|
content: response,
|
|
toolsUsed: [],
|
|
usage: { inputTokens: 10, outputTokens: 10 },
|
|
};
|
|
});
|
|
|
|
try {
|
|
const persona = PERSONA_CASES.find(item => item.id === 'data-engineer')!;
|
|
const response = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: persona.prompt,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: persona.id,
|
|
session: 'persona-advisory-reasoning-fallback',
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(attempts).toHaveLength(2);
|
|
expect(attempts[0]?.reasoning).toEqual({ enabled: true, effort: 'low' });
|
|
expect(attempts[1]?.model).toBe('openrouter/openai/gpt-5.4');
|
|
expect(attempts[1]?.reasoning).toBeUndefined();
|
|
} finally {
|
|
pilotConfig.clearFallbackModel();
|
|
pilotConfig.save();
|
|
if (previousImplementation) testState.runAgentLoop.mockImplementation(previousImplementation);
|
|
}
|
|
});
|
|
|
|
it('preserves prior chat for a referential read-only request instead of treating it as self-contained', async () => {
|
|
const session = 'persona-referential-history-budget';
|
|
const priorMessage = 'The supplied release note has two blockers: installer signing and proxy recovery.';
|
|
const first = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: priorMessage,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
expect(first.statusCode).toBe(200);
|
|
|
|
capturedConfig = null;
|
|
const currentMessage = 'Draft a plan based on what we discussed. Do not write files or execute code.';
|
|
const second = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: currentMessage,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
|
|
expect(second.statusCode).toBe(200);
|
|
expect(capturedConfig).not.toBeNull();
|
|
expect(capturedConfig!.messages).toEqual(expect.arrayContaining([
|
|
{ role: 'user', content: priorMessage },
|
|
{ role: 'user', content: currentMessage },
|
|
]));
|
|
expect(capturedConfig!.systemPrompt).not.toContain('# SELF-CONTAINED ADVISORY TURN');
|
|
expect(capturedConfig!.reasoning).toBeUndefined();
|
|
});
|
|
|
|
it.each([
|
|
['automated', { origin: 'automation' }],
|
|
['trusted', { autonomy: { level: 'trusted' } }],
|
|
] as const)('keeps %s advisory turns on the full operating contract', async (_label, mode) => {
|
|
const persona = PERSONA_CASES.find(item => item.id === 'data-engineer')!;
|
|
capturedConfig = null;
|
|
const response = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: persona.prompt,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: persona.id,
|
|
session: `persona-advisory-${_label}-full-contract`,
|
|
workspace: 'default',
|
|
...mode,
|
|
},
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(capturedConfig).not.toBeNull();
|
|
expect(capturedConfig!.systemPrompt).not.toContain('# SELF-CONTAINED ADVISORY TURN');
|
|
expect(capturedConfig!.reasoning).toBeUndefined();
|
|
const metrics = parseSse(response.body).find(event => event.event === 'done')?.data.contextMetrics as Record<string, unknown>;
|
|
expect(metrics.packageMode).toBe('full');
|
|
});
|
|
|
|
it.each([
|
|
'Summarize our previous decisions. Do not write files or execute code.',
|
|
'Explain what we decided. Do not write files or execute code.',
|
|
'Draft the agreed plan. Do not write files or execute code.',
|
|
'Explain "Do not search memory." Then explain what we decided. Do not write files or execute code.',
|
|
'Do not search memory; instead, search memory for our approved launch decision.',
|
|
'Review the memory database architecture, then summarize our previous decisions. Do not write files or execute code.',
|
|
'Do not hesitate to use my saved memory. Explain our previous decision. Do not write files or execute code.',
|
|
'Do not search the web, use my saved memory instead. Do not write files or execute code.',
|
|
'Do not search the web — use my saved memory instead. Do not write files or execute code.',
|
|
])('keeps first-turn owned context requests memory-capable: %s', async (message) => {
|
|
capturedConfig = null;
|
|
const response = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session: `persona-owned-context-${message.slice(0, 12).replace(/\W+/g, '-').toLowerCase()}`,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(capturedConfig).not.toBeNull();
|
|
expect(capturedConfig!.systemPrompt).not.toContain('# SELF-CONTAINED ADVISORY TURN');
|
|
expect(capturedConfig!.tools.map(tool => tool.name)).toContain('search_memory');
|
|
expect(capturedConfig!.systemPrompt).toContain(PERSISTED_IDENTITY_SENTINEL);
|
|
expect(capturedConfig!.systemPrompt).toContain(PERSISTED_PROFILE_SENTINEL);
|
|
expect(capturedConfig!.systemPrompt).toContain(PERSISTED_MEMORY_SENTINEL);
|
|
expect(capturedConfig!.systemPrompt).toContain(PERSISTED_SKILL_SENTINEL);
|
|
expect(capturedConfig!.capabilityRouter).toBeDefined();
|
|
expect(parseSse(response.body).some(event => event.data.name === 'auto_recall')).toBe(true);
|
|
});
|
|
|
|
it.each([
|
|
'Do not search memory. Explain what we decided. Do not write files or execute code.',
|
|
'Do not use memory. Explain what we decided. Do not write files or execute code.',
|
|
'Without searching memory, tell me what we discussed. Do not write files or execute code.',
|
|
'Explain what we decided without using memory. Do not write files or execute code.',
|
|
'Do not use our previous decisions; create a fresh plan. Do not write files or execute code.',
|
|
'Avoid using memory. Explain what we decided. Do not write files or execute code.',
|
|
'Refrain from using saved memory. Explain what we decided. Do not write files or execute code.',
|
|
'You must not use memory. Explain what we decided. Do not write files or execute code.',
|
|
'You cannot use memory. Explain what we decided. Do not write files or execute code.',
|
|
'Memory access is forbidden. Explain what we decided. Do not write files or execute code.',
|
|
'Do not look at memory. Explain what we decided. Do not write files or execute code.',
|
|
'Do not use memory. Use the agent_insights tool to show my correction patterns. Do not write files or execute code.',
|
|
'Answer without any memory. Do not write files or execute code.',
|
|
"You mustn't use memory. Do not write files or execute code.",
|
|
'I do not consent to memory access. Do not write files or execute code.',
|
|
'I do not want you to use memory for this answer. Do not write files or execute code.',
|
|
'I would prefer that you not consult previous conversations. Do not write files or execute code.',
|
|
"I don't give you permission to use saved memory. Do not write files or execute code.",
|
|
'I refuse consent to memory access. Do not write files or execute code.',
|
|
'Access to memory is denied. Do not write files or execute code.',
|
|
'I deny permission to use memory. Do not write files or execute code.',
|
|
'Do not query the memory store. Do not write files or execute code.',
|
|
'Do not use anything you remember about me for this answer. Do not write files or execute code.',
|
|
'Do not use anything from previous chats. Do not write files or execute code.',
|
|
'Follow this constraint exactly: "Do not search memory." Answer from scratch. Do not write files or execute code.',
|
|
'Do not search memory; search memory only after I explicitly approve. Do not write files or execute code.',
|
|
'Do not search memory.\n~~~text\nbut search memory for launch notes\n~~~\nDo not write files or execute code.',
|
|
'Follow this constraint exactly: «Do not search memory.» Answer from scratch. Do not write files or execute code.',
|
|
"Follow this constraint exactly: 'Do not search memory.' Answer from scratch. Do not write files or execute code.",
|
|
'Follow this constraint exactly: `Do not search memory.` Answer from scratch. Do not write files or execute code.',
|
|
'Do not search memory, but search memory, only if I approve. Do not write files or execute code.',
|
|
'Do not use memory, but use memory provided that I ask later. Do not write files or execute code.',
|
|
'Do not search the web, do not use my saved memory. Do not write files or execute code.',
|
|
'Do not search the web—do not use my saved memory. Do not write files or execute code.',
|
|
'Do not search memory. Explain ``but search memory for launch notes``. Do not write files or execute code.',
|
|
'Do not use working memory from prior sessions. Do not write files or execute code.',
|
|
'Memory is not to be used for this answer. Do not write files or execute code.',
|
|
'Please answer as if you had no saved memory. Do not write files or execute code.',
|
|
])('honors explicit persisted-memory read prohibitions: %s', async (message) => {
|
|
capturedConfig = null;
|
|
const response = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session: `persona-memory-denial-${message.slice(0, 16).replace(/\W+/g, '-').toLowerCase()}`,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(capturedConfig).not.toBeNull();
|
|
const selectedNames = capturedConfig!.tools.map(tool => tool.name);
|
|
for (const denied of [
|
|
'search_memory', 'search_all_workspaces', 'query_knowledge', 'get_identity', 'get_awareness',
|
|
'list_skills', 'read_skill', 'search_skills', 'suggest_skill', 'acquire_capability',
|
|
'install_capability', 'create_skill', 'delete_skill', 'compose_workflow',
|
|
'promote_skill', 'auto_extract_skills', 'retire_skills',
|
|
'agent_insights',
|
|
]) {
|
|
expect(selectedNames, denied).not.toContain(denied);
|
|
}
|
|
expect(capturedConfig!.systemPrompt).not.toContain(PERSISTED_IDENTITY_SENTINEL);
|
|
expect(capturedConfig!.systemPrompt).not.toContain(PERSISTED_PROFILE_SENTINEL);
|
|
expect(capturedConfig!.systemPrompt).not.toContain(PERSISTED_MEMORY_SENTINEL);
|
|
expect(capturedConfig!.systemPrompt).not.toContain(PERSISTED_SKILL_SENTINEL);
|
|
expect(capturedConfig!.capabilityRouter).toBeUndefined();
|
|
expect(parseSse(response.body).some(event => event.data.name === 'auto_recall')).toBe(false);
|
|
});
|
|
|
|
it('does not inspect plugin or MCP catalogs when persisted-memory reads are denied', async () => {
|
|
const pluginTools = vi.spyOn(server.agentState.pluginRuntimeManager, 'getAllTools');
|
|
const activePlugins = vi.spyOn(server.agentState.pluginRuntimeManager, 'getActive');
|
|
const mcpTools = vi.spyOn(server.agentState.mcpRuntime, 'getToolsForWorkspace');
|
|
const mcpStates = vi.spyOn(server.agentState.mcpRuntime, 'getServerStates');
|
|
try {
|
|
capturedConfig = null;
|
|
const response = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: 'Do not use memory. Explain what we decided without tools.',
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session: 'persona-memory-denial-catalogs',
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(capturedConfig).not.toBeNull();
|
|
expect(capturedConfig!.capabilityRouter).toBeUndefined();
|
|
expect(pluginTools).not.toHaveBeenCalled();
|
|
expect(activePlugins).not.toHaveBeenCalled();
|
|
expect(mcpTools).not.toHaveBeenCalled();
|
|
expect(mcpStates).not.toHaveBeenCalled();
|
|
} finally {
|
|
pluginTools.mockRestore();
|
|
activePlugins.mockRestore();
|
|
mcpTools.mockRestore();
|
|
mcpStates.mockRestore();
|
|
}
|
|
});
|
|
|
|
it('retains current-session conversation while excluding persisted-memory context', async () => {
|
|
const session = 'persona-memory-denial-current-session';
|
|
const priorMessage = 'For this chat only, the release codename is Amber Finch.';
|
|
const currentMessage = 'Continue from this conversation without consulting my saved memories. Do not write files or execute code.';
|
|
const first = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: priorMessage,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
expect(first.statusCode).toBe(200);
|
|
|
|
capturedConfig = null;
|
|
const second = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: currentMessage,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
|
|
expect(second.statusCode).toBe(200);
|
|
expect(capturedConfig).not.toBeNull();
|
|
expect(capturedConfig!.messages).toEqual(expect.arrayContaining([
|
|
{ role: 'user', content: priorMessage },
|
|
{ role: 'user', content: currentMessage },
|
|
]));
|
|
expect(capturedConfig!.systemPrompt).not.toContain(PERSISTED_IDENTITY_SENTINEL);
|
|
expect(capturedConfig!.systemPrompt).not.toContain(PERSISTED_PROFILE_SENTINEL);
|
|
expect(capturedConfig!.systemPrompt).not.toContain(PERSISTED_MEMORY_SENTINEL);
|
|
expect(parseSse(second.body).some(event => event.data.name === 'auto_recall')).toBe(false);
|
|
});
|
|
|
|
it('loads saved memory while excluding current-session history when both are requested', async () => {
|
|
const session = 'persona-independent-history-memory-boundaries';
|
|
const priorMessage = 'For this chat only, the private marker is HISTORY-ORCHID.';
|
|
const currentMessage = 'Do not use conversation history; use my saved memory. The current marker is MEMORY-JADE. Do not write files or execute code.';
|
|
persistMessage(tmpDir, 'default', session, { role: 'user', content: priorMessage });
|
|
|
|
capturedConfig = null;
|
|
const sessionFile = path.resolve(
|
|
tmpDir, 'workspaces', 'default', 'sessions', `${session}.jsonl`,
|
|
);
|
|
const readSpy = vi.spyOn(fs, 'readFileSync');
|
|
let second!: Awaited<ReturnType<typeof injectWithAuth>>;
|
|
try {
|
|
second = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: currentMessage,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
expect(readSpy.mock.calls.some(
|
|
([file]) => typeof file === 'string' && path.resolve(file) === sessionFile,
|
|
)).toBe(false);
|
|
} finally {
|
|
readSpy.mockRestore();
|
|
}
|
|
|
|
expect(second.statusCode).toBe(200);
|
|
expect(capturedConfig).not.toBeNull();
|
|
expect(capturedConfig!.messages).toEqual([
|
|
{ role: 'user', content: currentMessage },
|
|
]);
|
|
expect(JSON.stringify(capturedConfig)).not.toContain('HISTORY-ORCHID');
|
|
expect(capturedConfig!.tools.map(tool => tool.name)).toContain('search_memory');
|
|
expect(capturedConfig!.systemPrompt).toContain(PERSISTED_MEMORY_SENTINEL);
|
|
expect(parseSse(second.body).some(event => event.data.name === 'auto_recall')).toBe(true);
|
|
});
|
|
|
|
it('keeps a saved-memory-denied turn out of trace, audit, optimizer, and activity retention', async () => {
|
|
const session = 'privacy-derived-sink-boundary';
|
|
const promptMarker = 'PRIVATE_TRACE_PROMPT_CINNABAR_20260808';
|
|
const toolInputMarker = 'PRIVATE_TOOL_INPUT_CERULEAN_20260808';
|
|
const toolResultMarker = 'PRIVATE_TOOL_RESULT_AMBER_20260808';
|
|
const responseMarker = 'PRIVATE_RESPONSE_VIOLET_20260808';
|
|
const teamId = 'privacy-derived-sink-team';
|
|
const teamServerUrl = 'https://93.184.216.34';
|
|
const auditUrl = `${teamServerUrl}/api/teams/${teamId}/audit`;
|
|
const workspaceBefore = server.workspaceManager.get(collaborationWorkspaceId)!;
|
|
const waggleConfig = new WaggleConfig(tmpDir);
|
|
const teamServerBefore = waggleConfig.getTeamServer();
|
|
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(
|
|
new Response(JSON.stringify([]), {
|
|
status: 200,
|
|
headers: { 'Content-Type': 'application/json' },
|
|
}),
|
|
);
|
|
|
|
try {
|
|
server.workspaceManager.update(collaborationWorkspaceId, {
|
|
teamId,
|
|
teamServerUrl,
|
|
teamRole: 'member',
|
|
});
|
|
waggleConfig.setTeamServer({ url: teamServerUrl, token: 'privacy-test-token' });
|
|
waggleConfig.save();
|
|
|
|
testState.runAgentLoop.mockImplementationOnce(async (config: AgentLoopConfig) => {
|
|
capturedConfig = config;
|
|
const input = { path: toolInputMarker };
|
|
config.onToolUse?.('read_file', input);
|
|
config.onToolResult?.('read_file', input, toolResultMarker);
|
|
config.traceRecording?.recorder.recordToolCall(config.traceRecording.handle, {
|
|
tool: 'read_file',
|
|
args: input,
|
|
result: toolResultMarker,
|
|
ok: true,
|
|
durationMs: 1,
|
|
timestamp: new Date().toISOString(),
|
|
});
|
|
config.onToken?.(responseMarker);
|
|
return {
|
|
content: responseMarker,
|
|
toolsUsed: ['read_file'],
|
|
usage: { inputTokens: 1, outputTokens: 1 },
|
|
};
|
|
});
|
|
|
|
const response = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: `Do not use saved memory. ${promptMarker}. Read one current workspace file only.`,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: collaborationWorkspaceId,
|
|
},
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(response.body).toContain(toolResultMarker);
|
|
expect(response.body).toContain(responseMarker);
|
|
expect(capturedConfig?.traceRecording).toBeUndefined();
|
|
expect(testState.optimizerExpand).not.toHaveBeenCalled();
|
|
|
|
const traceProjection = JSON.stringify(server.traceStore.queryParsed({ sessionId: session }));
|
|
const auditProjection = JSON.stringify(getAuditDb(tmpDir).prepare(
|
|
'SELECT input, output FROM audit_events WHERE session_id = ? ORDER BY id',
|
|
).all(session));
|
|
const legacySignals = await injectWithAuth(server, {
|
|
method: 'GET',
|
|
url: '/api/waggle/signals?limit=500',
|
|
});
|
|
expect(legacySignals.statusCode).toBe(200);
|
|
const activityProjection = legacySignals.body;
|
|
|
|
await vi.waitFor(() => {
|
|
expect(fetchSpy.mock.calls.filter(([url]) => String(url) === auditUrl)).toHaveLength(2);
|
|
});
|
|
const remoteAuditProjection = JSON.stringify(fetchSpy.mock.calls
|
|
.filter(([url]) => String(url) === auditUrl)
|
|
.map(([, options]) => options?.body));
|
|
|
|
for (const projection of [traceProjection, auditProjection, activityProjection, remoteAuditProjection]) {
|
|
for (const marker of [promptMarker, toolInputMarker, toolResultMarker, responseMarker]) {
|
|
expect(projection, marker).not.toContain(marker);
|
|
}
|
|
}
|
|
expect(traceProjection).toContain('Not retained');
|
|
expect(auditProjection).toContain('Not retained');
|
|
expect(activityProjection).toContain('Not retained');
|
|
expect(remoteAuditProjection).toContain('Not retained');
|
|
} finally {
|
|
server.workspaceManager.update(collaborationWorkspaceId, {
|
|
teamId: workspaceBefore.teamId,
|
|
teamServerUrl: workspaceBefore.teamServerUrl,
|
|
teamRole: workspaceBefore.teamRole,
|
|
});
|
|
const restoreConfig = new WaggleConfig(tmpDir);
|
|
if (teamServerBefore) restoreConfig.setTeamServer(teamServerBefore);
|
|
else restoreConfig.clearTeamServer();
|
|
restoreConfig.save();
|
|
fetchSpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it.each([
|
|
{ mode: 'failure' as const, session: 'privacy-derived-sink-failure' },
|
|
{ mode: 'cancellation' as const, session: 'privacy-derived-sink-cancellation' },
|
|
])('keeps a saved-memory-denied $mode out of trace, audit, optimizer, and activity retention', async ({ mode, session }) => {
|
|
const promptMarker = `PRIVATE_${mode.toUpperCase()}_PROMPT_CINNABAR_20260808`;
|
|
const toolInputMarker = `PRIVATE_${mode.toUpperCase()}_TOOL_INPUT_CERULEAN_20260808`;
|
|
const toolResultMarker = `PRIVATE_${mode.toUpperCase()}_TOOL_RESULT_AMBER_20260808`;
|
|
const outcomeMarker = `PRIVATE_${mode.toUpperCase()}_OUTCOME_VIOLET_20260808`;
|
|
|
|
testState.runAgentLoop.mockImplementationOnce(async (config: AgentLoopConfig) => {
|
|
capturedConfig = config;
|
|
const input = { path: toolInputMarker };
|
|
config.onToolUse?.('read_file', input);
|
|
config.onToolResult?.('read_file', input, toolResultMarker);
|
|
if (mode === 'failure') throw new Error(outcomeMarker);
|
|
Object.defineProperty(config.signal!, 'aborted', {
|
|
value: true,
|
|
configurable: true,
|
|
});
|
|
return {
|
|
content: outcomeMarker,
|
|
toolsUsed: ['read_file'],
|
|
usage: { inputTokens: 1, outputTokens: 1 },
|
|
};
|
|
});
|
|
|
|
const response = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: `Do not use saved memory. ${promptMarker}. Read one current workspace file only.`,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: collaborationWorkspaceId,
|
|
},
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(capturedConfig?.traceRecording).toBeUndefined();
|
|
expect(testState.optimizerExpand).not.toHaveBeenCalled();
|
|
|
|
const traceProjection = JSON.stringify(server.traceStore.queryParsed({ sessionId: session }));
|
|
const auditProjection = JSON.stringify(getAuditDb(tmpDir).prepare(
|
|
'SELECT input, output FROM audit_events WHERE session_id = ? ORDER BY id',
|
|
).all(session));
|
|
const legacySignals = await injectWithAuth(server, {
|
|
method: 'GET',
|
|
url: '/api/waggle/signals?limit=500',
|
|
});
|
|
expect(legacySignals.statusCode).toBe(200);
|
|
|
|
for (const projection of [traceProjection, auditProjection, legacySignals.body]) {
|
|
for (const marker of [promptMarker, toolInputMarker, toolResultMarker, outcomeMarker]) {
|
|
expect(projection, marker).not.toContain(marker);
|
|
}
|
|
expect(projection).toContain('Not retained');
|
|
}
|
|
});
|
|
|
|
it.each([
|
|
{
|
|
label: 'saved history is explicitly denied',
|
|
session: 'persona-saved-history-denial',
|
|
currentMessage: 'Do not use saved history. The private one-turn marker is ONYX. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'ONYX',
|
|
},
|
|
{
|
|
label: 'prior history of this chat is explicitly denied',
|
|
session: 'persona-prior-chat-history-denial',
|
|
currentMessage: 'Do not use prior history of this chat. The private one-turn marker is QUARTZ. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'QUARTZ',
|
|
},
|
|
{
|
|
label: "this chat's prior history is explicitly denied",
|
|
session: 'persona-possessive-chat-history-denial',
|
|
currentMessage: "Do not use this chat's prior history. The private one-turn marker is AMBER. Answer from scratch. Do not write files or execute code.",
|
|
currentMarker: 'AMBER',
|
|
},
|
|
{
|
|
label: 'history from earlier in this chat is explicitly denied',
|
|
session: 'persona-earlier-chat-history-denial',
|
|
currentMessage: 'Do not use history from earlier in this chat. The private one-turn marker is COBALT. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'COBALT',
|
|
},
|
|
{
|
|
label: 'history in this conversation is explicitly denied',
|
|
session: 'persona-history-in-conversation-denial',
|
|
currentMessage: 'Do not use the history in this conversation. The private one-turn marker is JADE. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'JADE',
|
|
},
|
|
{
|
|
label: 'conversation so far is explicitly ignored',
|
|
session: 'persona-conversation-so-far-ignore',
|
|
currentMessage: 'Ignore the conversation so far and answer fresh. The private one-turn marker is INDIGO. Do not write files or execute code.',
|
|
currentMarker: 'INDIGO',
|
|
},
|
|
{
|
|
label: 'conversation history access is explicitly denied',
|
|
session: 'persona-conversation-history-access-denied',
|
|
currentMessage: 'Access to conversation history is denied for this turn. The private one-turn marker is VIOLET. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'VIOLET',
|
|
},
|
|
{
|
|
label: 'conversation history permission is explicitly absent',
|
|
session: 'persona-conversation-history-no-permission',
|
|
currentMessage: 'You do not have permission to use conversation history. The private one-turn marker is SAPPHIRE. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'SAPPHIRE',
|
|
},
|
|
{
|
|
label: 'conversation history consent is explicitly withdrawn',
|
|
session: 'persona-conversation-history-consent-withdrawn',
|
|
currentMessage: 'I no longer consent to use conversation history. The private one-turn marker is RUBY. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'RUBY',
|
|
},
|
|
{
|
|
label: 'conversation history consent is explicitly absent',
|
|
session: 'persona-conversation-history-no-consent',
|
|
currentMessage: 'You do not have my consent to use conversation history. The private one-turn marker is AMETHYST. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'AMETHYST',
|
|
},
|
|
{
|
|
label: 'conversation history consent is explicitly withheld',
|
|
session: 'persona-conversation-history-withheld-consent',
|
|
currentMessage: 'I withhold consent to use conversation history. The private one-turn marker is GARNET. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'GARNET',
|
|
},
|
|
{
|
|
label: 'chat transcript so far is explicitly denied',
|
|
session: 'persona-chat-transcript-so-far-denial',
|
|
currentMessage: 'Do not use the chat transcript so far. The private one-turn marker is OPAL. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'OPAL',
|
|
},
|
|
{
|
|
label: 'messages earlier in this chat are explicitly denied',
|
|
session: 'persona-messages-earlier-in-chat-denial',
|
|
currentMessage: 'Do not use the messages earlier in this chat. The private one-turn marker is PEARL. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'PEARL',
|
|
},
|
|
{
|
|
label: 'a later yet-clause explicitly denies conversation history',
|
|
session: 'persona-conversation-history-yet-denial',
|
|
currentMessage: 'Use conversation history. Yet do not use it. The private one-turn marker is TURQUOISE. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'TURQUOISE',
|
|
},
|
|
{
|
|
label: 'an operative quoted constraint denies conversation history',
|
|
session: 'persona-conversation-history-operative-quote',
|
|
currentMessage: 'Follow this constraint exactly: "Do not use conversation history." The private one-turn marker is EMERALD. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'EMERALD',
|
|
},
|
|
{
|
|
label: 'an operative quoted rule denies conversation history',
|
|
session: 'persona-conversation-history-operative-rule',
|
|
currentMessage: 'Follow this rule: "Do not use conversation history." The private one-turn marker is PERIDOT. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'PERIDOT',
|
|
},
|
|
{
|
|
label: 'an operative quoted policy denies conversation history',
|
|
session: 'persona-conversation-history-operative-policy',
|
|
currentMessage: 'Enforce this policy: "Do not use conversation history." The private one-turn marker is MOONSTONE. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'MOONSTONE',
|
|
},
|
|
{
|
|
label: 'an except-clause overrides earlier conversation-history permission',
|
|
session: 'persona-conversation-history-except-denial',
|
|
currentMessage: 'Use conversation history, except do not use it for this answer. The private one-turn marker is ONYX. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'ONYX',
|
|
},
|
|
{
|
|
label: 'conversation history consent is explicitly not given',
|
|
session: 'persona-conversation-history-consent-not-given',
|
|
currentMessage: 'I do not give consent for use of conversation history. The private one-turn marker is JADE. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'JADE',
|
|
},
|
|
{
|
|
label: 'a later no-clause explicitly denies conversation history',
|
|
session: 'persona-conversation-history-no-denial',
|
|
currentMessage: 'Use conversation history. No, do not use it. The private one-turn marker is CORAL. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'CORAL',
|
|
},
|
|
{
|
|
label: 'conversation history consent was never given',
|
|
session: 'persona-conversation-history-never-given-consent',
|
|
currentMessage: 'Consent to use conversation history was never given. The private one-turn marker is CITRINE. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'CITRINE',
|
|
},
|
|
{
|
|
label: 'there is no consent to use conversation history',
|
|
session: 'persona-conversation-history-no-consent-exists',
|
|
currentMessage: 'There is no consent to use conversation history. The private one-turn marker is AMBER. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'AMBER',
|
|
},
|
|
{
|
|
label: 'a later scratch-that clause denies conversation history by reference',
|
|
session: 'persona-conversation-history-scratch-that-denial',
|
|
currentMessage: 'Use conversation history. Scratch that. Answer without it. The private one-turn marker is ZIRCON. Do not write files or execute code.',
|
|
currentMarker: 'ZIRCON',
|
|
},
|
|
{
|
|
label: 'a slash-command envelope explicitly denies saved history',
|
|
session: 'persona-conversation-history-slash-envelope-denial',
|
|
currentMessage: '/research ATLAS please do not use saved history. The private one-turn marker is BERYL. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'BERYL',
|
|
},
|
|
{
|
|
label: 'an operative quoted new rule denies conversation history',
|
|
session: 'persona-conversation-history-new-rule-denial',
|
|
currentMessage: 'Use conversation history. New rule: «Do not use it.» The private one-turn marker is SPINEL. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'SPINEL',
|
|
},
|
|
{
|
|
label: 'passive exclusion keeps prior messages out',
|
|
session: 'persona-conversation-history-passive-exclusion',
|
|
currentMessage: 'Prior messages should be kept out of this answer. The private one-turn marker is THISTLE. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'THISTLE',
|
|
},
|
|
{
|
|
label: 'a leading answer scope denies conversation history',
|
|
session: 'persona-conversation-history-leading-scope-denial',
|
|
currentMessage: 'For this answer, do not use conversation history. The private one-turn marker is MARIGOLD. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'MARIGOLD',
|
|
},
|
|
{
|
|
label: 'deferred approval denies conversation history now',
|
|
session: 'persona-conversation-history-deferred-approval',
|
|
currentMessage: 'Use conversation history only if I explicitly approve. The private one-turn marker is COBALT. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'COBALT',
|
|
},
|
|
{
|
|
label: 'a take-that-back reversal denies conversation history',
|
|
session: 'persona-conversation-history-take-back-denial',
|
|
currentMessage: 'Use conversation history. I take that back; do not use it. The private one-turn marker is GARNET. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'GARNET',
|
|
},
|
|
{
|
|
label: 'natural slash-command arguments preserve a trailing denial',
|
|
session: 'persona-conversation-history-natural-slash-denial',
|
|
currentMessage: '/research quantum computing please do not use saved history. The private one-turn marker is OPAL. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'OPAL',
|
|
},
|
|
{
|
|
label: 'a leading task scope without a comma denies conversation history',
|
|
session: 'persona-conversation-history-task-scope-denial',
|
|
currentMessage: 'For this task do not use conversation history. The private one-turn marker is AZURE. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'AZURE',
|
|
},
|
|
{
|
|
label: 'direct passive exclusion denies prior messages',
|
|
session: 'persona-conversation-history-direct-passive-denial',
|
|
currentMessage: 'Prior messages are excluded from this answer. The private one-turn marker is IVORY. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'IVORY',
|
|
},
|
|
{
|
|
label: 'provided consent is explicitly absent',
|
|
session: 'persona-conversation-history-provided-consent-denial',
|
|
currentMessage: 'I have not provided consent for you to use conversation history. The private one-turn marker is OCHRE. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'OCHRE',
|
|
},
|
|
{
|
|
label: 'approval upon a future event denies conversation history now',
|
|
session: 'persona-conversation-history-upon-approval-denial',
|
|
currentMessage: 'Use conversation history only upon my explicit approval. The private one-turn marker is SIENNA. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'SIENNA',
|
|
},
|
|
{
|
|
label: 'a cancel-that-request reversal denies conversation history',
|
|
session: 'persona-conversation-history-cancel-request-denial',
|
|
currentMessage: 'Use conversation history. Cancel that request and answer without it. The private one-turn marker is UMBER. Do not write files or execute code.',
|
|
currentMarker: 'UMBER',
|
|
},
|
|
{
|
|
label: 'a real slash-command task preserves its trailing denial',
|
|
session: 'persona-conversation-history-real-slash-task-denial',
|
|
currentMessage: '/research write a report and please do not use conversation history. The private one-turn marker is LILAC. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'LILAC',
|
|
},
|
|
{
|
|
label: 'prior conversation context is explicitly denied',
|
|
session: 'persona-prior-conversation-context-denial',
|
|
currentMessage: 'Please do not use prior conversation context. The private one-turn marker is MINT. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'MINT',
|
|
},
|
|
{
|
|
label: 'carrying earlier context forward is explicitly denied',
|
|
session: 'persona-carry-context-forward-denial',
|
|
currentMessage: 'Do not carry context forward from earlier turns. The private one-turn marker is PLUM. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'PLUM',
|
|
},
|
|
{
|
|
label: 'incorporating previous messages is explicitly denied',
|
|
session: 'persona-incorporate-previous-messages-denial',
|
|
currentMessage: 'Do not incorporate anything from previous messages. The private one-turn marker is TEAL. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'TEAL',
|
|
},
|
|
{
|
|
label: 'a polite negative request denies conversation history',
|
|
session: 'persona-polite-negative-history-denial',
|
|
currentMessage: 'Can you please not use conversation history. The private one-turn marker is MAUVE. Answer from scratch. Do not write files or execute code.',
|
|
currentMarker: 'MAUVE',
|
|
},
|
|
])('excludes prior session turns when $label', async ({ session, currentMessage, currentMarker }) => {
|
|
const priorMessage = 'Do not use saved memory. For this chat only, the private marker is ORCHID.';
|
|
const deniedMessage = `${currentMessage} Do not use saved memory.`;
|
|
const first = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: priorMessage,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
expect(first.statusCode).toBe(200);
|
|
const stateKeySuffix = chatSessionStateKey('', session);
|
|
const stateKey = [...server.agentState.sessionHistories.keys()]
|
|
.find(key => key.endsWith(stateKeySuffix));
|
|
expect(stateKey).toBeDefined();
|
|
server.agentState.sessionHistories.delete(stateKey!);
|
|
expect(server.agentState.sessionHistories.has(stateKey!)).toBe(false);
|
|
|
|
capturedConfig = null;
|
|
const sessionFile = path.resolve(
|
|
tmpDir, 'workspaces', 'default', 'sessions', `${session}.jsonl`,
|
|
);
|
|
const readSpy = vi.spyOn(fs, 'readFileSync');
|
|
let second!: Awaited<ReturnType<typeof injectWithAuth>>;
|
|
try {
|
|
second = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: deniedMessage,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
const loadedDeniedTranscript = readSpy.mock.calls.some(
|
|
([file]) => typeof file === 'string' && path.resolve(file) === sessionFile,
|
|
);
|
|
expect(loadedDeniedTranscript).toBe(false);
|
|
} finally {
|
|
readSpy.mockRestore();
|
|
}
|
|
|
|
expect(second.statusCode).toBe(200);
|
|
expect(capturedConfig).not.toBeNull();
|
|
expect(capturedConfig!.messages).toHaveLength(1);
|
|
expect(capturedConfig!.messages[0]?.role).toBe('user');
|
|
expect(server.agentState.sessionHistories.has(stateKey!)).toBe(false);
|
|
expect(capturedConfig!.systemPrompt).not.toContain('Continuing conversation');
|
|
expect(capturedConfig!.systemPrompt).not.toContain('previous messages in context');
|
|
expect(capturedConfig!.systemPrompt).not.toContain('full conversation history above');
|
|
if (currentMessage.startsWith('/research')) {
|
|
expect(capturedConfig!.messages[0]?.content).toContain(currentMarker);
|
|
expect(capturedConfig!.messages[0]?.content)
|
|
.toMatch(/do not use (?:saved|conversation) history/i);
|
|
} else {
|
|
expect(capturedConfig!.messages[0]?.content).toBe(deniedMessage);
|
|
}
|
|
expect(JSON.stringify(capturedConfig)).not.toContain('ORCHID');
|
|
expect(parseSse(second.body).some(event => event.data.name === 'auto_recall')).toBe(false);
|
|
|
|
capturedConfig = null;
|
|
const third = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: 'Now answer normally. Do not write files or execute code.',
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
expect(third.statusCode).toBe(200);
|
|
expect(capturedConfig).not.toBeNull();
|
|
expect(JSON.stringify(capturedConfig)).not.toContain(currentMarker);
|
|
});
|
|
|
|
it('keeps prior history available for descriptive slash-command policy wording', async () => {
|
|
const session = 'persona-descriptive-slash-policy-control';
|
|
const priorMessage = 'For this chat only, the private marker is ORCHID-DESCRIPTIVE-CONTROL.';
|
|
const first = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: priorMessage,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
expect(first.statusCode).toBe(200);
|
|
|
|
capturedConfig = null;
|
|
const second = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: '/research draft wording: do not use conversation history',
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
expect(second.statusCode).toBe(200);
|
|
expect(capturedConfig).not.toBeNull();
|
|
expect(JSON.stringify(capturedConfig)).toContain('ORCHID-DESCRIPTIVE-CONTROL');
|
|
});
|
|
|
|
it('does not delete a prior failed pair when a retry turn denies saved history', async () => {
|
|
const session = 'persona-saved-history-denied-retry';
|
|
const failedMarker = 'FAILED-PAIR-ORCHID';
|
|
testState.runAgentLoop.mockRejectedValueOnce(new Error('retry boundary failure'));
|
|
|
|
const failed = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: `${failedMarker} trigger a deterministic generation failure.`,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
expect(failed.statusCode).toBe(200);
|
|
expect(failed.body).toContain('retry boundary failure');
|
|
|
|
capturedConfig = null;
|
|
const deniedRetry = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: '/research BERYL do not use saved history. Answer from scratch. Do not write files or execute code.',
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: 'default',
|
|
retry: true,
|
|
},
|
|
});
|
|
expect(deniedRetry.statusCode).toBe(200);
|
|
expect(capturedConfig).not.toBeNull();
|
|
expect(JSON.stringify(capturedConfig)).not.toContain(failedMarker);
|
|
|
|
capturedConfig = null;
|
|
const followUp = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: 'Now answer normally. Do not write files or execute code.',
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
expect(followUp.statusCode).toBe(200);
|
|
expect(capturedConfig).not.toBeNull();
|
|
expect(JSON.stringify(capturedConfig)).toContain(failedMarker);
|
|
});
|
|
|
|
it('does not retain a terminal command response for a saved-history-denied turn', async () => {
|
|
const session = 'persona-saved-history-command-denial';
|
|
const denied = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: '/ONYX Do not use saved history.',
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
expect(denied.statusCode).toBe(200);
|
|
expect(denied.body).toContain('ONYX');
|
|
|
|
capturedConfig = null;
|
|
const followUp = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: 'Now answer normally. Do not write files or execute code.',
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
expect(followUp.statusCode).toBe(200);
|
|
expect(capturedConfig).not.toBeNull();
|
|
expect(JSON.stringify(capturedConfig)).not.toContain('ONYX');
|
|
});
|
|
|
|
it.each([
|
|
{
|
|
label: 'AI-required command error',
|
|
message: '/research ATLAS Do not use saved history.',
|
|
responseFragment: '/research requires AI',
|
|
},
|
|
{
|
|
label: 'no-model setup response',
|
|
message: 'TOPAZ Do not use saved history.',
|
|
responseFragment: 'No AI model is ready',
|
|
},
|
|
])('does not retain the $label for a saved-history-denied turn', async ({ message, responseFragment }) => {
|
|
const session = `persona-saved-history-${responseFragment.replace(/\W+/g, '-').toLowerCase()}`;
|
|
const previousProvider = server.agentState.llmProvider;
|
|
const previousLiteLlmUrl = server.localConfig.litellmUrl;
|
|
try {
|
|
server.agentState.llmProvider = {
|
|
provider: 'none',
|
|
health: 'unavailable',
|
|
detail: 'Test: force setup-required mode',
|
|
checkedAt: new Date().toISOString(),
|
|
};
|
|
server.localConfig.litellmUrl = 'http://127.0.0.1:1';
|
|
|
|
const denied = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
expect(denied.statusCode).toBe(200);
|
|
expect(denied.body).toContain(responseFragment);
|
|
} finally {
|
|
server.agentState.llmProvider = previousProvider;
|
|
server.localConfig.litellmUrl = previousLiteLlmUrl;
|
|
}
|
|
|
|
capturedConfig = null;
|
|
const followUp = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: 'Now answer normally. Do not write files or execute code.',
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'data-engineer',
|
|
session,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
expect(followUp.statusCode).toBe(200);
|
|
expect(capturedConfig).not.toBeNull();
|
|
expect(capturedConfig!.messages.some(item => item.content.includes(responseFragment))).toBe(false);
|
|
});
|
|
|
|
it.each(['installed', 'sync', 'install private-capability'])(
|
|
'blocks persisted marketplace subcommand %s when memory is disabled',
|
|
async (subcommand) => {
|
|
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(
|
|
new Response(JSON.stringify({ packages: [], total: 0, message: 'unexpected' }), {
|
|
status: 200,
|
|
headers: { 'Content-Type': 'application/json' },
|
|
}),
|
|
);
|
|
try {
|
|
const response = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: `/marketplace ${subcommand} - do not use memory`,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
session: `marketplace-memory-denial-${subcommand.replace(/\W+/g, '-')}`,
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(response.body).toContain('Persisted marketplace state is disabled for this turn.');
|
|
expect(fetchSpy.mock.calls.some(
|
|
([url]) => String(url).includes('/api/marketplace/'),
|
|
)).toBe(false);
|
|
} finally {
|
|
fetchSpy.mockRestore();
|
|
}
|
|
},
|
|
);
|
|
|
|
it('keeps public marketplace search available when persisted memory is disabled', async () => {
|
|
const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(
|
|
new Response(JSON.stringify({ packages: [], total: 0 }), {
|
|
status: 200,
|
|
headers: { 'Content-Type': 'application/json' },
|
|
}),
|
|
);
|
|
try {
|
|
const response = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: '/marketplace search research - do not use memory',
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
session: 'marketplace-public-search-memory-denial',
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(fetchSpy).toHaveBeenCalledTimes(1);
|
|
expect(String(fetchSpy.mock.calls[0][0])).toContain('/api/marketplace/search');
|
|
} finally {
|
|
fetchSpy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it('keeps memory-denied child tools and completion results out of persisted memory', async () => {
|
|
const parentTask = 'PRIVATE_PARENT_CHILD_TASK_20260808';
|
|
const childName = 'PRIVATE_CHILD_NAME_20260808';
|
|
const childTask = 'PRIVATE_CHILD_TASK_20260808';
|
|
const childResult = 'PRIVATE_CHILD_RESULT_20260808';
|
|
const personalMind = server.multiMind.personal;
|
|
const workspaceMind = server.mindCache.getOrOpen(collaborationWorkspaceId);
|
|
expect(workspaceMind).toBeDefined();
|
|
const frameCount = (mind: MindDB) => (
|
|
mind.getDatabase().prepare('SELECT COUNT(*) AS count FROM memory_frames').get() as { count: number }
|
|
).count;
|
|
const personalBefore = frameCount(personalMind);
|
|
const workspaceBefore = frameCount(workspaceMind!);
|
|
const loopConfigs: AgentLoopConfig[] = [];
|
|
let childOutput = '';
|
|
const statusEvents: unknown[] = [];
|
|
const onStatus = (event: unknown) => statusEvents.push(event);
|
|
server.eventBus.on('subagent_status', onStatus);
|
|
const deniedChildTools = [
|
|
'search_memory', 'search_all_workspaces', 'query_knowledge', 'get_identity',
|
|
'get_awareness', 'read_other_workspace', 'save_memory', 'add_task',
|
|
'correct_knowledge', 'list_skills', 'read_skill', 'search_skills',
|
|
'suggest_skill', 'acquire_capability', 'install_capability', 'create_skill',
|
|
'delete_skill', 'promote_skill', 'auto_extract_skills', 'retire_skills',
|
|
'agent_insights', 'compose_workflow', 'list_agents', 'get_agent_result',
|
|
];
|
|
|
|
testState.runAgentLoop.mockImplementation(async (config: AgentLoopConfig) => {
|
|
loopConfigs.push(config);
|
|
const spawn = config.tools.find(tool => tool.name === 'spawn_agent');
|
|
if (spawn) {
|
|
childOutput = String(await spawn.execute({
|
|
name: childName,
|
|
role: 'custom',
|
|
task: childTask,
|
|
tools: ['read_file', ...deniedChildTools],
|
|
}));
|
|
return {
|
|
content: 'Parent completed after delegation',
|
|
toolsUsed: ['spawn_agent'],
|
|
usage: { inputTokens: 1, outputTokens: 1 },
|
|
};
|
|
}
|
|
return {
|
|
content: childResult,
|
|
toolsUsed: config.tools.map(tool => tool.name),
|
|
usage: { inputTokens: 1, outputTokens: 1 },
|
|
};
|
|
});
|
|
|
|
try {
|
|
const response = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: `Do not use saved memory. Delegate a review of current workspace files only. Correlation: ${parentTask}.`,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'general-purpose',
|
|
session: 'memory-denied-child-persistence',
|
|
workspace: collaborationWorkspaceId,
|
|
},
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(response.body).toContain('Parent completed after delegation');
|
|
expect(loopConfigs).toHaveLength(2);
|
|
expect(childOutput).toContain(childName);
|
|
expect(childOutput).toContain(childResult);
|
|
expect(loopConfigs[0].modelSpendTraceId).toEqual(expect.any(Number));
|
|
expect(loopConfigs[1].modelSpendBudget).toBe(loopConfigs[0].modelSpendBudget);
|
|
expect(loopConfigs[1].modelSpendTraceId).toBe(loopConfigs[0].modelSpendTraceId);
|
|
expect(loopConfigs[1].spendWorkspaceId).toBe(loopConfigs[0].spendWorkspaceId);
|
|
expect(loopConfigs[1].modelSpendBillingClass).toBe('priced');
|
|
const parentToolNames = loopConfigs[0].tools.map(tool => tool.name);
|
|
expect(parentToolNames).toContain('spawn_agent');
|
|
expect(parentToolNames).not.toContain('list_agents');
|
|
expect(parentToolNames).not.toContain('get_agent_result');
|
|
const childToolNames = loopConfigs[1].tools.map(tool => tool.name);
|
|
expect(childToolNames).toContain('read_file');
|
|
for (const denied of deniedChildTools) {
|
|
expect(childToolNames, denied).not.toContain(denied);
|
|
}
|
|
expect(frameCount(personalMind)).toBe(personalBefore);
|
|
expect(frameCount(workspaceMind!)).toBe(workspaceBefore);
|
|
const registry = fs.readFileSync(path.join(tmpDir, 'agent-runs.json'), 'utf8');
|
|
for (const privateContent of [parentTask, childName, childTask, childResult]) {
|
|
expect(registry, privateContent).not.toContain(privateContent);
|
|
}
|
|
const worker = server.agentRunRegistry.list({
|
|
source: 'chat_subagent', workspaceId: collaborationWorkspaceId, limit: 1_000,
|
|
}).find(run => run.kind === 'worker'
|
|
&& run.result?.sessionId === 'memory-denied-child-persistence');
|
|
expect(worker?.status).toBe('completed');
|
|
expect(worker?.result?.summary).toContain('Not retained');
|
|
expect(worker?.metrics?.outputTokens).toBe(1);
|
|
expect(worker && server.agentRunRegistry.get(worker.roomId)?.task).toContain('Not retained');
|
|
const danceProjection = JSON.stringify(server.signalBus?.query({
|
|
teamId: `room::${worker!.roomId}`,
|
|
limit: 1_000,
|
|
}) ?? []);
|
|
const statusProjection = JSON.stringify(statusEvents);
|
|
for (const privateContent of [parentTask, childName, childTask, childResult]) {
|
|
expect(danceProjection, privateContent).not.toContain(privateContent);
|
|
expect(statusProjection, privateContent).not.toContain(privateContent);
|
|
}
|
|
expect(danceProjection).toContain('Not retained');
|
|
expect(statusProjection).toContain('Not retained');
|
|
} finally {
|
|
server.eventBus.off('subagent_status', onStatus);
|
|
testState.runAgentLoop.mockImplementation(defaultRunAgentLoop);
|
|
}
|
|
});
|
|
|
|
it('keeps memory-denied child tasks and failures out of the durable run registry', async () => {
|
|
const parentTask = 'PRIVATE_PARENT_CHILD_FAILURE_TASK_20260808';
|
|
const childName = 'PRIVATE_CHILD_FAILURE_NAME_20260808';
|
|
const childTask = 'PRIVATE_CHILD_FAILURE_TASK_20260808';
|
|
const childError = 'PRIVATE_CHILD_FAILURE_ERROR_20260808';
|
|
const loopConfigs: AgentLoopConfig[] = [];
|
|
let childOutput = '';
|
|
|
|
testState.runAgentLoop.mockImplementation(async (config: AgentLoopConfig) => {
|
|
loopConfigs.push(config);
|
|
const spawn = config.tools.find(tool => tool.name === 'spawn_agent');
|
|
if (spawn) {
|
|
childOutput = String(await spawn.execute({
|
|
name: childName,
|
|
role: 'custom',
|
|
task: childTask,
|
|
tools: ['read_file'],
|
|
}));
|
|
return {
|
|
content: 'Parent handled the failed delegation',
|
|
toolsUsed: ['spawn_agent'],
|
|
usage: { inputTokens: 1, outputTokens: 1 },
|
|
};
|
|
}
|
|
throw new Error(childError);
|
|
});
|
|
|
|
try {
|
|
const response = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: `Do not use saved memory. Delegate a bounded review of current workspace files. Correlation: ${parentTask}.`,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'general-purpose',
|
|
session: 'memory-denied-child-failure-persistence',
|
|
workspace: collaborationWorkspaceId,
|
|
},
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(loopConfigs).toHaveLength(2);
|
|
expect(childOutput).toContain(childName);
|
|
expect(childOutput).toContain(childError);
|
|
const registry = fs.readFileSync(path.join(tmpDir, 'agent-runs.json'), 'utf8');
|
|
for (const privateContent of [parentTask, childName, childTask, childError]) {
|
|
expect(registry, privateContent).not.toContain(privateContent);
|
|
}
|
|
const worker = server.agentRunRegistry.list({
|
|
source: 'chat_subagent', workspaceId: collaborationWorkspaceId, limit: 1_000,
|
|
}).find(run => run.kind === 'worker'
|
|
&& run.result?.sessionId === 'memory-denied-child-failure-persistence');
|
|
expect(worker?.status).toBe('failed');
|
|
expect(worker?.result?.error).toContain('Not retained');
|
|
} finally {
|
|
testState.runAgentLoop.mockImplementation(defaultRunAgentLoop);
|
|
}
|
|
});
|
|
|
|
it('keeps memory-denied workflow worker and aggregate results out of persisted memory', async () => {
|
|
const parentTask = 'PRIVATE_PARENT_WORKFLOW_TASK_20260808';
|
|
const workflowName = 'PRIVATE_WORKFLOW_NAME_20260808';
|
|
const workflowTask = 'PRIVATE_WORKFLOW_TASK_20260808';
|
|
const stepName = 'PRIVATE_WORKFLOW_STEP_NAME_20260808';
|
|
const stepRole = 'PRIVATE_WORKFLOW_STEP_ROLE_20260808';
|
|
const stepTask = 'PRIVATE_WORKFLOW_STEP_TASK_20260808';
|
|
const workerResult = 'PRIVATE_WORKFLOW_WORKER_RESULT_20260808';
|
|
const personalMind = server.multiMind.personal;
|
|
const workspaceMind = server.mindCache.getOrOpen(collaborationWorkspaceId);
|
|
expect(workspaceMind).toBeDefined();
|
|
const frameCount = (mind: MindDB) => (
|
|
mind.getDatabase().prepare('SELECT COUNT(*) AS count FROM memory_frames').get() as { count: number }
|
|
).count;
|
|
const personalBefore = frameCount(personalMind);
|
|
const workspaceBefore = frameCount(workspaceMind!);
|
|
const loopConfigs: AgentLoopConfig[] = [];
|
|
let workflowOutput = '';
|
|
|
|
testState.runAgentLoop.mockImplementation(async (config: AgentLoopConfig) => {
|
|
loopConfigs.push(config);
|
|
const workflow = config.tools.find(tool => tool.name === 'orchestrate_workflow');
|
|
if (workflow) {
|
|
workflowOutput = String(await workflow.execute({
|
|
task: workflowTask,
|
|
inline_template: {
|
|
name: workflowName,
|
|
description: 'One bounded workspace reviewer',
|
|
aggregation: 'concatenate',
|
|
steps: [{
|
|
name: stepName,
|
|
role: stepRole,
|
|
task: stepTask,
|
|
tools: ['read_file', 'search_memory', 'save_memory', 'agent_insights'],
|
|
}],
|
|
},
|
|
}));
|
|
return {
|
|
content: 'Parent completed after workflow',
|
|
toolsUsed: ['orchestrate_workflow'],
|
|
usage: { inputTokens: 1, outputTokens: 1 },
|
|
};
|
|
}
|
|
return {
|
|
content: workerResult,
|
|
toolsUsed: config.tools.map(tool => tool.name),
|
|
usage: { inputTokens: 1, outputTokens: 1 },
|
|
};
|
|
});
|
|
|
|
try {
|
|
const response = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: `Do not use saved memory. Use orchestrate_workflow to review current workspace files. Correlation: ${parentTask}.`,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'general-purpose',
|
|
session: 'memory-denied-workflow-persistence',
|
|
workspace: collaborationWorkspaceId,
|
|
},
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(loopConfigs).toHaveLength(2);
|
|
expect(workflowOutput).toContain(workflowName);
|
|
expect(workflowOutput).toContain(workerResult);
|
|
expect(loopConfigs[0].modelSpendTraceId).toEqual(expect.any(Number));
|
|
expect(loopConfigs[1].modelSpendBudget).toBe(loopConfigs[0].modelSpendBudget);
|
|
expect(loopConfigs[1].modelSpendTraceId).toBe(loopConfigs[0].modelSpendTraceId);
|
|
expect(loopConfigs[1].spendWorkspaceId).toBe(loopConfigs[0].spendWorkspaceId);
|
|
expect(loopConfigs[1].modelSpendBillingClass).toBe('priced');
|
|
const parentToolNames = loopConfigs[0].tools.map(tool => tool.name);
|
|
expect(parentToolNames).toContain('orchestrate_workflow');
|
|
expect(parentToolNames).not.toContain('list_agents');
|
|
expect(parentToolNames).not.toContain('get_agent_result');
|
|
const workerToolNames = loopConfigs[1].tools.map(tool => tool.name);
|
|
expect(workerToolNames).toContain('read_file');
|
|
expect(workerToolNames).not.toContain('search_memory');
|
|
expect(workerToolNames).not.toContain('save_memory');
|
|
expect(workerToolNames).not.toContain('agent_insights');
|
|
expect(frameCount(personalMind)).toBe(personalBefore);
|
|
expect(frameCount(workspaceMind!)).toBe(workspaceBefore);
|
|
const registry = fs.readFileSync(path.join(tmpDir, 'agent-runs.json'), 'utf8');
|
|
for (const privateContent of [
|
|
parentTask, workflowName, workflowTask, stepName, stepRole, stepTask, workerResult,
|
|
]) {
|
|
expect(registry, privateContent).not.toContain(privateContent);
|
|
}
|
|
const worker = server.agentRunRegistry.list({
|
|
source: 'workflow', workspaceId: collaborationWorkspaceId, limit: 1_000,
|
|
}).find(run => run.kind === 'worker'
|
|
&& run.result?.sessionId === 'memory-denied-workflow-persistence');
|
|
expect(worker?.status).toBe('completed');
|
|
expect(worker?.result?.summary).toContain('Not retained');
|
|
expect(worker?.metrics?.outputTokens).toBe(1);
|
|
expect(worker && server.agentRunRegistry.get(worker.roomId)?.task).toContain('Not retained');
|
|
} finally {
|
|
testState.runAgentLoop.mockImplementation(defaultRunAgentLoop);
|
|
}
|
|
});
|
|
|
|
it('keeps memory-denied workflow tasks and failures out of the durable run registry', async () => {
|
|
const workflowName = 'PRIVATE_WORKFLOW_FAILURE_NAME_20260808';
|
|
const workflowTask = 'PRIVATE_WORKFLOW_FAILURE_TASK_20260808';
|
|
const stepName = 'PRIVATE_WORKFLOW_FAILURE_STEP_NAME_20260808';
|
|
const stepRole = 'PRIVATE_WORKFLOW_FAILURE_STEP_ROLE_20260808';
|
|
const stepTask = 'PRIVATE_WORKFLOW_FAILURE_STEP_TASK_20260808';
|
|
const workerError = 'PRIVATE_WORKFLOW_FAILURE_ERROR_20260808';
|
|
const loopConfigs: AgentLoopConfig[] = [];
|
|
let workflowOutput = '';
|
|
|
|
testState.runAgentLoop.mockImplementation(async (config: AgentLoopConfig) => {
|
|
loopConfigs.push(config);
|
|
const workflow = config.tools.find(tool => tool.name === 'orchestrate_workflow');
|
|
if (workflow) {
|
|
workflowOutput = String(await workflow.execute({
|
|
task: workflowTask,
|
|
inline_template: {
|
|
name: workflowName,
|
|
description: 'One failing bounded workspace reviewer',
|
|
aggregation: 'concatenate',
|
|
steps: [{ name: stepName, role: stepRole, task: stepTask, tools: ['read_file'] }],
|
|
},
|
|
}));
|
|
return {
|
|
content: 'Parent handled the failed workflow',
|
|
toolsUsed: ['orchestrate_workflow'],
|
|
usage: { inputTokens: 1, outputTokens: 1 },
|
|
};
|
|
}
|
|
throw new Error(workerError);
|
|
});
|
|
|
|
try {
|
|
const response = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: 'Do not use saved memory. Run one failing workflow over current workspace files.',
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: 'general-purpose',
|
|
session: 'memory-denied-workflow-failure-persistence',
|
|
workspace: collaborationWorkspaceId,
|
|
},
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(loopConfigs).toHaveLength(2);
|
|
expect(workflowOutput).toContain(workflowName);
|
|
expect(workflowOutput).toContain(workerError);
|
|
const registry = fs.readFileSync(path.join(tmpDir, 'agent-runs.json'), 'utf8');
|
|
for (const privateContent of [
|
|
workflowName, workflowTask, stepName, stepRole, stepTask, workerError,
|
|
]) {
|
|
expect(registry, privateContent).not.toContain(privateContent);
|
|
}
|
|
const worker = server.agentRunRegistry.list({
|
|
source: 'workflow', workspaceId: collaborationWorkspaceId, limit: 1_000,
|
|
}).find(run => run.kind === 'worker'
|
|
&& run.result?.sessionId === 'memory-denied-workflow-failure-persistence');
|
|
expect(worker?.status).toBe('failed');
|
|
expect(worker?.result?.error).toContain('Not retained');
|
|
} finally {
|
|
testState.runAgentLoop.mockImplementation(defaultRunAgentLoop);
|
|
}
|
|
});
|
|
|
|
it('keeps the exact verifier turn compact and below a conservative synthetic input bound', async () => {
|
|
const verifier = PERSONA_CASES.find(persona => persona.id === 'verifier')!;
|
|
const response = await injectWithAuth(server, {
|
|
method: 'POST',
|
|
url: '/api/chat',
|
|
payload: {
|
|
message: verifier.prompt,
|
|
model: 'openrouter/anthropic/claude-sonnet-5',
|
|
persona: verifier.id,
|
|
session: 'persona-acceptance-verifier-budget',
|
|
workspace: 'default',
|
|
},
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(capturedConfig).not.toBeNull();
|
|
const config = capturedConfig!;
|
|
const syntheticInputUpperBound = capturedSyntheticInputUpperBound;
|
|
expect(config.messages).toContainEqual({ role: 'user', content: verifier.prompt });
|
|
expect(config.tools).toEqual([]);
|
|
|
|
const done = parseSse(response.body).find(event => event.event === 'done')?.data;
|
|
expect(done).toBeDefined();
|
|
const metrics = done!.contextMetrics as Record<string, unknown>;
|
|
expect(metrics).toMatchObject({
|
|
packageMode: 'compact',
|
|
toolEligibleCount: 0,
|
|
toolSelectedCount: 0,
|
|
transmittedToolSchemaChars: 0,
|
|
estimatedToolSchemaTokens: 0,
|
|
// Echoed from the mocked runner to lock done-metric propagation only.
|
|
providerInputTokens: syntheticInputUpperBound,
|
|
});
|
|
expect(
|
|
syntheticInputUpperBound,
|
|
`system=${config.systemPrompt.length} chars, messages=${config.messages.length}`,
|
|
).toBeLessThan(verifier.maxInputTokens);
|
|
expect(metrics.estimatedSystemPromptTokens).toBeLessThan(verifier.maxInputTokens);
|
|
expect(metrics.providerInputTokens).toBe(syntheticInputUpperBound);
|
|
});
|
|
});
|