The reference-set comparison showed the cross-encoder adds nothing to mem0's ranking and costs about 130 ms per search.
777 lines
32 KiB
TypeScript
777 lines
32 KiB
TypeScript
import {
|
|
PERSONAL_SCOPE,
|
|
workspaceScope,
|
|
scopeKey,
|
|
type MemoryStore,
|
|
type MemoryScope,
|
|
type MemoryRecord,
|
|
type MemoryHit,
|
|
} from '@waggle/memory-contract';
|
|
import {
|
|
IdentityLayer,
|
|
AwarenessLayer,
|
|
WaggleDB,
|
|
createCoreLogger,
|
|
evaluateExternalMemoryIngress,
|
|
TEMPORAL_GUIDANCE,
|
|
renderReferenceDateLine,
|
|
parseDateWindow,
|
|
createInProcessReranker,
|
|
type Reranker,
|
|
} from '@waggle/core';
|
|
import { createMindTools, type ToolDefinition } from './tools.js';
|
|
import { buildSelfAwareness, type AgentCapabilities } from './self-awareness.js';
|
|
import { renderGoalAncestry } from './goal-ancestry.js';
|
|
import type { GoalAncestry } from '@waggle/shared';
|
|
import { scanForInjection } from './injection-scanner.js';
|
|
import { runPatternWriteBack } from './pattern-write-back.js';
|
|
import {
|
|
fetchRecentMemories,
|
|
loadRecentContext as loadRecentContextImpl,
|
|
loadRecentContextFrames as loadRecentContextFramesImpl,
|
|
type ContextFrames as ContextFramesImpl,
|
|
} from './context-loader.js';
|
|
import { toRecallFrame, type RecallFrame } from './recall-frame.js';
|
|
|
|
// Re-export ContextFrames for back-compat — tests + apps import this type
|
|
// from `./orchestrator` per the pre-PR-F surface.
|
|
export type ContextFrames = ContextFramesImpl;
|
|
// H-AUDIT-1 contract: turnId is a per-turn trace ID (UUID v4) generated at
|
|
// chat-route turn entry and propagated EXPLICITLY through every downstream
|
|
// stage (agent-loop → orchestrator → retrieval → prompt-assembler →
|
|
// tool-calls). No AsyncLocalStorage, no globals — propagation is
|
|
// tsc-verifiable via optional `turnId?: string` params on each entry
|
|
// function. See `turn-context.ts` for the generator + logging helpers.
|
|
import { logTurnEvent } from './turn-context.js';
|
|
import { tierForModel, type ModelTier } from './model-tier.js';
|
|
import type { AgentPersona } from './personas.js';
|
|
import {
|
|
isClosedWorldRewriteRequest,
|
|
PromptAssembler,
|
|
type AssembleOptions,
|
|
type AssembledPrompt,
|
|
type RecalledMemory,
|
|
} from './prompt-assembler.js';
|
|
|
|
const logger = createCoreLogger('orchestrator');
|
|
|
|
// Content-length constants now live in `./content-constants.ts` (single
|
|
// source of truth shared with the pattern-write-back extractor). Imports
|
|
// below pull only the ones this file still references.
|
|
import {
|
|
RECALL_LINE_LENGTH,
|
|
RECALLED_SNIPPET_LENGTH,
|
|
} from './content-constants.js';
|
|
|
|
export interface OrchestratorConfig {
|
|
/** Memory (ADR-0007). Personal memory plus the active workspace's memory. */
|
|
memory: MemoryStore;
|
|
/**
|
|
* Waggle's own database, which holds the profile card and awareness items.
|
|
* Omitting it keeps them in memory for the lifetime of the orchestrator;
|
|
* only tests should do that.
|
|
*/
|
|
waggleDb?: WaggleDB;
|
|
apiKey?: string;
|
|
model?: string;
|
|
mode?: 'local' | 'team';
|
|
version?: string;
|
|
skills?: string[];
|
|
/**
|
|
* W4.2: optional cross-encoder reranker injected for tests. When absent and
|
|
* WAGGLE_RERANKER=1, a lazy in-process reranker (Xenova/ms-marco-MiniLM-L-6-v2,
|
|
* ~22MB ONNX) is created on first recall; creation failure soft-fails to the
|
|
* store's own ordering. Off by default: on the reference set it did not
|
|
* improve mem0's ranking and cost about 130 ms per search
|
|
* (docs/oss/research/memory-search-quality.md).
|
|
*/
|
|
reranker?: Reranker;
|
|
/** Optional managed cache root for the lazy in-process reranker model. */
|
|
rerankerCacheDir?: string;
|
|
/** AI-OS #6 — durable "why" breadcrumb injected into buildSystemPrompt. */
|
|
goalAncestry?: GoalAncestry;
|
|
}
|
|
|
|
/**
|
|
* Options for tier-adaptive recall. When omitted, `recallMemory` uses the
|
|
* default ordering and no score floor.
|
|
*/
|
|
export interface RecallOptions {
|
|
/** Drop results whose score is below this floor. Default: no filter. */
|
|
scoreFloor?: number;
|
|
/** Model tier hint — recorded for downstream consumers (PromptAssembler). */
|
|
tier?: ModelTier;
|
|
/** H-AUDIT-1: per-turn trace ID (UUID v4). Logs memory-recall stage. */
|
|
turnId?: string;
|
|
}
|
|
|
|
/** Candidates sent to the cross-encoder before the top `limit` are kept. */
|
|
const RERANK_POOL = 30;
|
|
/** Critical/important memories that reach recall on every query. */
|
|
const IMPORTANCE_LANE_K = 5;
|
|
/** Most recent events rendered in chronological order. */
|
|
const EVENT_LANE_K = 40;
|
|
|
|
const IMPORTANCE_RANK: Record<string, number> = { critical: 0, important: 1, normal: 2, temporary: 3 };
|
|
|
|
export class Orchestrator {
|
|
private memory: MemoryStore;
|
|
private identity: IdentityLayer;
|
|
private awareness: AwarenessLayer;
|
|
private tools: ToolDefinition[];
|
|
private model: string;
|
|
private mode: 'local' | 'team';
|
|
private version: string;
|
|
private skills: string[];
|
|
/** AI-OS #6 — durable "why" breadcrumb; null = no section rendered. */
|
|
private goalAncestry: GoalAncestry | null = null;
|
|
|
|
/** Active workspace (null when only personal memory is in play). */
|
|
private workspaceId: string | null = null;
|
|
/** W4.2: memoized reranker promise — resolves undefined on creation failure. */
|
|
private rerankerPromise: Promise<Reranker | undefined> | null = null;
|
|
private readonly rerankerCacheDir: string | undefined;
|
|
|
|
/**
|
|
* Section cache — stores computed values and their inputs.
|
|
* Cached sections are recomputed only when their input changes.
|
|
*/
|
|
private _sectionCache = new Map<string, { input: string; output: string }>();
|
|
|
|
constructor(config: OrchestratorConfig) {
|
|
this.memory = config.memory;
|
|
this.model = config.model ?? 'unknown';
|
|
this.mode = config.mode ?? 'local';
|
|
this.version = config.version ?? '0.0.0';
|
|
this.skills = config.skills ?? [];
|
|
this.rerankerCacheDir = config.rerankerCacheDir;
|
|
this.goalAncestry = config.goalAncestry ?? null;
|
|
const waggleDb = config.waggleDb ?? new WaggleDB(':memory:');
|
|
this.identity = new IdentityLayer(waggleDb);
|
|
this.awareness = new AwarenessLayer(waggleDb);
|
|
if (config.reranker) this.rerankerPromise = Promise.resolve(config.reranker);
|
|
|
|
this.tools = createMindTools({
|
|
memory: this.memory,
|
|
identity: this.identity,
|
|
awareness: this.awareness,
|
|
// Checked at call time so tools follow workspace switches.
|
|
getWorkspaceId: () => this.workspaceId,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Make a workspace's memory active alongside personal memory, or pass null
|
|
* to go back to personal memory only. Identity always stays personal.
|
|
*/
|
|
setWorkspace(workspaceId: string | null): void {
|
|
if (this.workspaceId && workspaceId && this.workspaceId !== workspaceId) {
|
|
logger.info('switching workspace memory');
|
|
}
|
|
this.workspaceId = workspaceId;
|
|
}
|
|
|
|
/** The active workspace, or null. */
|
|
getWorkspaceId(): string | null {
|
|
return this.workspaceId;
|
|
}
|
|
|
|
/** Whether a workspace's memory is currently active. */
|
|
hasWorkspace(): boolean {
|
|
return this.workspaceId !== null;
|
|
}
|
|
|
|
/**
|
|
* AI-OS #6 — set/replace the goal-ancestry breadcrumb rendered by the next
|
|
* buildSystemPrompt(). Pass null to clear it. Mutable like setWorkspace
|
|
* so the per-session caller can populate it once context is resolved.
|
|
*/
|
|
setGoalAncestry(ancestry: GoalAncestry | null): void {
|
|
this.goalAncestry = ancestry;
|
|
}
|
|
|
|
private workspaceMemoryScope(): MemoryScope | null {
|
|
return this.workspaceId ? workspaceScope(this.workspaceId) : null;
|
|
}
|
|
|
|
/** Workspace memory when a workspace is active, else personal. */
|
|
private activeScope(): MemoryScope {
|
|
return this.workspaceMemoryScope() ?? PERSONAL_SCOPE;
|
|
}
|
|
|
|
/** Personal memory plus the active workspace's. */
|
|
private readableScopes(): MemoryScope[] {
|
|
const ws = this.workspaceMemoryScope();
|
|
return ws ? [PERSONAL_SCOPE, ws] : [PERSONAL_SCOPE];
|
|
}
|
|
|
|
async getMemoryStats(): Promise<{ memoryCount: number }> {
|
|
return { memoryCount: await this.memory.count({ scopes: this.readableScopes() }) };
|
|
}
|
|
|
|
/**
|
|
* Load recent context from memory for session preloading. Delegates to
|
|
* `loadRecentContextImpl` — see `./context-loader.ts`.
|
|
*/
|
|
async loadRecentContext(limit = 5): Promise<string> {
|
|
return loadRecentContextImpl(this.contextLoaderDeps(), limit);
|
|
}
|
|
|
|
/**
|
|
* Typed counterpart for `PromptAssembler`. Delegates to
|
|
* `loadRecentContextFramesImpl` — see `./context-loader.ts`. Pure data;
|
|
* injection scanning is the assembler's responsibility (it has tier
|
|
* context needed to decide drop vs sanitize).
|
|
*/
|
|
async loadRecentContextFrames(limit = 10): Promise<ContextFrames> {
|
|
return loadRecentContextFramesImpl(this.contextLoaderDeps(), limit);
|
|
}
|
|
|
|
private contextLoaderDeps() {
|
|
return {
|
|
memory: this.memory,
|
|
workspaceId: this.workspaceId,
|
|
awareness: this.awareness,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Return a cached section value if the input hasn't changed.
|
|
* Avoids redundant string construction for stable sections (e.g., identity).
|
|
*/
|
|
private cachedSection(name: string, input: string, compute: () => string): string {
|
|
const cached = this._sectionCache.get(name);
|
|
if (cached && cached.input === input) return cached.output;
|
|
const output = compute();
|
|
this._sectionCache.set(name, { input, output });
|
|
return output;
|
|
}
|
|
|
|
async buildSystemPrompt(modelOverride = this.model): Promise<string> {
|
|
// ── IDENTITY (always personal, stable within a session) ──
|
|
// Cache key must hash the full identity content — updated_at alone
|
|
// has only second precision in SQLite, so rapid successive edits
|
|
// (and fresh-mind tests) collide on the timestamp.
|
|
const identitySection = this.cachedSection(
|
|
'identity',
|
|
this.identity.exists() ? JSON.stringify(this.identity.get()) : 'empty',
|
|
() => this.identity.exists() ? '# Identity\n' + this.identity.toContext() : '',
|
|
);
|
|
|
|
// ── GOAL ANCESTRY (the durable "why"; changes only when re-set) ──
|
|
const goalAncestrySection = this.cachedSection(
|
|
'goal_ancestry',
|
|
JSON.stringify(this.goalAncestry) || 'empty',
|
|
() => renderGoalAncestry(this.goalAncestry),
|
|
);
|
|
|
|
// ── SELF-AWARENESS (runtime context, changes every call) ──
|
|
const caps: AgentCapabilities = {
|
|
tools: this.tools.map(t => ({ name: t.name, description: t.description })),
|
|
skills: this.skills,
|
|
model: modelOverride,
|
|
memoryStats: await this.getMemoryStats(),
|
|
mode: this.mode,
|
|
version: this.version,
|
|
};
|
|
const awarenessSection = buildSelfAwareness(caps);
|
|
|
|
// ── PRELOADED CONTEXT (per-session memory, changes every call) ──
|
|
const recentContext = await this.loadRecentContext();
|
|
const contextSection = recentContext
|
|
? '# Context From Your Memory\nThis was automatically loaded — you already know this:\n' + recentContext
|
|
: '';
|
|
|
|
const parts = [identitySection, goalAncestrySection, awarenessSection, contextSection].filter(Boolean);
|
|
return parts.join('\n\n');
|
|
}
|
|
|
|
/**
|
|
* PromptAssembler integration — produces a tier-adaptive, typed,
|
|
* scaffolded prompt via the new sixth layer.
|
|
*
|
|
* Consumers: agent-loop.ts when `isEnabled('PROMPT_ASSEMBLER')`.
|
|
* Feature-flagged, default off — callers outside that gate should keep
|
|
* using `buildSystemPrompt()` + `recallMemory()` directly.
|
|
*/
|
|
async buildAssembledPrompt(
|
|
query: string,
|
|
persona: AgentPersona | null = null,
|
|
opts: AssembleOptions & { model?: string } = {},
|
|
): Promise<AssembledPrompt> {
|
|
const effectiveModel = opts.model ?? this.model;
|
|
const tier = tierForModel(effectiveModel);
|
|
const closedWorldRewrite = isClosedWorldRewriteRequest(query);
|
|
const corePrompt = closedWorldRewrite ? '' : await this.buildSystemPrompt(effectiveModel);
|
|
const context: ContextFramesImpl = closedWorldRewrite
|
|
? {
|
|
stateFrames: [],
|
|
recentChanges: [],
|
|
activeWork: [],
|
|
keyEntities: [],
|
|
personalPreferences: [],
|
|
}
|
|
: await this.loadRecentContextFrames();
|
|
|
|
let recalled: RecalledMemory;
|
|
if (closedWorldRewrite) {
|
|
recalled = {
|
|
workspace: [],
|
|
personal: [],
|
|
scanSafe: true,
|
|
renderedText: '',
|
|
};
|
|
} else if (opts.recalledText !== undefined) {
|
|
// W4.5 (plan bug #9-2, double-compute): the caller already ran
|
|
// recallMemory this turn — reuse its rendered block instead of
|
|
// re-running the searches. recallMemory scans for injection itself
|
|
// (returns '' on a hit), so scanSafe is true by construction here.
|
|
recalled = {
|
|
workspace: [],
|
|
personal: [],
|
|
scanSafe: true,
|
|
renderedText: opts.recalledText,
|
|
};
|
|
} else {
|
|
// Direct search for raw memories (recallMemory returns formatted text;
|
|
// the assembler consumes RecallFrame[] and applies its own rendering).
|
|
const wsScope = this.workspaceMemoryScope();
|
|
const personalResults = await this.searchScope(PERSONAL_SCOPE, query, 10);
|
|
const workspaceResults = wsScope ? await this.searchScope(wsScope, query, 10) : [];
|
|
|
|
// Brief §8: run the injection scan here; assembler trusts scanSafe and
|
|
// must not re-scan. On a poisoned hit, frames still flow through but
|
|
// scanSafe=false causes the assembler to ignore the recall section.
|
|
const joinedContent = [...workspaceResults, ...personalResults].map(r => r.content).join('\n');
|
|
const scanSafe = joinedContent.length === 0
|
|
? true
|
|
: scanForInjection(joinedContent, 'tool_output').safe;
|
|
|
|
recalled = {
|
|
workspace: workspaceResults,
|
|
personal: personalResults,
|
|
scanSafe,
|
|
};
|
|
}
|
|
|
|
return new PromptAssembler().assemble(
|
|
{
|
|
corePrompt,
|
|
persona,
|
|
context,
|
|
recalled,
|
|
query,
|
|
tier,
|
|
},
|
|
opts,
|
|
);
|
|
}
|
|
|
|
/**
|
|
* W4.2: lazy cross-encoder reranker, opt-in with WAGGLE_RERANKER=1. First use
|
|
* downloads the ~22MB model (cached at the configured managed path); creation
|
|
* failure (offline, OOM) memoizes undefined: recall soft-fails to the store's
|
|
* ordering, never throws.
|
|
*/
|
|
private getReranker(): Promise<Reranker | undefined> {
|
|
if (this.rerankerPromise) return this.rerankerPromise;
|
|
if (process.env['WAGGLE_RERANKER'] !== '1') {
|
|
this.rerankerPromise = Promise.resolve(undefined);
|
|
return this.rerankerPromise;
|
|
}
|
|
const rerankerConfig = this.rerankerCacheDir
|
|
? { cacheDir: this.rerankerCacheDir }
|
|
: undefined;
|
|
this.rerankerPromise = createInProcessReranker(rerankerConfig).catch((e: unknown) => {
|
|
logger.warn('reranker unavailable — falling back to store ordering', {
|
|
error: e instanceof Error ? e.message : String(e),
|
|
});
|
|
return undefined;
|
|
});
|
|
return this.rerankerPromise;
|
|
}
|
|
|
|
/**
|
|
* Ranked search in one scope. With a reranker, a wider pool is fetched and
|
|
* re-ordered by the cross-encoder before the top `limit` are kept.
|
|
*/
|
|
private async searchScope(
|
|
scope: MemoryScope,
|
|
query: string,
|
|
limit: number,
|
|
window?: { since: string; until: string } | null,
|
|
reranker?: Reranker,
|
|
): Promise<RecallFrame[]> {
|
|
const hits = await this.memory.search(query, {
|
|
scopes: [scope],
|
|
limit: reranker ? Math.max(limit, RERANK_POOL) : limit,
|
|
...(window && { since: startOfDay(window.since), until: endOfDay(window.until) }),
|
|
});
|
|
let ordered: MemoryHit[] = hits;
|
|
if (reranker && hits.length > 1) {
|
|
try {
|
|
const scores = await reranker.scoreBatch(query, hits.map(h => h.text));
|
|
ordered = hits
|
|
.map((hit, i) => ({ ...hit, score: scores[i] ?? hit.score }))
|
|
.sort((a, b) => b.score - a.score);
|
|
} catch (err) {
|
|
logger.warn('reranker failed — keeping store ordering', {
|
|
error: err instanceof Error ? err.message : String(err),
|
|
});
|
|
}
|
|
}
|
|
return ordered.slice(0, limit).map(hit => toRecallFrame(hit, hit.score));
|
|
}
|
|
|
|
/**
|
|
* Automatic memory recall: search for memories relevant to the user's query.
|
|
* Searches BOTH personal and workspace memory when a workspace is active.
|
|
* Returns formatted recall text with source attribution.
|
|
*/
|
|
async recallMemory(
|
|
query: string,
|
|
limit = 10,
|
|
opts?: RecallOptions,
|
|
): Promise<{ text: string; count: number; recalled?: string[]; recalledFrames?: Array<{ source: string }> }> {
|
|
const scoreFloor = opts?.scoreFloor;
|
|
logTurnEvent(opts?.turnId, { stage: 'orchestrator.recallMemory.enter', queryChars: query.length, limit });
|
|
const empty = { text: '', count: 0, recalled: [], recalledFrames: [] };
|
|
try {
|
|
const wsScope = this.workspaceMemoryScope();
|
|
const active = this.activeScope();
|
|
const anyMemory = await this.memory.list({ scopes: this.readableScopes(), limit: 1 });
|
|
if (anyMemory.length === 0) {
|
|
logTurnEvent(opts?.turnId, {
|
|
stage: 'orchestrator.recallMemory.exit',
|
|
totalCount: 0,
|
|
blocked: false,
|
|
emptyMindFastPath: true,
|
|
});
|
|
return empty;
|
|
}
|
|
|
|
// Detect catch-up intent — these queries need importance-based recall, not literal text matching
|
|
const catchUpPatterns = [
|
|
/\bcatch me up\b/i, /\bwhere (?:are|were) we\b/i, /\bwhat matters\b/i,
|
|
/\bwhat did we decide\b/i, /\bwhere did we leave off\b/i, /\bwhat['']?s the status\b/i,
|
|
/\bwhat should I do\b/i, /\bwhat['']?s next\b/i, /\bsummariz/i, /\bbrief me\b/i,
|
|
/\bget me up to speed\b/i, /\bwhat['']?s going on\b/i, /\bremind me\b/i,
|
|
];
|
|
const isCatchUp = catchUpPatterns.some(p => p.test(query));
|
|
|
|
let personalResults: RecallFrame[];
|
|
let workspaceResults: RecallFrame[];
|
|
// W4.1b/W4.3b: parsed explicit-period window — drives since/until in the
|
|
// normal branch AND the "Events during X" render section below.
|
|
let dateWindow: ReturnType<typeof parseDateWindow> = null;
|
|
|
|
if (isCatchUp && wsScope) {
|
|
// Catch-up: important memories and decisions by importance + recency,
|
|
// plus the most recent ones for recency context. Dedup by id.
|
|
const [important, decisions, recent] = await Promise.all([
|
|
this.memory.list({ scopes: [wsScope], minImportance: 'important', limit }),
|
|
this.memory.list({ scopes: [wsScope], kinds: ['decision'], limit }),
|
|
fetchRecentMemories(this.memory, wsScope, Math.min(limit, 3), { excludeTemporary: true }),
|
|
]);
|
|
const byId = new Map<string, MemoryRecord>();
|
|
for (const r of [...important, ...decisions]) byId.set(r.id, r);
|
|
const ranked = [...byId.values()].sort(byImportanceThenNewest);
|
|
for (const r of recent) if (!byId.has(r.id)) ranked.push(r);
|
|
workspaceResults = ranked.slice(0, limit).map(r => toRecallFrame(r, 1));
|
|
personalResults = await this.searchScope(PERSONAL_SCOPE, query, 2);
|
|
} else {
|
|
// W4.1b (#3) — deterministic date-window lane: when the query names an
|
|
// explicit period ("in May 2026", "on 13 October 2025", "in 2024"),
|
|
// restrict recall to memories created in that window. A window that
|
|
// matches nothing falls back to unwindowed search — the lane must
|
|
// never LOSE recall, only sharpen it.
|
|
dateWindow = parseDateWindow(query);
|
|
const reranker = await this.getReranker();
|
|
|
|
personalResults = await this.searchScope(PERSONAL_SCOPE, query, limit, dateWindow, reranker);
|
|
workspaceResults = wsScope
|
|
? await this.searchScope(wsScope, query, limit, dateWindow, reranker)
|
|
: [];
|
|
|
|
if (dateWindow && personalResults.length === 0 && workspaceResults.length === 0) {
|
|
logTurnEvent(opts?.turnId, { stage: 'orchestrator.recallMemory.dateWindowEmpty', label: dateWindow.label });
|
|
personalResults = await this.searchScope(PERSONAL_SCOPE, query, limit, null, reranker);
|
|
workspaceResults = wsScope
|
|
? await this.searchScope(wsScope, query, limit, null, reranker)
|
|
: [];
|
|
}
|
|
|
|
// W4.1 (#2) — importance lane: critical/important memories reach recall
|
|
// on EVERY query. Active scope only; rendered BEFORE semantic hits;
|
|
// deduped by id so a memory surfaced by both lanes renders once.
|
|
const lane = (await this.memory.list({
|
|
scopes: [active],
|
|
minImportance: 'important',
|
|
limit: 50,
|
|
})).sort(byImportanceThenNewest).slice(0, IMPORTANCE_LANE_K);
|
|
if (lane.length > 0) {
|
|
const laneIds = new Set(lane.map(r => r.id));
|
|
const laneResults = lane.map(r => toRecallFrame(r, 1));
|
|
if (wsScope) {
|
|
workspaceResults = [...laneResults, ...workspaceResults.filter(r => !laneIds.has(r.id))];
|
|
} else {
|
|
personalResults = [...laneResults, ...personalResults.filter(r => !laneIds.has(r.id))];
|
|
}
|
|
}
|
|
}
|
|
|
|
// R2 sign-gate: self-incapacity memories are stored at 'temporary'
|
|
// importance so they don't re-enter the prompt as authoritative recall.
|
|
const isAuthoritative = (r: RecallFrame): boolean => r.importance !== 'temporary';
|
|
personalResults = personalResults.filter(isAuthoritative);
|
|
workspaceResults = workspaceResults.filter(isAuthoritative);
|
|
|
|
// Optional score floor (PromptAssembler opt-in).
|
|
if (scoreFloor !== undefined) {
|
|
const passes = (r: RecallFrame): boolean => r.score >= scoreFloor;
|
|
personalResults = personalResults.filter(passes);
|
|
workspaceResults = workspaceResults.filter(passes);
|
|
}
|
|
|
|
// ── Profile and event lanes ────────────────────────────────────────
|
|
// What the model learned about the user (kind 'profile') renders in
|
|
// full; events render chronologically, most recent EVENT_LANE_K.
|
|
const [profiles, eventsNewestFirst] = await Promise.all([
|
|
this.memory.list({ scopes: this.readableScopes(), kinds: ['profile'], limit: 200 }),
|
|
this.memory.list({ scopes: [active], kinds: ['event'], limit: 500 }),
|
|
]);
|
|
const profileFrames = profiles
|
|
.filter(r => r.importance !== 'temporary')
|
|
.reverse();
|
|
const eventFramesAll = eventsNewestFirst
|
|
.filter(r => r.importance !== 'temporary')
|
|
.reverse();
|
|
const eventFrames = eventFramesAll.slice(-EVENT_LANE_K);
|
|
|
|
// Dedup: lane memories never double-render via the search lanes.
|
|
const laneIds = new Set<string>([
|
|
...profileFrames.map(r => r.id),
|
|
...eventFramesAll.map(r => r.id),
|
|
]);
|
|
const notLane = (r: RecallFrame): boolean => !laneIds.has(r.id);
|
|
personalResults = personalResults.filter(notLane);
|
|
workspaceResults = workspaceResults.filter(notLane);
|
|
|
|
const allLines: string[] = [];
|
|
|
|
if (profileFrames.length > 0) {
|
|
allLines.push('## Profiles');
|
|
for (const r of profileFrames) {
|
|
allLines.push(`- ${r.text.slice(0, 1200)}`);
|
|
}
|
|
}
|
|
if (eventFrames.length > 0) {
|
|
allLines.push('## Events (chronological)');
|
|
for (const r of eventFrames) {
|
|
allLines.push(`- [${r.createdAt.slice(0, 10)}] ${r.text.slice(0, RECALL_LINE_LENGTH)}`);
|
|
}
|
|
}
|
|
// W4.3b: explicit-period queries surface the events INSIDE the window as
|
|
// a dedicated section so the model binds to the right event instead of a
|
|
// similar one from another month.
|
|
if (dateWindow) {
|
|
const windowEvents = eventFramesAll.filter(r => {
|
|
const d = r.createdAt.slice(0, 10);
|
|
return d >= dateWindow!.since && d <= dateWindow!.until;
|
|
});
|
|
if (windowEvents.length > 0) {
|
|
allLines.push(`## Events during ${dateWindow.label}`);
|
|
for (const r of windowEvents) {
|
|
allLines.push(`- [${r.createdAt.slice(0, 10)}] ${r.text.slice(0, RECALL_LINE_LENGTH)}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (workspaceResults.length > 0) {
|
|
allLines.push('## Workspace Memory');
|
|
for (const r of workspaceResults) {
|
|
allLines.push(`- [${r.created_at.slice(0, 10)}, ${r.importance}] ${r.content.slice(0, RECALL_LINE_LENGTH)}`);
|
|
}
|
|
}
|
|
|
|
if (personalResults.length > 0) {
|
|
allLines.push('## Personal Memory');
|
|
if (wsScope) {
|
|
allLines.push('_(Cross-workspace personal knowledge — not specific to this workspace)_');
|
|
}
|
|
for (const r of personalResults) {
|
|
allLines.push(`- [${r.created_at.slice(0, 10)}, ${r.importance}] ${r.content.slice(0, RECALL_LINE_LENGTH)}`);
|
|
}
|
|
}
|
|
|
|
const laneRecords = [...profileFrames, ...eventFrames];
|
|
const totalCount = personalResults.length + workspaceResults.length + laneRecords.length;
|
|
if (totalCount === 0) {
|
|
logTurnEvent(opts?.turnId, { stage: 'orchestrator.recallMemory.exit', totalCount: 0, blocked: false });
|
|
return empty;
|
|
}
|
|
|
|
// Content snippets for UI display (B5 fix) and per-memory provenance for
|
|
// the auto_recall step pill (PR3.5).
|
|
const recalled: string[] = [];
|
|
const recalledFrames: Array<{ source: string }> = [];
|
|
for (const r of [...workspaceResults, ...personalResults]) {
|
|
recalled.push(r.content.slice(0, RECALLED_SNIPPET_LENGTH));
|
|
recalledFrames.push({ source: r.source });
|
|
}
|
|
for (const r of laneRecords) {
|
|
recalled.push(r.text.slice(0, RECALLED_SNIPPET_LENGTH));
|
|
recalledFrames.push({ source: r.source });
|
|
}
|
|
|
|
// Scan recalled memory for injection — a poisoned memory (e.g. text
|
|
// with an embedded "ignore previous instructions") must never silently
|
|
// flow into model context.
|
|
const joinedLines = allLines.join('\n');
|
|
const scan = scanForInjection(joinedLines, 'tool_output');
|
|
if (!scan.safe) {
|
|
logger.warn('recalled-memory injection detected — blocking recall', {
|
|
score: scan.score,
|
|
flags: scan.flags,
|
|
count: totalCount,
|
|
});
|
|
logTurnEvent(opts?.turnId, { stage: 'orchestrator.recallMemory.exit', totalCount, blocked: true, injectionScore: scan.score });
|
|
return empty;
|
|
}
|
|
|
|
// W4.1 (#1): anchor = latest creation time across all rendered memories.
|
|
const anchorLine = renderReferenceDateLine([
|
|
...[...workspaceResults, ...personalResults].map(r => r.created_at),
|
|
...laneRecords.map(r => r.createdAt),
|
|
]);
|
|
|
|
const text = '# Recalled Memories\n'
|
|
+ "These are facts saved in this WORKSPACE'S memory, retrieved for the user's current message. "
|
|
+ 'They may come from earlier sessions, other sessions, or imported sources — NOT necessarily from this conversation.\n'
|
|
+ 'IMPORTANT — ground your response in them, but attribute provenance HONESTLY:\n'
|
|
+ '- Attribute saved / earlier-session memory EXPLICITLY as memory: "your saved memory shows…", "in an earlier session you noted…", "from your workspace notes…". Never imply an ongoing relationship — do NOT say "welcome back", "you\'re back in context", "as we\'ve been discussing", or "from our last session", even when the recalled memory is real and cross-session. Reserve "you just said" / "as you mentioned" strictly for things said earlier in THIS same conversation.\n'
|
|
+ '- On the user\'s first message, do NOT claim continuity ("welcome back", "as we discussed", "you\'re back in context") — you have no prior turn with them yet.\n'
|
|
+ '- State ONLY what the memories below actually say. Do NOT add specifics — runway figures, headcounts, dollar amounts, dates, percentages, entity COUNTS, or competitor names — unless they appear verbatim in the memories. A detail that feels plausible but is not written below is CONFABULATION: ask instead of asserting. (Observed failures to avoid: stating "4 months runway" or "227 entities tracked" when neither appears in the memories.)\n'
|
|
+ '- Do NOT ignore relevant memories. Do NOT present memory content as your own reasoning — attribute it.\n'
|
|
// W4.1 (#1): temporal guidance rides with the recalled block (NOT the
|
|
// global system prompt) + a reference-date anchor so the model has a
|
|
// concrete "now" to resolve relative time against.
|
|
+ TEMPORAL_GUIDANCE + '\n\n'
|
|
+ (anchorLine ? anchorLine + '\n' : '')
|
|
+ joinedLines;
|
|
|
|
logTurnEvent(opts?.turnId, {
|
|
stage: 'orchestrator.recallMemory.exit',
|
|
totalCount,
|
|
workspaceHits: workspaceResults.length,
|
|
personalHits: personalResults.length,
|
|
textChars: text.length,
|
|
});
|
|
return { text, count: totalCount, recalled, recalledFrames };
|
|
} catch (err) {
|
|
// Surface failures visibly — silent empty results train the model
|
|
// to confabulate "I don't remember" instead of recalling real memory.
|
|
logger.error('recallMemory failed', err);
|
|
return {
|
|
text: '[Memory recall temporarily unavailable. Proceed without prior context.]',
|
|
count: 0,
|
|
recalled: [],
|
|
recalledFrames: [],
|
|
};
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Post-response heuristic write-back. Delegates to `runPatternWriteBack` —
|
|
* the regex pattern set + extractor logic live in `./pattern-write-back.ts`.
|
|
* Routes preferences/corrections/style to personal memory; decisions and
|
|
* work-output to the workspace (or personal when no workspace is active).
|
|
*/
|
|
async autoSaveFromExchange(
|
|
userMsg: string,
|
|
assistantMsg: string,
|
|
opts?: { traceId?: string; sessionId?: string },
|
|
): Promise<string[]> {
|
|
return runPatternWriteBack(
|
|
{ memory: this.memory, workspaceId: this.workspaceId },
|
|
userMsg,
|
|
assistantMsg,
|
|
opts,
|
|
);
|
|
}
|
|
|
|
/**
|
|
* #12: persist the context-compaction summary as a memory — the dual use of
|
|
* the one compaction LLM call (already re-injected into live context by the
|
|
* compressor; this makes it durable). One memory per session, updated in
|
|
* place on later compaction passes (each pass is a superset — the previous
|
|
* summary feeds the summarizer). Goes to the active workspace when there is
|
|
* one, else personal (mirrors save_memory). Returns the memory id, or null
|
|
* when nothing was persisted.
|
|
*/
|
|
async persistCompactionSummary(
|
|
summary: string,
|
|
sessionKey: string,
|
|
priorMemoryId?: string | null,
|
|
): Promise<string | null> {
|
|
if (!summary.trim()) return null;
|
|
// Deliberately NO sign-gate here: the summary is a multi-section
|
|
// COMPACTION_PROMPT aggregate, and a single boilerplate "you'll need to
|
|
// run X" line inside it would downgrade the whole session gist to
|
|
// 'temporary' (recall-invisible). Provenance is source='system'.
|
|
const marker = `[Session summary — ${sessionKey}]`;
|
|
const content = `${marker}\n\n${summary}`;
|
|
if (evaluateExternalMemoryIngress({ content }).action !== 'allow') return null;
|
|
|
|
const scope = this.activeScope();
|
|
// Update in place ONLY when the prior memory is verifiably this session's
|
|
// summary in the same scope: after a workspace switch the caller's id may
|
|
// belong to another scope, and a blind update would overwrite it there.
|
|
if (priorMemoryId) {
|
|
const existing = await this.memory.get(priorMemoryId);
|
|
if (existing?.text.startsWith(marker) && scopeKey(existing.scope) === scopeKey(scope)) {
|
|
await this.memory.update(priorMemoryId, { text: content });
|
|
return priorMemoryId;
|
|
}
|
|
}
|
|
const [change] = await this.memory.remember({
|
|
scope,
|
|
text: content,
|
|
kind: 'summary',
|
|
importance: 'normal',
|
|
source: 'system',
|
|
origin: 'compaction',
|
|
sessionId: sessionKey,
|
|
});
|
|
return change?.record.id ?? null;
|
|
}
|
|
|
|
getTools(): ToolDefinition[] {
|
|
return this.tools;
|
|
}
|
|
|
|
async executeTool(name: string, args: Record<string, unknown>): Promise<string> {
|
|
const tool = this.tools.find(t => t.name === name);
|
|
if (!tool) throw new Error(`Unknown tool: ${name}`);
|
|
return tool.execute(args);
|
|
}
|
|
|
|
getIdentity(): IdentityLayer { return this.identity; }
|
|
getAwareness(): AwarenessLayer { return this.awareness; }
|
|
getMemory(): MemoryStore { return this.memory; }
|
|
}
|
|
|
|
function byImportanceThenNewest(a: MemoryRecord, b: MemoryRecord): number {
|
|
return (IMPORTANCE_RANK[a.importance] ?? 9) - (IMPORTANCE_RANK[b.importance] ?? 9)
|
|
|| b.createdAt.localeCompare(a.createdAt);
|
|
}
|
|
|
|
/** Date windows are whole days; memory timestamps are full ISO 8601. */
|
|
function startOfDay(date: string): string {
|
|
return date.length === 10 ? `${date}T00:00:00.000Z` : date;
|
|
}
|
|
|
|
function endOfDay(date: string): string {
|
|
return date.length === 10 ? `${date}T23:59:59.999Z` : date;
|
|
}
|