refactor(memory): run agent recall, memory tools and the server on the memory contract (P1.5)

- Orchestrator, memory tools, context loader and pattern write-back use a
  MemoryStore and a workspace scope instead of MindDB layers; the knowledge
  graph tools, cognify, memory linker and feedback handler are gone (graph
  hidden in v1, ADR-0002).
- The server opens one mem0 store in <dataDir>/memory/, embedded by the
  in-process model on first use, and extracts facts about the user in the
  background with the model the user chose (memoryExtraction setting).
- Legacy .mind memories are copied into the store once, in the background;
  the legacy files are only read.
- Weaver consolidation, harvest, wiki, memory import, memory lanes, vector
  backfill and index reconciliation are removed; agent runs, loops, sub-agent
  results, connector fetch and the failed-turn capture write through the
  contract. Workspace sessions no longer hold a mind handle.
- FileIndexer and the install audit move to waggle.db; recall date helpers and
  the reranker are copied into core.
- The CLI REPL opens the same memory store.
This commit is contained in:
shumeiko.it
2026-10-24 11:56:26 +02:00
parent f3eb6b6652
commit 784ca770b1
86 changed files with 2520 additions and 8444 deletions

View File

@@ -1,7 +1,7 @@
/**
* Pattern-based memory write-back. Inspects (userMsg, assistantMsg) for
* preference / decision / correction / structured-finding signals and writes
* the distilled frames into the appropriate mind store.
* the distilled memories into personal or workspace memory.
*
* Extracted from Orchestrator.autoSaveFromExchange (PR-A, 2026-05-27).
* Behavior preserved verbatim; only the call surface moved.
@@ -14,23 +14,21 @@
*
* SOTA target: this is the regex-based predecessor of an LLM-driven memory
* writer (PR-H). The patterns here are deliberately conservative — false-
* positive suppression matters more than recall, because every saved frame
* positive suppression matters more than recall, because every saved memory
* becomes recallable context.
*/
import {
type FrameStore,
type SessionStore,
type Importance,
type FrameSource,
type MemoryFrame,
type MindDB,
type TeamSync,
createCoreLogger,
evaluateExternalMemoryIngress,
} from '@waggle/core';
PERSONAL_SCOPE,
workspaceScope,
type MemoryImportance,
type MemoryKind,
type MemoryRecord,
type MemorySource,
type MemoryStore,
} from '@waggle/memory-contract';
import { createCoreLogger, evaluateExternalMemoryIngress } from '@waggle/core';
import { isSelfIncapacityAssertion } from './memory-sign-gate.js';
import type { CognifyPipeline } from './cognify.js';
import {
MIN_CONTENT_LENGTH,
DEDUP_SLICE_LENGTH,
@@ -43,26 +41,14 @@ import {
const logger = createCoreLogger('pattern-write-back');
/**
* Stores + facilities the extractor needs to persist frames to the right
* mind (workspace vs personal) and optionally push to team-sync. The seam is
* kept tight on purpose — the extractor doesn't need to know about identity,
* awareness, search, or knowledge layers.
* Where the extractor writes. Preferences, corrections and style notes go to
* personal memory; decisions and work output go to the active workspace, or
* to personal memory when no workspace is active.
*/
export interface PatternWriteBackDeps {
/** Personal mind layers (preferences, corrections, style notes route here) */
personal: {
db: MindDB;
frames: FrameStore;
sessions: SessionStore;
};
/** Workspace mind layers (decisions, work output route here). Null = personal-only mode. */
workspace: {
frames: FrameStore;
sessions: SessionStore;
cognify: CognifyPipeline;
} | null;
/** Optional team-sync client; receives pushFrame for workspace writes only */
teamSync: TeamSync | null;
memory: MemoryStore;
/** Active workspace, or null for personal-only mode. */
workspaceId: string | null;
}
// ── Pattern definitions (lifted from inline orchestrator function) ──
@@ -135,93 +121,64 @@ type SaveTarget = 'workspace' | 'personal';
/**
* Post-response heuristic write-back. Scans (userMsg, assistantMsg) for
* save-worthy signals and writes distilled memory frames. Returns short
* save-worthy signals and writes distilled memories. Returns short
* previews of what was saved (DEDUP_SLICE_LENGTH chars each).
*
* Routing:
* target='workspace' → uses workspace cognify+frames when active, else personal
* target='workspace' → the active workspace's memory, else personal
* target='personal' → always personal (preferences, corrections, style)
*/
export async function runPatternWriteBack(
deps: PatternWriteBackDeps,
userMsg: string,
assistantMsg: string,
opts?: { traceId?: string },
opts?: { traceId?: string; sessionId?: string },
): Promise<string[]> {
const saved: string[] = [];
// Helper: persist one frame, applying the sign-gate downgrade and the
// team-sync push for workspace writes. Returns the created frame so
// teamSync.pushFrame gets the actual write, not whatever is "latest" at
// read time (the bug that motivated Review #9).
// Helper: persist one memory, applying the ingress guard and the sign-gate
// downgrade. Returns the stored record (an identical earlier memory when the
// text already exists), or null when the guard refused it.
const save = async (
content: string,
importance: Importance,
importance: MemoryImportance,
kind: MemoryKind,
target: SaveTarget = 'workspace',
// PR3.5 honesty (review H-1): this is a heuristic EXTRACTOR, so its writes
// default to 'agent_inferred' — NOT the schema default 'user_stated', which
// would make the Memory-Trust provenance pill claim the user said things the
// agent inferred. Genuinely user-authored frames (preferences, corrections)
// pass 'user_stated' explicitly at their call sites.
source: FrameSource = 'agent_inferred',
): Promise<MemoryFrame | null> => {
// default to 'agent_inferred'. Genuinely user-authored memories
// (preferences, corrections) pass 'user_stated' explicitly.
source: MemorySource = 'agent_inferred',
): Promise<MemoryRecord | null> => {
if (evaluateExternalMemoryIngress({ content }).action !== 'allow') return null;
// R2 sign gate (DEFECT-2): self-incapacity assertions persist at
// 'temporary' so they're audit-visible but cannot re-enter the prompt as
// authoritative recall (recall path excludes 'temporary').
// authoritative recall (recall excludes 'temporary').
if (importance !== 'temporary' && isSelfIncapacityAssertion(content)) {
logger.debug('autoSave sign-gate: self-incapacity frame downgraded to temporary', {
logger.debug('autoSave sign-gate: self-incapacity memory downgraded to temporary', {
preview: content.slice(0, DEDUP_SLICE_LENGTH),
});
importance = 'temporary';
}
const useWorkspace = target === 'workspace' && deps.workspace !== null;
const frames = useWorkspace ? deps.workspace!.frames : deps.personal.frames;
const sessions = useWorkspace ? deps.workspace!.sessions : deps.personal.sessions;
const cognify = useWorkspace ? deps.workspace!.cognify : null;
const scope = target === 'workspace' && deps.workspaceId
? workspaceScope(deps.workspaceId)
: PERSONAL_SCOPE;
let createdFrame: MemoryFrame | null = null;
if (cognify) {
const result = await cognify.cognify(content, importance, undefined, undefined, source);
createdFrame = frames.getById(result.frameId) ?? null;
} else {
// ensureActive is transaction-wrapped — concurrent saves on a fresh
// mind don't race into twin sessions with frames split across them.
const session = sessions.ensureActive();
const gopId = session.gop_id;
const latestI = frames.getLatestIFrame(gopId);
createdFrame = latestI
? frames.createPFrame(gopId, content, latestI.id, importance, source)
: frames.createIFrame(gopId, content, importance, source);
}
// PR3.5 memory↔trace backlink: the execution-trace id that produced this
// memory, so the "Why did you do that?" view can resolve the decision.
// An identical earlier memory keeps the trace that originally wrote it.
const [change] = await deps.memory.remember({
scope,
text: content,
kind,
importance,
source,
origin: 'chat-write-back',
...(opts?.sessionId && { sessionId: opts.sessionId }),
...(opts?.traceId && { metadata: { trace_id: opts.traceId } }),
});
saved.push(content.slice(0, DEDUP_SLICE_LENGTH));
// PR3.5 frame↔trace backlink: stamp the execution-trace id that produced
// this frame so the Memory-Trust "Why did you do that?" view can resolve
// the real decision (GET /api/memory/:id/trace). Additive metadata only —
// never touches content/importance/dedup, so it's byte-identical for any
// caller that doesn't pass a traceId (every benchmark path).
if (opts?.traceId && createdFrame) {
let existingMeta: Record<string, unknown> = {};
try {
existingMeta = createdFrame.metadata ? JSON.parse(createdFrame.metadata) as Record<string, unknown> : {};
} catch { /* malformed metadata → start clean */ }
// Preserve an existing backlink (review L-1): createIFrame dedups on
// content hash, so createdFrame may be an OLDER frame already linked to
// the trace that originally wrote it. Keep that originating trace rather
// than re-pointing it to this turn's re-assertion.
if (existingMeta.trace_id === undefined) {
frames.setMetadata(createdFrame.id, JSON.stringify({ ...existingMeta, trace_id: opts.traceId }));
}
}
if (deps.teamSync && useWorkspace && createdFrame) {
deps.teamSync.pushFrame(createdFrame).catch(() => { /* non-blocking */ });
}
return createdFrame;
return change?.record ?? null;
};
// ── Early gate: short messages skip unless they look like a decision ack ──
@@ -243,7 +200,7 @@ export async function runPatternWriteBack(
const sentences = userMsg.split(/[.!?\n]+/).filter(s => pat.test(s));
if (sentences.length > 0) {
// User stated the preference directly → genuine 'user_stated' provenance.
await save(`User preference: ${sentences[0].trim()}`, 'normal', 'personal', 'user_stated');
await save(`User preference: ${sentences[0].trim()}`, 'normal', 'preference', 'personal', 'user_stated');
}
break;
}
@@ -253,13 +210,8 @@ export async function runPatternWriteBack(
if (saved.length === 0 && userMsg.length > MIN_CONTENT_LENGTH) {
for (const { pattern, note } of STYLE_SIGNALS) {
if (pattern.test(userMsg)) {
const personalRaw = deps.personal.db.getDatabase();
const existing = personalRaw.prepare(
`SELECT id FROM memory_frames WHERE content = ? LIMIT 1`
).get(note) as { id: number } | undefined;
if (!existing) {
await save(note, 'normal', 'personal');
}
// The store does not write identical text twice in one scope.
await save(note, 'normal', 'preference', 'personal');
break;
}
}
@@ -294,13 +246,13 @@ export async function runPatternWriteBack(
}
if (decisionText) {
await save(`Decision: ${decisionText}`, 'important');
await save(`Decision: ${decisionText}`, 'important', 'decision');
}
// ── Pattern: user correction ──
if (CORRECTION_PATTERNS.some(p => p.test(userMsg))) {
// User stated the correction directly → genuine 'user_stated' provenance.
await save(`Correction from user: ${userMsg.slice(0, CONTEXT_PREVIEW_LENGTH)}`, 'important', 'personal', 'user_stated');
await save(`Correction from user: ${userMsg.slice(0, CONTEXT_PREVIEW_LENGTH)}`, 'important', 'fact', 'personal', 'user_stated');
}
// ── Pattern: research output with external sources ──
@@ -317,7 +269,7 @@ export async function runPatternWriteBack(
...(urls.length > 0 ? [`Sources: ${urls.join(', ')}`] : []),
].join('. ');
if (findingSummary.length > 20) {
await save(`Research findings: ${findingSummary.slice(0, FINDINGS_SLICE_LENGTH)}`, 'important');
await save(`Research findings: ${findingSummary.slice(0, FINDINGS_SLICE_LENGTH)}`, 'important', 'summary');
}
}
@@ -338,7 +290,7 @@ export async function runPatternWriteBack(
// not a user-stated fact. Persist it audit-visible but at 'temporary'
// so the recall path (which excludes 'temporary') can't re-surface a
// confabulated specific as authoritative memory on a later turn.
if ((await save(`Recommendation: ${text.slice(0, RECALL_LINE_LENGTH)}`, 'temporary')) !== null) {
if ((await save(`Recommendation: ${text.slice(0, RECALL_LINE_LENGTH)}`, 'temporary', 'note')) !== null) {
savedStructured = true;
break inlineDecision;
}
@@ -351,7 +303,7 @@ export async function runPatternWriteBack(
s.startsWith('User preference:') || s.startsWith('Correction from user:') || s.startsWith('Decision:')
);
if (!alreadyCapturedUser) {
if ((await save(`User asked: ${userMsg.slice(0, RECALL_LINE_LENGTH)}`, 'temporary')) !== null) {
if ((await save(`User asked: ${userMsg.slice(0, RECALL_LINE_LENGTH)}`, 'temporary', 'note')) !== null) {
savedStructured = true;
}
}
@@ -370,7 +322,7 @@ export async function runPatternWriteBack(
const prefix = heading ? `${heading}: ` : 'Key points: ';
// Agent-extracted bullets from its own reply — audit-visible but
// 'temporary' (recall-excluded) so confabulated specifics can't loop back.
if ((await save(`${prefix}${keyPoints.slice(0, FINDINGS_SLICE_LENGTH)}`, 'temporary')) !== null) {
if ((await save(`${prefix}${keyPoints.slice(0, FINDINGS_SLICE_LENGTH)}`, 'temporary', 'note')) !== null) {
savedStructured = true;
}
}
@@ -386,7 +338,7 @@ export async function runPatternWriteBack(
: firstMeaningful || 'Work output produced';
// Agent's own summary of its reply — audit-visible but 'temporary'
// (recall-excluded) to sever the confabulation→save→recall-as-fact loop.
await save(`Work completed: ${summary.slice(0, RECALL_LINE_LENGTH)}`, 'temporary');
await save(`Work completed: ${summary.slice(0, RECALL_LINE_LENGTH)}`, 'temporary', 'note');
}
}