refactor(server): search memory through the contract in artifacts, agent runs, route proposals and onboarding (P1.5)

- Artifact Center related search lists personal and workspace memories
  through server.memory and finds related chat sessions in the session
  files (UF-034).
- /api/agent/run searches the requested workspace's memory, or personal
  memory when no known workspace is given.
- Executor briefs search only the workspace's own memory and leave out
  temporary memories in the query; route proposals no longer borrow a
  mind from the cache.
- Onboarding treats any personal memory as evidence of prior use.
- The service no longer creates personal.mind on a fresh install, and
  isFirstRun also looks for waggle.db.
This commit is contained in:
shumeiko.it
2026-10-24 12:50:15 +02:00
parent e9ed8c0765
commit f310333970
13 changed files with 239 additions and 219 deletions

View File

@@ -1,6 +1,6 @@
import { createHash } from 'node:crypto';
import { redactSecrets, scanForInjection } from '@waggle/agent';
import type { SearchResult } from '@waggle/core';
import { workspaceScope, type MemoryStore } from '@waggle/memory-contract';
const DEFAULT_MAX_CHARS = 8_000;
const DEFAULT_MAX_ITEMS = 6;
@@ -9,14 +9,7 @@ const PREVIEW_LIMIT = 120;
const REDACTION_SKIP_THRESHOLD = 0.3;
const REDACTION_MARKER = /\[REDACTED:[^\]]+\]/g;
interface BriefSearchOptions {
limit?: number;
excludeDeprecated?: boolean;
}
export interface HybridSearchLike {
search(query: string, options?: BriefSearchOptions): Promise<SearchResult[]>;
}
export type BriefMemory = Pick<MemoryStore, 'search'>;
export interface BriefItem {
frameId: string;
@@ -43,36 +36,35 @@ export interface BuildExecutorBriefOptions {
excludeFrameIds?: string[];
}
/** Searches only the workspace's own memory; temporary memories never enter a brief. */
export async function buildExecutorBrief(
deps: { search: HybridSearchLike },
deps: { memory: BriefMemory },
opts: BuildExecutorBriefOptions,
): Promise<ExecutorBrief> {
const maxChars = normalizeLimit(opts.maxChars, DEFAULT_MAX_CHARS);
const maxItems = normalizeLimit(opts.maxItems, DEFAULT_MAX_ITEMS);
const excludedIds = new Set((opts.excludeFrameIds ?? []).map(String));
const searchLimit = Math.max(1, maxItems * 3, maxItems + excludedIds.size);
const results = await deps.search.search(opts.prompt, {
const hits = await deps.memory.search(opts.prompt, {
scopes: [workspaceScope(opts.workspaceId)],
minImportance: 'normal',
limit: searchLimit,
excludeDeprecated: true,
});
const candidates: BriefItem[] = [];
for (const { frame } of results) {
const frameId = String(frame.id);
for (const hit of hits) {
if (
candidates.length >= maxItems
|| excludedIds.has(frameId)
|| frame.importance === 'deprecated'
|| frame.importance === 'temporary'
|| isUnreviewedImport(frame.source, frame.metadata)
|| excludedIds.has(hit.id)
|| isUnreviewedImport(hit.source, hit.metadata)
) {
continue;
}
const redacted = redactSecrets(frame.content);
const redacted = redactSecrets(hit.text);
if (
redacted.found.length > 0
&& redactedShare(frame.content, redacted.text) > REDACTION_SKIP_THRESHOLD
&& redactedShare(hit.text, redacted.text) > REDACTION_SKIP_THRESHOLD
) {
continue;
}
@@ -81,9 +73,9 @@ export async function buildExecutorBrief(
? `${redacted.text}\n[Waggle redacted secret types: ${redacted.found.join(', ')}]`
: redacted.text;
candidates.push({
frameId,
date: frame.created_at.slice(0, 10),
source: frame.source,
frameId: hit.id,
date: hit.createdAt.slice(0, 10),
source: hit.source,
preview: content.slice(0, PREVIEW_LIMIT),
content,
});
@@ -158,14 +150,9 @@ function normalizeLimit(value: number | undefined, fallback: number): number {
return Math.max(0, Math.floor(value));
}
function isUnreviewedImport(source: string, metadata: string | undefined): boolean {
if (source !== 'import' || !metadata) return false;
try {
const parsed = JSON.parse(metadata) as Record<string, unknown>;
return parsed.status === 'unreviewed' || parsed.reviewed === false || parsed.unreviewed === true;
} catch {
return false;
}
function isUnreviewedImport(source: string, metadata: Readonly<Record<string, unknown>>): boolean {
if (source !== 'import') return false;
return metadata.status === 'unreviewed' || metadata.reviewed === false || metadata.unreviewed === true;
}
function redactedShare(original: string, redacted: string): number {

View File

@@ -13,7 +13,7 @@
// loop's onProgress callback (Phase 3.4 — AgentRunProgressEvent).
import type { FastifyPluginAsync } from 'fastify';
import { HybridSearch } from '@waggle/core';
import { PERSONAL_SCOPE, workspaceScope } from '@waggle/memory-contract';
import {
runRetrievalAgentLoop,
listShapes,
@@ -195,30 +195,29 @@ export const agentRunRoutes: FastifyPluginAsync = async (server) => {
};
}
/** Build the HybridSearch-backed search fn for the requested workspace. */
/**
* Search fn over the requested workspace's memory; personal memory when no
* known workspace is given.
*/
function makeSearch(workspaceId: string | undefined): RetrievalSearchFn | null {
const personalMindDb = server.multiMind?.personal;
if (!personalMindDb) return null;
const embedder = server.embeddingProvider;
if (!embedder) return null;
const memory = server.memory;
if (!memory) return null;
const targetMindDb =
workspaceId && workspaceId !== 'personal'
? server.agentState?.getWorkspaceMindDb?.(workspaceId) ?? personalMindDb
: personalMindDb;
const hybrid = new HybridSearch(targetMindDb, embedder);
const scope =
workspaceId && workspaceId !== 'personal' && server.workspaceManager?.get(workspaceId)
? workspaceScope(workspaceId)
: PERSONAL_SCOPE;
return async ({ query, limit }) => {
const hits = await hybrid.search(query, { limit: limit ?? 8 });
const hits = await memory.search(query, { scopes: [scope], limit: limit ?? 8 });
return {
formattedResults:
hits.length === 0
? ''
: hits
.map(
(s, i) =>
`[result ${i + 1}, score ${s.finalScore.toFixed(3)}]\n${s.frame.content}`,
(hit, i) =>
`[result ${i + 1}, score ${hit.score.toFixed(3)}]\n${hit.text}`,
)
.join('\n\n---\n\n'),
resultCount: hits.length,
@@ -242,7 +241,7 @@ export const agentRunRoutes: FastifyPluginAsync = async (server) => {
const search = makeSearch(workspaceId);
if (!search) {
return reply.status(503).send({
error: 'multi-mind or embedding provider not initialized',
error: 'memory not initialized',
});
}

View File

@@ -2,12 +2,16 @@ import type { FastifyPluginAsync } from 'fastify';
import type {
Artifact, ArtifactKind, ArtifactStatus, Memory, RelatedRef, RelatedSearchResult,
} from '@waggle/shared';
import { FrameStore, SessionStore } from '@waggle/core';
import path from 'node:path';
import {
PERSONAL_SCOPE, workspaceScope, type MemoryRecord, type MemoryScope,
} from '@waggle/memory-contract';
import {
readArtifactIndex, addArtifact, getArtifactInWorkspace,
patchArtifactInWorkspace, deleteArtifactFromWorkspace, type NewArtifactInput,
} from './artifact-index.js';
import { normalizeToMemory } from './memory-center.js';
import { memoryStatus } from '../memory-status.js';
import { searchSessions } from './session-utils.js';
import { readTasks } from './tasks.js';
import { emitAuditEvent } from './events.js';
import { assertSafeSegment } from './validate.js';
@@ -52,7 +56,6 @@ const MAX_LIST = 200;
const MAX_WORKSPACE_FANOUT = 24;
const PER_SOURCE_CAP = 10;
const MEMORY_SCAN = 100;
const SESSION_SCAN = 50;
const asKind = (v: unknown): ArtifactKind | undefined =>
ARTIFACT_KINDS.includes(v as ArtifactKind) ? (v as ArtifactKind) : undefined;
@@ -63,6 +66,34 @@ const clampStr = (s: unknown, max: number): string => String(s ?? '').slice(0, m
const clampStrArray = (a: unknown, maxItems: number, maxLen: number): string[] =>
Array.isArray(a) ? a.slice(0, maxItems).map((x) => clampStr(x, maxLen)) : [];
/** First non-empty line of a memory, capped for display. */
function memoryTitle(text: string): string {
const firstLine = text.split('\n').find((l) => l.trim().length > 0)?.trim() ?? text.trim();
return firstLine.length > 80 ? `${firstLine.slice(0, 79)}…` : firstLine;
}
const stringArray = (v: unknown): string[] | undefined =>
Array.isArray(v) ? v.filter((x): x is string => typeof x === 'string') : undefined;
/** Project a memory into the shared `Memory` shape the Artifact Center renders. */
function toRelatedMemory(record: MemoryRecord): Memory {
const meta = record.metadata;
return {
id: record.id,
kind: record.kind === 'decision' || record.kind === 'preference' ? record.kind : 'fact',
title: typeof meta.title === 'string' && meta.title.trim() ? meta.title : memoryTitle(record.text),
content: record.text,
scope: record.scope.kind,
...(record.scope.kind === 'workspace' ? { workspaceId: record.scope.workspaceId } : {}),
source: record.source,
importance: record.importance,
tags: stringArray(meta.tags),
status: memoryStatus(record),
createdAt: record.createdAt,
...(record.updatedAt ? { updatedAt: record.updatedAt } : {}),
};
}
/** Case-insensitive substring match over an artifact's title + tags. */
function artifactMatches(a: Artifact, ql: string): boolean {
if (a.title.toLowerCase().includes(ql)) return true;
@@ -159,23 +190,23 @@ export const artifactRoutes: FastifyPluginAsync = async (server) => {
.sort((a, b) => (b.updatedAt ?? '').localeCompare(a.updatedAt ?? ''))
.slice(0, PER_SOURCE_CAP);
// memories — personal + (scoped|all) workspace minds, via the shared normalizer.
// memories — recent personal memories, then each (scoped|all) workspace's.
const memories: Memory[] = [];
try {
const scan = (db: import('@waggle/core').MindDB | null | undefined, mind: string, wsId?: string) => {
if (!db || memories.length >= PER_SOURCE_CAP) return;
for (const f of new FrameStore(db).getRecent(MEMORY_SCAN)) {
const m = normalizeToMemory(f, mind, wsId);
const scan = async (scope: MemoryScope) => {
if (memories.length >= PER_SOURCE_CAP) return;
for (const record of await server.memory.list({ scopes: [scope], limit: MEMORY_SCAN })) {
const m = toRelatedMemory(record);
if (m.content.toLowerCase().includes(ql) || m.title.toLowerCase().includes(ql)) {
memories.push(m);
if (memories.length >= PER_SOURCE_CAP) break;
}
}
};
scan(server.multiMind.personal, 'personal');
await scan(PERSONAL_SCOPE);
for (const wsId of workspaceIds(only)) {
if (memories.length >= PER_SOURCE_CAP) break;
scan(server.agentState.getWorkspaceMindDb(wsId), 'workspace', wsId);
await scan(workspaceScope(wsId));
}
} catch {
// degrade to whatever was collected
@@ -197,29 +228,21 @@ export const artifactRoutes: FastifyPluginAsync = async (server) => {
// skip
}
// sessions — bounded best-effort: recent active sessions whose summary matches.
// (Full closed-session search lives on the dedicated /sessions/search route.)
// sessions — chat sessions whose title, summary or messages match, the
// same search the dedicated /sessions/search route runs.
const sessions: RelatedRef[] = [];
try {
const scanSessions = (db: import('@waggle/core').MindDB | null | undefined, wsId?: string) => {
if (!db || sessions.length >= PER_SOURCE_CAP) return;
for (const s of new SessionStore(db).getActive().slice(0, SESSION_SCAN)) {
const hay = `${s.summary ?? ''} ${s.gop_id}`.toLowerCase();
if (hay.includes(ql)) {
sessions.push({
id: s.gop_id,
title: s.summary?.slice(0, 120) || s.gop_id,
workspaceId: wsId,
kind: s.status,
});
if (sessions.length >= PER_SOURCE_CAP) break;
}
}
};
scanSessions(server.multiMind.personal);
for (const wsId of workspaceIds(only)) {
if (sessions.length >= PER_SOURCE_CAP) break;
scanSessions(server.agentState.getWorkspaceMindDb(wsId), wsId);
const sessionsDir = path.join(dataDir, 'workspaces', wsId, 'sessions');
for (const hit of searchSessions(sessionsDir, q, PER_SOURCE_CAP - sessions.length)) {
sessions.push({
id: hit.sessionId,
title: hit.title.slice(0, 120),
workspaceId: wsId,
...(hit.snippets[0] ? { snippet: hit.snippets[0].text } : {}),
});
}
}
} catch {
// skip

View File

@@ -1,7 +1,7 @@
import type { FastifyPluginAsync } from 'fastify';
import fs from 'node:fs';
import path from 'node:path';
import { FrameStore } from '@waggle/core';
import { PERSONAL_SCOPE } from '@waggle/memory-contract';
/**
* P4 (UX-Refactor) — server-authoritative onboarding status.
@@ -17,9 +17,9 @@ import { FrameStore } from '@waggle/core';
* Tauri `mark_first_launch_complete` command writes for default installs)
* is stamped by POST /api/onboarding/complete when any client finishes the
* wizard, and
* - LEGACY EVIDENCE covers installs that predate the flag: real usage in the
* personal mind (any frame) or user-created workspaces beyond the seeded
* default.
* - LEGACY EVIDENCE covers installs that predate the flag: real usage in
* personal memory (any memory) or user-created workspaces beyond the
* seeded default.
*
* Fail direction: no flag + no evidence → completed:false → the wizard runs.
* Better to show it once too often than to silently skip it (same rule as the
@@ -37,9 +37,9 @@ export const onboardingRoutes: FastifyPluginAsync = async (server) => {
// that finds no flag and no evidence stamps a durable PENDING marker — from
// then on, only explicit completion flips the status. Without the latch,
// the wizard's own pre-completion writes count as "evidence" (step 1's
// PUT /api/profile writes a 'User identity:' frame; C33 import writes
// frames; workspace-create makes list().length > 1) — and so does the
// monthly-assessment CRON frame, with zero user interaction. Any of those
// PUT /api/profile writes a 'User identity:' memory; C33 import writes
// memories; workspace-create makes list().length > 1) — and so do
// background writes, with zero user interaction. Any of those
// plus a lost localStorage (fresh WebView profile on reinstall, site-data
// clear) would silently skip the wizard for a user who never finished it —
// the exact failure direction this surface exists to prevent (review P4,
@@ -52,11 +52,10 @@ export const onboardingRoutes: FastifyPluginAsync = async (server) => {
return { completed: false, source: 'pending' };
}
try {
const frames = new FrameStore(server.multiMind.personal);
if (frames.getRecent(1).length > 0) {
if ((await server.memory.list({ scopes: [PERSONAL_SCOPE], limit: 1 })).length > 0) {
return { completed: true, source: 'legacy-evidence' };
}
} catch { /* personal mind unavailable → not evidence of prior use */ }
} catch { /* memory unavailable → not evidence of prior use */ }
try {
if (server.workspaceManager.list().length > 1) {
return { completed: true, source: 'legacy-evidence' };

View File

@@ -1,7 +1,6 @@
import { randomUUID } from 'node:crypto';
import type { FastifyInstance, FastifyPluginAsync } from 'fastify';
import { z } from 'zod';
import { HybridSearch } from '@waggle/core';
import {
classifyRateLimitError,
classifyTask,
@@ -288,12 +287,7 @@ export const routeProposalRoutes: FastifyPluginAsync = async (server) => {
async function provideBrief(server: FastifyInstance, opts: BriefOptions): Promise<ExecutorBrief> {
if (server.routeProposalBriefProvider) return server.routeProposalBriefProvider(opts);
const db = server.mindCache.acquire(opts.workspaceId);
try {
return await buildExecutorBrief({ search: new HybridSearch(db, server.embeddingProvider) }, opts);
} finally {
server.mindCache.release(opts.workspaceId);
}
return buildExecutorBrief({ memory: server.memory }, opts);
}
function proposalResponse(proposal: RouteProposal) {

View File

@@ -4,7 +4,7 @@ import net from 'node:net';
import path from 'node:path';
import os from 'node:os';
import type { FastifyInstance } from 'fastify';
import { needsMigration, migrateToMultiMind, MindDB, VaultStore, WaggleConfig } from '@waggle/core';
import { needsMigration, migrateToMultiMind, VaultStore, WaggleConfig, WAGGLE_DB_FILE } from '@waggle/core';
import { buildLocalServer } from './index.js';
import type { LlmHealthStatus } from './index.js';
import { startLiteLLM, stopLiteLLM, type LiteLLMStatus } from './lifecycle.js';
@@ -28,7 +28,7 @@ const log = createLogger('service');
// ── Startup progress types ─────────────────────────────────────────
export type StartupPhase = 'init' | 'migration' | 'creating-mind' | 'litellm' | 'server' | 'ready';
export type StartupPhase = 'init' | 'migration' | 'litellm' | 'server' | 'ready';
export interface StartupEvent {
phase: StartupPhase;
@@ -114,13 +114,13 @@ export function resolveServicePort(optionPort?: number): number {
}
/**
* Check if this is a fresh install (no personal.mind, no default.mind).
* Check if this is a fresh install: no waggle.db and no legacy personal.mind
* or default.mind.
*/
export function isFirstRun(dataDir: string): boolean {
if (!fs.existsSync(dataDir)) return true;
const hasPersonal = fs.existsSync(path.join(dataDir, 'personal.mind'));
const hasDefault = fs.existsSync(path.join(dataDir, 'default.mind'));
return !hasPersonal && !hasDefault;
return ![WAGGLE_DB_FILE, 'personal.mind', 'default.mind']
.some((file) => fs.existsSync(path.join(dataDir, file)));
}
/**
@@ -174,10 +174,9 @@ function getConfiguredProviderIds(dataDir: string, server?: FastifyInstance): st
*
* 1. Resolves/creates dataDir (option > WAGGLE_DATA_DIR > ~/.waggle)
* 2. Runs migration if needed (default.mind -> personal.mind)
* 3. Creates personal.mind if fresh install
* 4. Starts the LiteLLM proxy when config.json enables it and it is not skipped
* 5. Builds & starts local Fastify server
* 6. Registers graceful shutdown handlers
* 3. Starts the LiteLLM proxy when config.json enables it and it is not skipped
* 4. Builds & starts local Fastify server
* 5. Registers graceful shutdown handlers
*
* Optionally accepts an onProgress callback to emit startup phase events
* for UI splash screen display.
@@ -248,14 +247,6 @@ export async function startService(options?: ServiceOptions): Promise<ServiceRes
// directory the server will actually serve.
log.info(`Data dir: ${dataDir}`);
// 3. Ensure personal.mind exists
const personalPath = path.join(dataDir, 'personal.mind');
if (!fs.existsSync(personalPath)) {
emit({ phase: 'creating-mind', message: 'Creating personal memory...', progress: 0.3 });
const mind = new MindDB(personalPath);
mind.close();
}
// 4. LiteLLM is optional and runs on the user's own Python, so it starts
// only when config.json enables it ("litellm": { "enabled": true }). The
// built-in provider proxy serves every request otherwise. When it does