moving
Some checks failed
Installer Smoke / installer-smoke (push) Has been cancelled

This commit is contained in:
Oleg Maslov
2026-09-02 10:10:29 +02:00
commit 0c3e2ead3b
3841 changed files with 970576 additions and 0 deletions

View File

@@ -0,0 +1,30 @@
{
"name": "@waggle/core",
"version": "0.1.0",
"description": "Waggle-specific orchestration: config, multi-mind, workspace, vault, telemetry, install-audit, cron, skills, file-store, file-indexer, memory-import, optimization-log, compliance. Re-exports substrate from @waggle/hive-mind-core for backward compatibility.",
"type": "module",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"exports": {
".": {
"types": "./dist/index.d.ts",
"import": "./dist/index.js"
}
},
"scripts": {
"build": "tsc",
"test": "vitest run"
},
"dependencies": {
"@waggle/hive-mind-core": "*",
"@waggle/shared": "*",
"better-sqlite3": "^12.6.2",
"cron-parser": "^4.9.0",
"glob": "^13.0.6"
},
"devDependencies": {
"@aws-sdk/client-s3": "^3.1019.0",
"@types/better-sqlite3": "^7.6.13"
},
"license": "MIT"
}

View File

@@ -0,0 +1,5 @@
export * from './types.js';
export { InteractionStore } from './interaction-store.js';
export { ComplianceStatusChecker } from './status-checker.js';
export { ReportGenerator } from './report-generator.js';
export { ComplianceTemplateStore } from './template-store.js';

View File

@@ -0,0 +1,208 @@
/**
* InteractionStore — CRUD for ai_interactions table (EU AI Act Art. 12).
*
* Records every AI interaction with model, tokens, cost, tools, and human oversight.
* Operates on the .mind DB alongside FrameStore and InstallAuditStore.
*/
import type { MindDB } from '@waggle/hive-mind-core';
import type { AIInteraction, RecordInteractionInput, HumanAction, ModelInventoryEntry, OversightLogEntry } from './types.js';
export class InteractionStore {
private db: MindDB;
constructor(db: MindDB) {
this.db = db;
// Review Major #8: previously ensureTable() duplicated the DDL from schema.ts and
// drifted whenever the canonical schema changed. Now schema.ts + MindDB.runMigrations()
// own the table definition, including the input_text/output_text columns and the
// append-only triggers (Critical #1, #3).
}
/** Record an AI interaction event. */
record(input: RecordInteractionInput): AIInteraction {
const raw = this.db.getDatabase();
// Review Major #4: use lastInsertRowid from .run() instead of reading back
// ORDER BY id DESC LIMIT 1 — under concurrent writes (WaggleDance multi-agent),
// the LIMIT 1 row may belong to a different writer.
const result = raw.prepare(`
INSERT INTO ai_interactions (
workspace_id, session_id, model, provider,
input_tokens, output_tokens, cost_usd,
tools_called, human_action, risk_context, imported_from, persona,
input_text, output_text
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
input.workspaceId ?? null,
input.sessionId ?? null,
input.model,
input.provider,
input.inputTokens,
input.outputTokens,
input.costUsd,
JSON.stringify(input.toolsCalled ?? []),
input.humanAction ?? 'none',
input.riskContext ?? null,
input.importedFrom ?? null,
input.persona ?? null,
input.inputText ?? null,
input.outputText ?? null,
);
const row = raw.prepare('SELECT * FROM ai_interactions WHERE id = ?').get(result.lastInsertRowid) as Record<string, unknown>;
return this.rowToInteraction(row);
}
/** Get interactions for a workspace within a date range. */
getByWorkspace(workspaceId: string, from?: string, to?: string): AIInteraction[] {
if (from && !/^\d{4}-\d{2}-\d{2}/.test(from)) throw new Error('Invalid date format');
if (to && !/^\d{4}-\d{2}-\d{2}/.test(to)) throw new Error('Invalid date format');
const raw = this.db.getDatabase();
let sql = 'SELECT * FROM ai_interactions WHERE workspace_id = ?';
const params: unknown[] = [workspaceId];
if (from) { sql += ' AND timestamp >= ?'; params.push(from); }
if (to) { sql += ' AND timestamp <= ?'; params.push(to); }
sql += ' ORDER BY timestamp DESC LIMIT 1000';
return (raw.prepare(sql).all(...params) as Record<string, unknown>[]).map(r => this.rowToInteraction(r));
}
/** Get all interactions within a date range. */
getByDateRange(from: string, to: string, workspaceId?: string): AIInteraction[] {
if (from && !/^\d{4}-\d{2}-\d{2}/.test(from)) throw new Error('Invalid date format');
if (to && !/^\d{4}-\d{2}-\d{2}/.test(to)) throw new Error('Invalid date format');
const raw = this.db.getDatabase();
let sql = 'SELECT * FROM ai_interactions WHERE timestamp >= ? AND timestamp <= ?';
const params: unknown[] = [from, to];
if (workspaceId) { sql += ' AND workspace_id = ?'; params.push(workspaceId); }
sql += ' ORDER BY timestamp DESC LIMIT 1000';
return (raw.prepare(sql).all(...params) as Record<string, unknown>[]).map(r => this.rowToInteraction(r));
}
/** Get total interaction count. */
count(workspaceId?: string): number {
const raw = this.db.getDatabase();
if (workspaceId) {
const row = raw.prepare('SELECT COUNT(*) as cnt FROM ai_interactions WHERE workspace_id = ?').get(workspaceId) as { cnt: number };
return row.cnt;
}
const row = raw.prepare('SELECT COUNT(*) as cnt FROM ai_interactions').get() as { cnt: number };
return row.cnt;
}
/** Get the oldest log timestamp (Art. 19 retention check). */
getOldestTimestamp(): string | null {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT MIN(timestamp) as oldest FROM ai_interactions').get() as { oldest: string | null };
return row.oldest;
}
/**
* Returns the `first_run_at` timestamp from the MindDB's `meta` table — set on
* schema initialization and backfilled for pre-existing DBs. Used by the Art. 19
* retention checker to distinguish 'new system' from 'pruned logs'.
*/
getFirstRunAt(): string | null {
return this.db.getFirstRunAt();
}
/** Get model inventory (aggregated usage per model). */
getModelInventory(from?: string, to?: string, workspaceId?: string): ModelInventoryEntry[] {
const raw = this.db.getDatabase();
let sql = `
SELECT model, provider,
COUNT(*) as calls,
SUM(input_tokens) as input_tokens,
SUM(output_tokens) as output_tokens,
SUM(cost_usd) as cost_usd
FROM ai_interactions WHERE 1=1
`;
const params: unknown[] = [];
if (from) { sql += ' AND timestamp >= ?'; params.push(from); }
if (to) { sql += ' AND timestamp <= ?'; params.push(to); }
if (workspaceId) { sql += ' AND workspace_id = ?'; params.push(workspaceId); }
sql += ' GROUP BY model, provider ORDER BY calls DESC';
return (raw.prepare(sql).all(...params) as Record<string, unknown>[]).map(r => ({
model: r.model as string,
provider: r.provider as string,
calls: r.calls as number,
inputTokens: r.input_tokens as number,
outputTokens: r.output_tokens as number,
costUsd: r.cost_usd as number,
}));
}
/** Get human oversight actions (Art. 14). */
getOversightLog(from?: string, to?: string, workspaceId?: string): OversightLogEntry[] {
const raw = this.db.getDatabase();
let sql = "SELECT * FROM ai_interactions WHERE human_action != 'none' AND human_action IS NOT NULL";
const params: unknown[] = [];
if (from) { sql += ' AND timestamp >= ?'; params.push(from); }
if (to) { sql += ' AND timestamp <= ?'; params.push(to); }
if (workspaceId) { sql += ' AND workspace_id = ?'; params.push(workspaceId); }
sql += ' ORDER BY timestamp DESC';
return (raw.prepare(sql).all(...params) as Record<string, unknown>[]).map(r => ({
timestamp: r.timestamp as string,
action: r.human_action as HumanAction,
tool: (JSON.parse(r.tools_called as string) as string[])[0] ?? 'none',
detail: r.persona ? `Persona: ${r.persona}` : '',
}));
}
/** Get human oversight counts for compliance check. */
getOversightCounts(workspaceId?: string): { total: number; approved: number; denied: number; modified: number } {
const raw = this.db.getDatabase();
let sql = "SELECT human_action, COUNT(*) as cnt FROM ai_interactions WHERE human_action != 'none' AND human_action IS NOT NULL";
const params: unknown[] = [];
if (workspaceId) { sql += ' AND workspace_id = ?'; params.push(workspaceId); }
sql += ' GROUP BY human_action';
const rows = raw.prepare(sql).all(...params) as { human_action: string; cnt: number }[];
const counts = { total: 0, approved: 0, denied: 0, modified: 0 };
for (const row of rows) {
counts.total += row.cnt;
if (row.human_action === 'approved') counts.approved = row.cnt;
if (row.human_action === 'denied') counts.denied = row.cnt;
if (row.human_action === 'modified') counts.modified = row.cnt;
}
return counts;
}
/** Get recent interactions. */
getRecent(limit: number = 20): AIInteraction[] {
const raw = this.db.getDatabase();
return (raw.prepare('SELECT * FROM ai_interactions ORDER BY id DESC LIMIT ?').all(limit) as Record<string, unknown>[])
.map(r => this.rowToInteraction(r));
}
private rowToInteraction(row: Record<string, unknown>): AIInteraction {
return {
id: row.id as number,
timestamp: row.timestamp as string,
workspaceId: row.workspace_id as string | null,
sessionId: row.session_id as string | null,
model: row.model as string,
provider: row.provider as string,
inputTokens: row.input_tokens as number,
outputTokens: row.output_tokens as number,
costUsd: row.cost_usd as number,
toolsCalled: JSON.parse(row.tools_called as string) as string[],
humanAction: (row.human_action as HumanAction) ?? 'none',
riskContext: row.risk_context as string | null,
importedFrom: row.imported_from as string | null,
persona: row.persona as string | null,
inputText: (row.input_text as string | null | undefined) ?? null,
outputText: (row.output_text as string | null | undefined) ?? null,
};
}
}

View File

@@ -0,0 +1,104 @@
/**
* ReportGenerator — generates AI Act compliance audit reports.
*
* Produces JSON reports (PDF generation deferred to document tooling).
* Reports cover: compliance status, model inventory, oversight log,
* harvest provenance, and interaction counts.
*/
import type { InteractionStore } from './interaction-store.js';
import type { HarvestSourceStore } from '@waggle/hive-mind-core';
import { ComplianceStatusChecker } from './status-checker.js';
import type { AuditReport, AuditReportRequest, AIActRiskLevel } from './types.js';
const REPORT_VERSION = '1.0';
export interface ReportGeneratorDeps {
interactionStore: InteractionStore;
harvestStore: HarvestSourceStore;
getWorkspaceRisk?: (workspaceId: string) => AIActRiskLevel;
getWorkspaceName?: (workspaceId: string) => string;
/**
* Return the ISO timestamp of the last risk classification for this
* workspace, or null if the workspace has no persisted classification
* date (e.g., was created before the field existed).
*/
getWorkspaceRiskClassifiedAt?: (workspaceId: string) => string | null;
}
export class ReportGenerator {
private interactions: InteractionStore;
private harvest: HarvestSourceStore;
private getWorkspaceRisk: (id: string) => AIActRiskLevel;
private getWorkspaceName: (id: string) => string;
private getWorkspaceRiskClassifiedAt: (id: string) => string | null;
constructor(deps: ReportGeneratorDeps) {
this.interactions = deps.interactionStore;
this.harvest = deps.harvestStore;
this.getWorkspaceRisk = deps.getWorkspaceRisk ?? (() => 'minimal');
this.getWorkspaceName = deps.getWorkspaceName ?? ((id) => id);
this.getWorkspaceRiskClassifiedAt = deps.getWorkspaceRiskClassifiedAt ?? (() => null);
}
/** Generate a full audit report. */
generate(request: AuditReportRequest): AuditReport {
const checker = new ComplianceStatusChecker(this.interactions);
const complianceStatus = checker.check(request.workspaceId);
const report: AuditReport = {
report: {
version: REPORT_VERSION,
generatedAt: new Date().toISOString(),
period: { from: request.from, to: request.to },
generatedBy: 'Waggle OS',
},
workspace: request.workspaceId ? {
id: request.workspaceId,
name: this.getWorkspaceName(request.workspaceId),
riskLevel: this.getWorkspaceRisk(request.workspaceId),
riskClassifiedAt: this.getWorkspaceRiskClassifiedAt(request.workspaceId),
} : null,
complianceStatus,
modelInventory: [],
humanOversightLog: [],
harvestProvenance: [],
interactionCount: 0,
};
// Model inventory
if (request.include.models) {
report.modelInventory = this.interactions.getModelInventory(
request.from, request.to, request.workspaceId,
);
}
// Human oversight log
if (request.include.oversight) {
report.humanOversightLog = this.interactions.getOversightLog(
request.from, request.to, request.workspaceId,
);
}
// Harvest provenance
if (request.include.provenance) {
const sources = this.harvest.getAll();
report.harvestProvenance = sources.map(s => ({
source: s.displayName,
importedAt: s.lastSyncedAt ?? s.createdAt,
itemsImported: s.itemsImported,
framesCreated: s.framesCreated,
}));
}
// Interaction count
if (request.include.interactions) {
const interactions = this.interactions.getByDateRange(
request.from, request.to, request.workspaceId,
);
report.interactionCount = interactions.length;
}
return report;
}
}

View File

@@ -0,0 +1,175 @@
/**
* ComplianceStatusChecker — evaluates AI Act compliance per workspace.
*
* Checks:
* - Art. 12: Automatic event logging (ai_interactions count)
* - Art. 14: Human oversight (approval/denial actions recorded)
* - Art. 19: Log retention (oldest log >= 6 months ago)
* - Art. 26: Deployer monitoring (active monitors)
* - Art. 50: Model transparency (models disclosed)
*/
import type { InteractionStore } from './interaction-store.js';
import type { ComplianceStatus, ArticleStatus } from './types.js';
const SIX_MONTHS_MS = 180 * 24 * 60 * 60 * 1000;
export class ComplianceStatusChecker {
private store: InteractionStore;
constructor(store: InteractionStore) {
this.store = store;
}
/** Evaluate full compliance status for a workspace (or all workspaces if null). */
check(workspaceId?: string): ComplianceStatus {
const art12 = this.checkArt12(workspaceId);
const art14 = this.checkArt14(workspaceId);
const art19 = this.checkArt19();
const art26 = this.checkArt26();
const art50 = this.checkArt50(workspaceId);
const statuses = [art12.status, art14.status, art19.status, art26.status, art50.status];
const overall = statuses.includes('non-compliant') ? 'non-compliant'
: statuses.includes('warning') ? 'warning'
: 'compliant';
return {
overall,
art12Logging: art12,
art14Oversight: art14,
art19Retention: art19,
art26Monitoring: art26,
art50Transparency: art50,
};
}
/** Art. 12: Automatic recording of events. */
private checkArt12(workspaceId?: string): ArticleStatus & { totalInteractions: number } {
const total = this.store.count(workspaceId);
if (total === 0) {
// M11: if the DB has been active for >24h with zero interactions, escalate to non-compliant
const firstRun = this.store.getFirstRunAt();
const activeOver24h = firstRun
? (Date.now() - new Date(firstRun).getTime()) > 24 * 60 * 60 * 1000
: false;
return {
status: activeOver24h ? 'non-compliant' : 'warning',
detail: activeOver24h
? 'No interactions logged despite system being active for over 24 hours. Verify logging pipeline.'
: 'No interactions logged yet. Logging activates automatically on first AI interaction.',
totalInteractions: 0,
};
}
return {
status: 'compliant',
detail: `${total} interactions logged with full model, token, cost, and tool tracking.`,
totalInteractions: total,
};
}
/** Art. 14: Human oversight capability. */
private checkArt14(workspaceId?: string): ArticleStatus & { humanActions: number; approvalRate: number } {
const counts = this.store.getOversightCounts(workspaceId);
if (counts.total === 0) {
return {
status: 'compliant',
detail: 'Human oversight capabilities available (approval gates, tool deny lists). No oversight actions recorded yet.',
humanActions: 0,
approvalRate: 0,
};
}
const approvalRate = counts.total > 0
? Math.round((counts.approved / counts.total) * 100)
: 0;
return {
status: 'compliant',
detail: `${counts.total} human oversight actions: ${counts.approved} approved, ${counts.denied} denied, ${counts.modified} modified.`,
humanActions: counts.total,
approvalRate,
};
}
/** Art. 19: Log retention (minimum 6 months). */
private checkArt19(): ArticleStatus & { oldestLogDate: string | null; retentionDays: number } {
const oldest = this.store.getOldestTimestamp();
if (!oldest) {
return {
status: 'compliant',
detail: 'No logs to retain yet. Retention policy is permanent by default.',
oldestLogDate: null,
retentionDays: 0,
};
}
const oldestDate = new Date(oldest);
const now = new Date();
const retentionMs = now.getTime() - oldestDate.getTime();
const retentionDays = Math.floor(retentionMs / (24 * 60 * 60 * 1000));
// Review Critical #2: the previous expression was a tautology
// (`retentionMs >= SIX_MONTHS_MS || retentionDays < 180`) that covered every
// non-negative value of retentionDays. A deployment that pruned logs after 30
// days still reported compliant.
//
// Proper fix requires distinguishing 'system is young' from 'logs were pruned'.
// We track system age via MindDB's `meta.first_run_at` entry (set on schema init,
// backfilled for pre-existing DBs). If the system has been running for 180+ days
// but the oldest log is younger than that, something pruned the logs and we
// correctly report warning.
const firstRun = this.store.getFirstRunAt();
const systemAgeMs = firstRun ? now.getTime() - new Date(firstRun).getTime() : retentionMs;
const hasBeenRunning6Months = systemAgeMs >= SIX_MONTHS_MS;
const logsOlderThan6Months = retentionMs >= SIX_MONTHS_MS;
const meetsMinimum = !hasBeenRunning6Months || logsOlderThan6Months;
return {
status: meetsMinimum ? 'compliant' : 'warning',
detail: meetsMinimum
? hasBeenRunning6Months
? `Logs retained since ${oldest.split('T')[0]} (${retentionDays} days). Art. 19 minimum (180 days) met.`
: `Logs retained since ${oldest.split('T')[0]} (${retentionDays} days). System is still within its first 180 days — retention compliance will be enforceable after 2026-${(new Date(firstRun ?? now).getMonth() + 7).toString().padStart(2, '0')}.`
: `Oldest log: ${oldest.split('T')[0]} (${retentionDays} days) but system is ${Math.floor(systemAgeMs / (24 * 60 * 60 * 1000))} days old. Logs appear to have been pruned — EU AI Act Art. 19 requires 180-day minimum retention.`,
oldestLogDate: oldest,
retentionDays,
};
}
/** Art. 26: Deployer monitoring obligations. */
private checkArt26(): ArticleStatus & { activeMonitors: string[] } {
// Waggle always has these monitors active
const monitors = [
'cost_tracking', // CostTracker in packages/agent
'tool_logging', // Tool calls logged per interaction
'model_identification', // Model recorded per interaction
'persona_tracking', // Persona recorded per interaction
];
return {
status: 'compliant',
detail: `${monitors.length} active monitors: cost, tools, model ID, persona.`,
activeMonitors: monitors,
};
}
/** Art. 50: Transparency — models disclosed. */
private checkArt50(workspaceId?: string): ArticleStatus & { modelsDisclosed: boolean } {
const inventory = this.store.getModelInventory(undefined, undefined, workspaceId);
const modelsDisclosed = inventory.length > 0;
return {
status: 'compliant',
detail: modelsDisclosed
? `${inventory.length} model(s) in use, all identified in StatusBar and interaction logs.`
: 'Model identification active. Models will be disclosed on first interaction.',
modelsDisclosed,
};
}
}

View File

@@ -0,0 +1,242 @@
/**
* ComplianceTemplateStore — CRUD for user-editable compliance report templates.
*
* M-03: Templates let a user save a named "report shape" (which sections to include,
* a risk-class override, and optional org/footer text) and re-apply it to subsequent
* audit-report exports. Sections MERGE with the runtime selection (union semantics).
*
* Logo support is deferred to Bucket 2. See docs/plans/COMPLIANCE-AUDIT-2026-04-20.md.
*
* DDL is self-contained here (following HarvestRunStore pattern) rather than threaded
* through mind/schema.ts — compliance_templates is not part of the Art. 12 audit
* substrate and doesn't need append-only triggers or cross-layer migration tracking.
*/
import type { MindDB } from '@waggle/hive-mind-core';
import type {
AIActRiskLevel,
ComplianceTemplate,
ComplianceTemplateSections,
CreateComplianceTemplateInput,
UpdateComplianceTemplateInput,
} from './types.js';
const CREATE_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS compliance_templates (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
description TEXT,
sections_json TEXT NOT NULL DEFAULT '{}',
risk_classification TEXT CHECK (risk_classification IN ('minimal', 'limited', 'high-risk', 'unacceptable')),
org_name TEXT,
footer_text TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
)
`;
// TODO Bucket 2 (M-05 custom branding): add `logo_blob BLOB` + `logo_mime TEXT`
// columns. The `Edit templates` modal will grow a file-picker; renderComplianceReportPdf
// will render the blob into the PDF header when present.
const CREATE_INDEX_SQL = `
CREATE INDEX IF NOT EXISTS idx_compliance_templates_name ON compliance_templates (name)
`;
const DEFAULT_SECTIONS: ComplianceTemplateSections = {
interactions: true,
oversight: true,
models: true,
provenance: true,
riskAssessment: true,
fria: false,
};
/**
* Canonical name for the seeded KVARK template. Used for idempotency — the
* seed only fires when no row with this exact name exists.
*/
export const KVARK_TEMPLATE_NAME = 'KVARK Enterprise Audit';
export class ComplianceTemplateStore {
private db: MindDB;
constructor(db: MindDB) {
this.db = db;
this.ensureTable();
}
private ensureTable(): void {
const raw = this.db.getDatabase();
const existsRow = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='compliance_templates'",
).get();
if (existsRow) return;
raw.prepare(CREATE_TABLE_SQL).run();
raw.prepare(CREATE_INDEX_SQL).run();
}
create(input: CreateComplianceTemplateInput): ComplianceTemplate {
const name = input.name.trim();
if (!name) throw new Error('Template name is required');
const raw = this.db.getDatabase();
const result = raw.prepare(`
INSERT INTO compliance_templates (
name, description, sections_json, risk_classification, org_name, footer_text
) VALUES (?, ?, ?, ?, ?, ?)
`).run(
name,
input.description ?? null,
JSON.stringify(this.normalizeSections(input.sections)),
input.riskClassification ?? null,
input.orgName ?? null,
input.footerText ?? null,
);
return this.getById(Number(result.lastInsertRowid))!;
}
getById(id: number): ComplianceTemplate | null {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT * FROM compliance_templates WHERE id = ?').get(id) as
| Record<string, unknown>
| undefined;
return row ? this.rowToTemplate(row) : null;
}
list(): ComplianceTemplate[] {
const raw = this.db.getDatabase();
return (
raw
.prepare('SELECT * FROM compliance_templates ORDER BY updated_at DESC')
.all() as Record<string, unknown>[]
).map(r => this.rowToTemplate(r));
}
update(id: number, patch: UpdateComplianceTemplateInput): ComplianceTemplate | null {
const existing = this.getById(id);
if (!existing) return null;
const next: ComplianceTemplate = {
...existing,
name: patch.name !== undefined ? patch.name.trim() : existing.name,
description: patch.description !== undefined ? patch.description : existing.description,
sections: patch.sections !== undefined ? this.normalizeSections(patch.sections) : existing.sections,
riskClassification:
patch.riskClassification !== undefined ? patch.riskClassification : existing.riskClassification,
orgName: patch.orgName !== undefined ? patch.orgName : existing.orgName,
footerText: patch.footerText !== undefined ? patch.footerText : existing.footerText,
};
if (!next.name) throw new Error('Template name is required');
const raw = this.db.getDatabase();
raw.prepare(`
UPDATE compliance_templates SET
name = ?,
description = ?,
sections_json = ?,
risk_classification = ?,
org_name = ?,
footer_text = ?,
updated_at = datetime('now')
WHERE id = ?
`).run(
next.name,
next.description,
JSON.stringify(next.sections),
next.riskClassification,
next.orgName,
next.footerText,
id,
);
return this.getById(id);
}
/** Returns true if the row existed and was removed, false otherwise. */
delete(id: number): boolean {
const raw = this.db.getDatabase();
const info = raw.prepare('DELETE FROM compliance_templates WHERE id = ?').run(id);
return info.changes > 0;
}
/**
* Merge a template's section flags with runtime flags. Union semantics: a section
* is included if EITHER the template OR the runtime toggle turns it on. This keeps
* the template's role "additive" — it can never silently hide a section the user
* asked for in the current export.
*/
static mergeSections(
template: ComplianceTemplateSections,
runtime: ComplianceTemplateSections,
): ComplianceTemplateSections {
return {
interactions: template.interactions || runtime.interactions,
oversight: template.oversight || runtime.oversight,
models: template.models || runtime.models,
provenance: template.provenance || runtime.provenance,
riskAssessment: template.riskAssessment || runtime.riskAssessment,
fria: template.fria || runtime.fria,
};
}
private normalizeSections(s: Partial<ComplianceTemplateSections> | undefined): ComplianceTemplateSections {
return { ...DEFAULT_SECTIONS, ...(s ?? {}) };
}
/**
* Idempotent: seed the built-in "KVARK Enterprise Audit" template if no
* template with that exact name exists yet (M-06).
*
* The KVARK template is a sensible-default shape for sovereign enterprise
* deployments — every section on (including FRIA, since KVARK serves
* high-risk enterprise systems), risk pinned at `high-risk`, and org/footer
* text that signals the sovereign-deployment narrative.
*
* Custom KVARK-specific sections (per-department risk breakdown, IAM
* audit columns, data-residency attestation text) are deferred to
* Bucket 2 along with the logo field — the current template schema
* doesn't support custom sections yet.
*
* Returns the seeded template, or null if one already existed.
*/
seedKvarkTemplateIfMissing(): ComplianceTemplate | null {
const raw = this.db.getDatabase();
const existing = raw.prepare(
'SELECT id FROM compliance_templates WHERE name = ? LIMIT 1',
).get(KVARK_TEMPLATE_NAME) as { id: number } | undefined;
if (existing) return null;
return this.create({
name: KVARK_TEMPLATE_NAME,
description:
'Sovereign AI Act audit shape for enterprise on-prem deployments. ' +
'Includes all monitored articles (12/14/19/26/50) plus FRIA since KVARK ' +
'customers typically operate high-risk systems. Data never leaves the ' +
"customer perimeter; this template's org + footer text signal that " +
'posture directly on every exported report.',
sections: { ...DEFAULT_SECTIONS, fria: true },
riskClassification: 'high-risk',
orgName: 'KVARK Sovereign — Enterprise Deployment',
footerText:
'Confidential · EU AI Act attestation · KVARK sovereign deployment · Data remains within customer perimeter',
});
}
private rowToTemplate(row: Record<string, unknown>): ComplianceTemplate {
let sections: ComplianceTemplateSections;
try {
sections = this.normalizeSections(JSON.parse((row.sections_json as string) ?? '{}'));
} catch {
sections = { ...DEFAULT_SECTIONS };
}
return {
id: row.id as number,
name: row.name as string,
description: (row.description as string | null) ?? null,
sections,
riskClassification: (row.risk_classification as AIActRiskLevel | null) ?? null,
orgName: (row.org_name as string | null) ?? null,
footerText: (row.footer_text as string | null) ?? null,
createdAt: row.created_at as string,
updatedAt: row.updated_at as string,
};
}
}

View File

@@ -0,0 +1,201 @@
/**
* AI Act Compliance Types — interaction audit, risk classification, report format.
*/
// ── Risk Classification (Art. 26) ──
export type AIActRiskLevel = 'minimal' | 'limited' | 'high-risk' | 'unacceptable';
export type HumanAction = 'approved' | 'denied' | 'modified' | 'none';
// ── Interaction Audit (Art. 12) ──
export interface AIInteraction {
id: number;
timestamp: string;
workspaceId: string | null;
sessionId: string | null;
model: string;
provider: string;
inputTokens: number;
outputTokens: number;
costUsd: number;
toolsCalled: string[];
humanAction: HumanAction;
riskContext: string | null;
importedFrom: string | null;
persona: string | null;
// Review Critical #3 (compliance): EU AI Act Art. 12.1(a) requires recording the
// actual inputs and outputs of the system, not just token counts. Nullable because
// pre-existing DBs may have rows from before the columns were added.
inputText: string | null;
outputText: string | null;
}
export interface RecordInteractionInput {
workspaceId?: string;
sessionId?: string;
model: string;
provider: string;
inputTokens: number;
outputTokens: number;
costUsd: number;
toolsCalled?: string[];
humanAction?: HumanAction;
riskContext?: string;
importedFrom?: string;
persona?: string;
// Review Critical #3: optional because echo-mode interactions may not have meaningful
// content. Callers on the live agent path SHOULD pass these; record() does not reject
// absent values but the compliance status checker flags them as a gap.
inputText?: string;
outputText?: string;
}
// ── Compliance Status ──
export interface ArticleStatus {
status: 'compliant' | 'warning' | 'non-compliant';
detail: string;
}
export interface ComplianceStatus {
overall: 'compliant' | 'warning' | 'non-compliant';
art12Logging: ArticleStatus & { totalInteractions: number };
art14Oversight: ArticleStatus & { humanActions: number; approvalRate: number };
art19Retention: ArticleStatus & { oldestLogDate: string | null; retentionDays: number };
art26Monitoring: ArticleStatus & { activeMonitors: string[] };
art50Transparency: ArticleStatus & { modelsDisclosed: boolean };
}
// ── Audit Report ──
export interface ModelInventoryEntry {
model: string;
provider: string;
calls: number;
inputTokens: number;
outputTokens: number;
costUsd: number;
}
export interface OversightLogEntry {
timestamp: string;
action: HumanAction;
tool: string;
detail: string;
user?: string;
}
export interface HarvestProvenanceEntry {
source: string;
importedAt: string;
itemsImported: number;
framesCreated: number;
}
export interface AuditReportRequest {
workspaceId?: string;
from: string;
to: string;
format: 'json' | 'pdf' | 'both';
include: {
interactions: boolean;
oversight: boolean;
models: boolean;
provenance: boolean;
riskAssessment: boolean;
fria: boolean;
};
}
export interface AuditReport {
report: {
version: string;
generatedAt: string;
period: { from: string; to: string };
generatedBy: string;
};
workspace: {
id: string | null;
name: string;
riskLevel: AIActRiskLevel;
riskClassifiedAt: string | null;
} | null;
complianceStatus: ComplianceStatus;
modelInventory: ModelInventoryEntry[];
humanOversightLog: OversightLogEntry[];
harvestProvenance: HarvestProvenanceEntry[];
interactionCount: number;
}
// ── Compliance Templates (M-03) ──
//
// Templates let a user save a preferred audit-report shape (sections + risk class +
// org/footer overrides) and re-apply it to exports. Sections MERGE (union) with the
// user's runtime section toggles — a template only turns sections ON, never off.
// `riskClassification` overrides the workspace's risk level for that one report.
// Logo support is deferred to Bucket 2 (post-launch).
export interface ComplianceTemplateSections {
interactions: boolean;
oversight: boolean;
models: boolean;
provenance: boolean;
riskAssessment: boolean;
fria: boolean;
}
export interface ComplianceTemplate {
id: number;
name: string;
description: string | null;
sections: ComplianceTemplateSections;
riskClassification: AIActRiskLevel | null;
orgName: string | null;
footerText: string | null;
createdAt: string;
updatedAt: string;
}
export interface CreateComplianceTemplateInput {
name: string;
description?: string | null;
sections: ComplianceTemplateSections;
riskClassification?: AIActRiskLevel | null;
orgName?: string | null;
footerText?: string | null;
}
export interface UpdateComplianceTemplateInput {
name?: string;
description?: string | null;
sections?: ComplianceTemplateSections;
riskClassification?: AIActRiskLevel | null;
orgName?: string | null;
footerText?: string | null;
}
// ── Template auto-suggestion ──
export const TEMPLATE_RISK_MAP: Record<string, AIActRiskLevel> = {
'legal-review': 'high-risk',
'hr-management': 'high-risk',
'recruiting': 'high-risk',
'finance': 'high-risk',
'insurance': 'high-risk',
'credit-scoring': 'high-risk',
'sales-pipeline': 'limited',
'marketing-campaign': 'limited',
'research-project': 'minimal',
'code-review': 'minimal',
'product-launch': 'limited',
'agency-consulting': 'limited',
'blank': 'minimal',
'content-creation': 'minimal',
'customer-support': 'limited',
'data-analysis': 'limited',
'project-management': 'minimal',
'education': 'limited',
'healthcare': 'high-risk',
};

281
packages/core/src/config.ts Normal file
View File

@@ -0,0 +1,281 @@
import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';
import type { EmbeddingProviderConfig, EmbeddingProviderType } from '@waggle/hive-mind-core';
export interface ProviderEntry {
apiKey: string;
models: string[];
baseUrl?: string;
}
export interface TeamServerConfig {
url: string;
token?: string;
userId?: string;
displayName?: string;
}
export interface CliConfig {
allowlist?: string[];
}
interface ConfigData {
defaultModel: string;
providers: Record<string, ProviderEntry>;
mindPath?: string;
teamServer?: TeamServerConfig;
/** Governed CLI programs the agent may execute. */
cli?: CliConfig;
/** F8: Daily cost budget in dollars. null = no limit. */
dailyBudget?: number | null;
/** When true, exceeding dailyBudget blocks agent. When false, warns only. */
budgetHardCap?: boolean;
/** Model Pilot: fallback model when primary fails (429/500/timeout) */
fallbackModel?: string;
/** Model Pilot: budget-saver model when daily spend hits threshold */
budgetModel?: string;
/** Model Pilot: budget threshold as 0.0-1.0 fraction. Default 0.8 */
budgetThreshold?: number;
/** Agent Intelligence: max LLM iterations per conversation. Default 90. */
maxIterations?: number;
/** M2-7: Telemetry opt-in (default: false — privacy first) */
telemetryEnabled?: boolean;
/** M2-1: Embedding provider configuration */
embedding?: {
provider?: EmbeddingProviderType | 'auto';
ollamaUrl?: string;
ollamaModel?: string;
inprocessModel?: string;
};
/** Steal #6: on-demand relevance gating for MCP tools. */
mcpToolRetrieval?: {
enabled?: boolean;
threshold?: number;
topK?: number;
};
}
/** Resolved MCP tool-retrieval config (all fields present). */
export interface McpToolRetrievalSettings {
enabled: boolean;
threshold: number;
topK: number;
}
const DEFAULT_MODEL = 'claude-sonnet-4-6';
function getDefaultConfigDir(): string {
const home = process.env.HOME || process.env.USERPROFILE || os.homedir();
return path.join(home, '.waggle');
}
export class WaggleConfig {
private readonly configDir: string;
private readonly configPath: string;
private data: ConfigData;
constructor(configDir?: string) {
this.configDir = configDir ?? getDefaultConfigDir();
this.configPath = path.join(this.configDir, 'config.json');
// Ensure config directory exists
if (!fs.existsSync(this.configDir)) {
fs.mkdirSync(this.configDir, { recursive: true });
}
// Load existing config or use defaults
this.data = this.load();
}
private load(): ConfigData {
if (fs.existsSync(this.configPath)) {
const raw = fs.readFileSync(this.configPath, 'utf-8');
return JSON.parse(raw) as ConfigData;
}
return {
defaultModel: DEFAULT_MODEL,
providers: {},
};
}
save(): void {
fs.writeFileSync(this.configPath, JSON.stringify(this.data, null, 2), 'utf-8');
}
getDefaultModel(): string {
return this.data.defaultModel;
}
setDefaultModel(model: string): void {
this.data.defaultModel = model;
}
getProviders(): Record<string, ProviderEntry> {
return { ...(this.data.providers ?? {}) };
}
setProvider(name: string, entry: ProviderEntry): void {
if (!this.data.providers) this.data.providers = {};
this.data.providers[name] = entry;
}
removeProvider(name: string): void {
delete this.data.providers[name];
}
getMindPath(): string {
return this.data.mindPath ?? path.join(this.configDir, 'default.mind');
}
getConfigDir(): string {
return this.configDir;
}
// F8: Daily cost budget
getDailyBudget(): number | null {
return this.data.dailyBudget ?? null;
}
setDailyBudget(budget: number | null): void {
this.data.dailyBudget = budget;
}
getBudgetHardCap(): boolean {
return this.data.budgetHardCap ?? false;
}
setBudgetHardCap(enabled: boolean): void {
this.data.budgetHardCap = enabled;
}
// --- Model Pilot ---
getFallbackModel(): string | null {
return this.data.fallbackModel ?? null;
}
setFallbackModel(model: string): void {
this.data.fallbackModel = model;
}
clearFallbackModel(): void {
delete this.data.fallbackModel;
}
getBudgetModel(): string | null {
return this.data.budgetModel ?? null;
}
setBudgetModel(model: string): void {
this.data.budgetModel = model;
}
clearBudgetModel(): void {
delete this.data.budgetModel;
}
getBudgetThreshold(): number {
return this.data.budgetThreshold ?? 0.8;
}
setBudgetThreshold(threshold: number): void {
this.data.budgetThreshold = Math.max(0.5, Math.min(0.95, threshold));
}
// --- Agent Intelligence ---
getMaxIterations(): number {
return this.data.maxIterations ?? 90;
}
setMaxIterations(max: number): void {
this.data.maxIterations = Math.max(5, Math.min(500, max));
}
// --- Team Server (Phase 5) ---
getTeamServer(): TeamServerConfig | null {
return this.data.teamServer ?? null;
}
setTeamServer(config: TeamServerConfig): void {
this.data.teamServer = config;
}
clearTeamServer(): void {
delete this.data.teamServer;
}
isTeamConnected(): boolean {
return this.data.teamServer !== null && this.data.teamServer !== undefined && typeof this.data.teamServer.url === 'string' && this.data.teamServer.url.length > 0;
}
// --- Governed CLI access ---
getCliAllowlist(): string[] {
return [...(this.data.cli?.allowlist ?? [])];
}
setCliAllowlist(allowlist: string[]): void {
const seen = new Set<string>();
const next = allowlist.reduce<string[]>((result, entry) => {
const value = entry.trim();
const key = value.toLowerCase();
if (value && !seen.has(key)) {
seen.add(key);
result.push(value);
}
return result;
}, []);
this.data.cli = { ...(this.data.cli ?? {}), allowlist: next };
}
// --- Telemetry (M2-7) ---
getTelemetryEnabled(): boolean {
return this.data.telemetryEnabled ?? false;
}
setTelemetryEnabled(enabled: boolean): void {
this.data.telemetryEnabled = enabled;
this.save();
}
// --- Embedding Provider (M2-1) ---
getEmbeddingConfig(): EmbeddingProviderConfig {
const emb = this.data.embedding;
const config: EmbeddingProviderConfig = {
provider: (process.env.EMBEDDING_PROVIDER as EmbeddingProviderType | 'auto' | undefined) ?? emb?.provider ?? 'auto',
targetDimensions: 1024,
inprocess: {
model: process.env.EMBEDDING_MODEL ?? emb?.inprocessModel,
cacheDir: path.join(this.configDir, 'models'),
},
ollama: {
baseUrl: process.env.OLLAMA_HOST ?? emb?.ollamaUrl,
model: process.env.OLLAMA_EMBED_MODEL ?? emb?.ollamaModel,
},
// API keys injected separately from Vault — not stored in config.json
};
return config;
}
setEmbeddingProvider(provider: EmbeddingProviderType | 'auto'): void {
if (!this.data.embedding) this.data.embedding = {};
this.data.embedding.provider = provider;
}
// --- MCP tool retrieval (Steal #6) ---
/** Resolve MCP tool-retrieval settings, filling defaults (ON, threshold 20, top-k 10). */
getMcpToolRetrieval(): McpToolRetrievalSettings {
const cfg = this.data.mcpToolRetrieval;
return {
enabled: cfg?.enabled ?? true,
threshold: cfg?.threshold ?? 20,
topK: cfg?.topK ?? 10,
};
}
}

View File

@@ -0,0 +1,588 @@
/**
* Cron Store — SQLite persistence for solo cron schedules.
*
* Stores cron job definitions with expression validation via cron-parser,
* due-job queries, and run tracking. Follows the same pattern as
* InstallAuditStore — operates on the .mind DB with lazy table creation.
*/
import cronParser from 'cron-parser';
const { parseExpression } = cronParser;
import type { MindDB } from '@waggle/hive-mind-core';
// ── Types ──────────────────────────────────────────────────────────────
export type CronJobType = 'agent_task' | 'memory_consolidation' | 'workspace_health' | 'proactive' | 'prompt_optimization' | 'monthly_assessment' | 'connector_fetch' | 'loop';
export const VALID_JOB_TYPES: Set<string> = new Set([
'agent_task',
'memory_consolidation',
'workspace_health',
'proactive',
'prompt_optimization',
'monthly_assessment',
'connector_fetch',
// Loop v0: a stateful, memory-powered, report-only (L1) scheduled automation.
// Composes recall + maker (toolless LLM) + checker (LLMJudge) + memory write.
// job_type TEXT has no CHECK constraint, so this is additive — no migration.
'loop',
]);
export interface CronSchedule {
id: number;
name: string;
cron_expr: string;
job_type: CronJobType;
job_config: string;
workspace_id: string | null;
enabled: number; // SQLite integer boolean
last_run_at: string | null;
next_run_at: string | null;
created_at: string;
}
export interface CreateScheduleInput {
name: string;
cronExpr: string;
jobType: CronJobType;
jobConfig?: Record<string, unknown>;
workspaceId?: string;
enabled?: boolean;
}
export interface CronExecutionRow {
id: number;
schedule_id: number;
schedule_name: string;
executed_at: string;
duration_ms: number | null;
success: number; // SQLite integer boolean
result_summary: string | null;
error: string | null;
}
export interface CronRunLeaseRow {
id: number;
schedule_id: number;
schedule_name: string | null;
started_at: string;
pid: number | null;
}
export interface NotificationRow {
id: number;
title: string;
body: string;
category: string;
action_url: string | null;
read: number; // SQLite integer boolean
created_at: string;
}
// ── L2 assisted loops: durable "held action" approval queue ──────────────
// A held action is a self-contained proposed tool call drafted by a headless
// run (e.g. an assist-mode Loop) that needs a human's one-click approval before
// it executes. Durable so it survives a sidecar restart (the in-memory live
// approval Promise does not). Execute-on-approve, never mid-run suspend/resume.
export type PendingActionStatus = 'held' | 'approved' | 'denied' | 'executed' | 'failed' | 'expired';
export interface PendingActionRow {
id: string; // requestId (uuid)
workspace_id: string | null;
source: string; // e.g. 'loop:<scheduleId>'
tool_name: string;
args_json: string;
summary: string | null; // the maker's plain-language rationale
risk_level: string; // low | medium | high | critical
approval_class: string;
status: PendingActionStatus;
result_summary: string | null;
error: string | null;
created_at: string;
decided_at: string | null;
executed_at: string | null;
expires_at: string | null;
}
export interface SavePendingActionInput {
id: string;
workspaceId: string | null;
source: string;
toolName: string;
argsJson: string;
summary?: string;
riskLevel: string;
approvalClass: string;
expiresAt?: string;
}
// ── Table DDL ──────────────────────────────────────────────────────────
export const CRON_SCHEDULES_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS cron_schedules (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
cron_expr TEXT NOT NULL,
job_type TEXT NOT NULL,
job_config TEXT NOT NULL DEFAULT '{}',
workspace_id TEXT,
enabled INTEGER NOT NULL DEFAULT 1,
last_run_at TEXT,
next_run_at TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_cron_enabled_next ON cron_schedules (enabled, next_run_at);
`;
// W5.12: Cron execution history table
export const CRON_HISTORY_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS cron_execution_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
schedule_id INTEGER NOT NULL,
schedule_name TEXT NOT NULL,
executed_at TEXT NOT NULL DEFAULT (datetime('now')),
duration_ms INTEGER,
success INTEGER NOT NULL DEFAULT 1,
result_summary TEXT,
error TEXT,
FOREIGN KEY (schedule_id) REFERENCES cron_schedules(id)
);
CREATE INDEX IF NOT EXISTS idx_cron_history_schedule ON cron_execution_history (schedule_id, executed_at);
`;
export const CRON_RUN_LEASES_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS cron_run_leases (
id INTEGER PRIMARY KEY AUTOINCREMENT,
schedule_id INTEGER NOT NULL,
schedule_name TEXT,
started_at TEXT NOT NULL DEFAULT (datetime('now')),
pid INTEGER
);
`;
// W5.10: Notification persistence table
export const NOTIFICATIONS_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS notifications (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT NOT NULL,
body TEXT NOT NULL,
category TEXT NOT NULL DEFAULT 'system',
action_url TEXT,
read INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_notifications_created ON notifications (created_at);
`;
// L2: durable held-action approval queue
export const PENDING_ACTIONS_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS pending_actions (
id TEXT PRIMARY KEY,
workspace_id TEXT,
source TEXT NOT NULL,
tool_name TEXT NOT NULL,
args_json TEXT NOT NULL,
summary TEXT,
risk_level TEXT NOT NULL,
approval_class TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'held',
result_summary TEXT,
error TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
decided_at TEXT,
executed_at TEXT,
expires_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_pending_actions_status ON pending_actions (status, created_at);
`;
// ── Helpers ────────────────────────────────────────────────────────────
/** Parse a cron expression and return the next run time as ISO string. Throws on invalid expr. */
function computeNextRun(cronExpr: string): string {
const interval = parseExpression(cronExpr);
return interval.next().toISOString();
}
/**
* Validate a cron expression with the SAME parser create()/update() use.
* Returns null when parseable, else the parser's error message. Lets route
* layers (e.g. the automations /test preview) reject an expression the store
* would refuse to persist, without duplicating the parser dependency.
*/
export function cronExprError(cronExpr: string): string | null {
try {
parseExpression(cronExpr);
return null;
} catch (err) {
return err instanceof Error ? err.message : String(err);
}
}
// ── Store ──────────────────────────────────────────────────────────────
export class CronStore {
private db: MindDB;
constructor(db: MindDB) {
this.db = db;
this.ensureTable();
}
private ensureTable(): void {
const raw = this.db.getDatabase();
const exists = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='cron_schedules'",
).get();
if (!exists) {
raw.exec(CRON_SCHEDULES_TABLE_SQL);
}
// W5.12: Ensure cron execution history table
const histExists = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='cron_execution_history'",
).get();
if (!histExists) {
raw.exec(CRON_HISTORY_TABLE_SQL);
}
const leaseExists = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='cron_run_leases'",
).get();
if (!leaseExists) {
raw.exec(CRON_RUN_LEASES_TABLE_SQL);
}
// W5.10: Ensure notifications table
const notifExists = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='notifications'",
).get();
if (!notifExists) {
raw.exec(NOTIFICATIONS_TABLE_SQL);
}
// L2: Ensure pending_actions (held-action approval queue) table
const pendingExists = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='pending_actions'",
).get();
if (!pendingExists) {
raw.exec(PENDING_ACTIONS_TABLE_SQL);
}
}
/** Create a new cron schedule. Validates cron expression and job type. */
create(input: CreateScheduleInput): CronSchedule {
// Validate job type
if (!VALID_JOB_TYPES.has(input.jobType)) {
throw new Error(`Invalid job type: "${input.jobType}". Must be one of: ${[...VALID_JOB_TYPES].join(', ')}`);
}
// agent_task requires workspaceId
if (input.jobType === 'agent_task' && !input.workspaceId) {
throw new Error('agent_task jobs require a workspace ID');
}
// Validate cron expression (throws on invalid)
const nextRun = computeNextRun(input.cronExpr);
const raw = this.db.getDatabase();
const enabled = input.enabled === false ? 0 : 1;
const result = raw.prepare(`
INSERT INTO cron_schedules (name, cron_expr, job_type, job_config, workspace_id, enabled, next_run_at)
VALUES (?, ?, ?, ?, ?, ?, ?)
`).run(
input.name,
input.cronExpr,
input.jobType,
JSON.stringify(input.jobConfig ?? {}),
input.workspaceId ?? null,
enabled,
nextRun,
);
return raw.prepare(
'SELECT * FROM cron_schedules WHERE id = ?',
).get(result.lastInsertRowid) as CronSchedule;
}
/** List all schedules ordered by name. */
list(): CronSchedule[] {
return this.db.getDatabase().prepare(
'SELECT * FROM cron_schedules ORDER BY name ASC',
).all() as CronSchedule[];
}
/** Get a schedule by ID. Returns undefined if not found. */
getById(id: number): CronSchedule | undefined {
return this.db.getDatabase().prepare(
'SELECT * FROM cron_schedules WHERE id = ?',
).get(id) as CronSchedule | undefined;
}
/** Update a schedule. Recomputes next_run_at if cronExpr changes. Returns updated schedule. */
update(id: number, changes: Partial<Pick<CreateScheduleInput, 'name' | 'cronExpr' | 'jobConfig' | 'workspaceId' | 'enabled'>>): CronSchedule {
const setClauses: string[] = [];
const values: unknown[] = [];
if (changes.name !== undefined) {
setClauses.push('name = ?');
values.push(changes.name);
}
if (changes.cronExpr !== undefined) {
const nextRun = computeNextRun(changes.cronExpr);
setClauses.push('cron_expr = ?');
values.push(changes.cronExpr);
setClauses.push('next_run_at = ?');
values.push(nextRun);
}
if (changes.jobConfig !== undefined) {
setClauses.push('job_config = ?');
values.push(JSON.stringify(changes.jobConfig));
}
if (changes.workspaceId !== undefined) {
setClauses.push('workspace_id = ?');
values.push(changes.workspaceId);
}
if (changes.enabled !== undefined) {
setClauses.push('enabled = ?');
values.push(changes.enabled ? 1 : 0);
}
if (setClauses.length > 0) {
values.push(id);
this.db.getDatabase().prepare(
`UPDATE cron_schedules SET ${setClauses.join(', ')} WHERE id = ?`,
).run(...values);
}
return this.getById(id)!;
}
/** Delete a schedule by ID. */
delete(id: number): void {
this.db.getDatabase().prepare(
'DELETE FROM cron_schedules WHERE id = ?',
).run(id);
}
/** Get all enabled schedules whose next_run_at is in the past. */
getDue(): CronSchedule[] {
return this.db.getDatabase().prepare(
"SELECT * FROM cron_schedules WHERE enabled = 1 AND next_run_at <= datetime('now')",
).all() as CronSchedule[];
}
/** Mark a schedule as having just run. Updates last_run_at and recomputes next_run_at. */
markRun(id: number): void {
const schedule = this.getById(id);
if (!schedule) return;
const nextRun = computeNextRun(schedule.cron_expr);
this.db.getDatabase().prepare(
"UPDATE cron_schedules SET last_run_at = datetime('now'), next_run_at = ? WHERE id = ?",
).run(nextRun, id);
}
/** Clear all schedules (for testing). */
clear(): void {
this.db.getDatabase().prepare('DELETE FROM cron_schedules').run();
}
// ── W5.12: Cron Execution History ──────────────────────────────────
/** Record a cron job execution result. */
recordExecution(scheduleId: number, scheduleName: string, opts: {
executedAt?: string;
durationMs?: number;
success: boolean;
resultSummary?: string;
error?: string;
}): void {
this.db.getDatabase().prepare(
`INSERT INTO cron_execution_history (schedule_id, schedule_name, executed_at, duration_ms, success, result_summary, error)
VALUES (?, ?, COALESCE(?, datetime('now')), ?, ?, ?, ?)`,
).run(scheduleId, scheduleName, opts.executedAt ?? null, opts.durationMs ?? null, opts.success ? 1 : 0, opts.resultSummary ?? null, opts.error ?? null);
}
/** #17: count today's (UTC) executions for a schedule — ai_task daily cap. */
countExecutionsToday(scheduleId: number): number {
const row = this.db.getDatabase().prepare(
"SELECT COUNT(*) AS n FROM cron_execution_history WHERE schedule_id = ? AND executed_at >= date('now')",
).get(scheduleId) as { n: number };
return row.n;
}
/** Get execution history for a schedule (most recent first). */
getExecutionHistory(scheduleId: number, limit = 20): CronExecutionRow[] {
return this.db.getDatabase().prepare(
'SELECT * FROM cron_execution_history WHERE schedule_id = ? ORDER BY executed_at DESC LIMIT ?',
).all(scheduleId, limit) as CronExecutionRow[];
}
/** Get the most recent executions for boot-time failure-state recovery. */
getRecentExecutions(scheduleId: number, limit = 5): CronExecutionRow[] {
return this.db.getDatabase().prepare(
'SELECT * FROM cron_execution_history WHERE schedule_id = ? ORDER BY executed_at DESC, id DESC LIMIT ?',
).all(scheduleId, limit) as CronExecutionRow[];
}
/** Prune execution-history rows older than N days. recordExecution writes a
* row per tick (UX-Refactor Phase 3, Journey 16), so without retention the
* table grows unbounded (a per-minute job ≈ 525k rows/year). Mirrors
* optStore.pruneOlderThan(30). Returns the number of rows deleted. */
pruneExecutionHistory(olderThanDays: number): number {
const days = Math.max(1, Math.floor(olderThanDays));
const result = this.db.getDatabase().prepare(
"DELETE FROM cron_execution_history WHERE executed_at < datetime('now', ?)",
).run(`-${days} days`);
return result.changes;
}
// ── Interrupted-run leases ────────────────────────────────────────
/** Acquire a durable lease before a scheduled job begins execution. */
acquireRunLease(scheduleId: number, name: string, pid: number): number {
const result = this.db.getDatabase().prepare(
'INSERT INTO cron_run_leases (schedule_id, schedule_name, pid) VALUES (?, ?, ?)',
).run(scheduleId, name, pid);
return Number(result.lastInsertRowid);
}
/** Release a run lease after its scheduled job finishes. */
releaseRunLease(leaseId: number): void {
this.db.getDatabase().prepare(
'DELETE FROM cron_run_leases WHERE id = ?',
).run(leaseId);
}
/** List leases left behind by an interrupted process. */
listStaleRunLeases(): CronRunLeaseRow[] {
return this.db.getDatabase().prepare(
'SELECT * FROM cron_run_leases ORDER BY started_at ASC, id ASC',
).all() as CronRunLeaseRow[];
}
/** Clear all interrupted-run leases after boot recovery. */
clearRunLeases(): void {
this.db.getDatabase().prepare('DELETE FROM cron_run_leases').run();
}
// ── W5.10: Notification Persistence ────────────────────────────────
/** Save a notification. */
saveNotification(title: string, body: string, category = 'system', actionUrl?: string): number {
const result = this.db.getDatabase().prepare(
'INSERT INTO notifications (title, body, category, action_url) VALUES (?, ?, ?, ?)',
).run(title, body, category, actionUrl ?? null);
return Number(result.lastInsertRowid);
}
/** Get recent notifications (newest first). */
getNotifications(opts?: { since?: string; limit?: number; unreadOnly?: boolean }): NotificationRow[] {
const limit = opts?.limit ?? 50;
let sql = 'SELECT * FROM notifications';
const conditions: string[] = [];
const params: unknown[] = [];
if (opts?.since) { conditions.push('created_at > ?'); params.push(opts.since); }
if (opts?.unreadOnly) { conditions.push('read = 0'); }
if (conditions.length > 0) sql += ' WHERE ' + conditions.join(' AND ');
sql += ' ORDER BY created_at DESC LIMIT ?';
params.push(limit);
return this.db.getDatabase().prepare(sql).all(...params) as NotificationRow[];
}
/** Mark a notification as read. */
markNotificationRead(id: number): void {
this.db.getDatabase().prepare('UPDATE notifications SET read = 1 WHERE id = ?').run(id);
}
/** Mark all notifications as read. Returns the number of rows updated. */
markAllRead(): number {
const result = this.db.getDatabase().prepare('UPDATE notifications SET read = 1 WHERE read = 0').run();
return result.changes;
}
/** Count unread notifications. */
countUnread(): number {
const row = this.db.getDatabase().prepare('SELECT COUNT(*) as cnt FROM notifications WHERE read = 0').get() as { cnt: number };
return row.cnt;
}
// ── L2: Held-action approval queue ─────────────────────────────────
/** Enqueue a held action (status 'held'). Returns the persisted row. */
savePendingAction(input: SavePendingActionInput): PendingActionRow {
this.db.getDatabase().prepare(`
INSERT INTO pending_actions (id, workspace_id, source, tool_name, args_json, summary, risk_level, approval_class, status, expires_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'held', ?)
`).run(
input.id,
input.workspaceId,
input.source,
input.toolName,
input.argsJson,
input.summary ?? null,
input.riskLevel,
input.approvalClass,
input.expiresAt ?? null,
);
return this.getPendingAction(input.id)!;
}
/** List held actions (newest first) — defaults to the 'held' queue. */
listPendingActions(status: PendingActionStatus = 'held'): PendingActionRow[] {
return this.db.getDatabase().prepare(
'SELECT * FROM pending_actions WHERE status = ? ORDER BY created_at DESC',
).all(status) as PendingActionRow[];
}
/** Get one held action by id. */
getPendingAction(id: string): PendingActionRow | undefined {
return this.db.getDatabase().prepare(
'SELECT * FROM pending_actions WHERE id = ?',
).get(id) as PendingActionRow | undefined;
}
/**
* Atomically claim a held action — transition 'held' → 'approved' | 'denied'.
* This is the idempotency gate: exactly ONE caller wins (the `WHERE status =
* 'held'` makes it atomic in SQLite), so a double-approve / approve-after-deny
* is a no-op. Returns the updated row if THIS call won the claim, else
* undefined (already decided or missing).
*/
claimPendingAction(id: string, status: 'approved' | 'denied', decidedAt: string): PendingActionRow | undefined {
const result = this.db.getDatabase().prepare(
"UPDATE pending_actions SET status = ?, decided_at = ? WHERE id = ? AND status = 'held'",
).run(status, decidedAt, id);
if (result.changes === 0) return undefined;
return this.getPendingAction(id);
}
/**
* Record the terminal outcome of a claimed action ('approved' → 'executed' |
* 'failed'). Unconditional — the caller already won claimPendingAction(), so
* it owns the row and no further guard is needed.
*/
updatePendingActionResult(id: string, changes: { status: 'executed' | 'failed'; resultSummary?: string; error?: string; executedAt?: string }): void {
const sets: string[] = ['status = ?'];
const vals: unknown[] = [changes.status];
if (changes.resultSummary !== undefined) { sets.push('result_summary = ?'); vals.push(changes.resultSummary); }
if (changes.error !== undefined) { sets.push('error = ?'); vals.push(changes.error); }
if (changes.executedAt !== undefined) { sets.push('executed_at = ?'); vals.push(changes.executedAt); }
vals.push(id);
this.db.getDatabase().prepare(
`UPDATE pending_actions SET ${sets.join(', ')} WHERE id = ?`,
).run(...vals);
}
/** Flip held actions past their expires_at to 'expired'. Returns rows changed. */
expireStalePendingActions(): number {
const result = this.db.getDatabase().prepare(
"UPDATE pending_actions SET status = 'expired' WHERE status = 'held' AND expires_at IS NOT NULL AND expires_at < datetime('now')",
).run();
return result.changes;
}
/** Clear all pending actions (for testing). */
clearPendingActions(): void {
this.db.getDatabase().prepare('DELETE FROM pending_actions').run();
}
}

View File

@@ -0,0 +1,259 @@
/**
* FileIndexer — auto-index workspace files into their workspace mind (L-20).
*
* Design decisions (see docs/plans/FILE-TOOLS-AUDIT-2026-04-20.md):
* • Trigger: on-upload / on-overwrite (called from route handlers)
* • Target mind: per-workspace (the mind you pass in the constructor)
* • Granularity: 1 frame per file
* • Formats: .md, .markdown, .txt only for v1 (Bucket 2 adds PDF/DOCX)
* • Cleanup: re-index on overwrite, delete index on file remove
*
* A `file_index` table tracks the file_path → frame_id link so overwrites can
* swap the old frame cleanly and moves/deletes can locate the right frame
* without scanning every frame in the workspace.
*
* Large files are truncated to MAX_CONTENT_BYTES with a marker appended; the
* full file is still on disk, only the indexed frame is shortened. Keeps the
* mind DB size bounded even if a user drops a 5MB markdown dump.
*/
import { createHash } from 'node:crypto';
import path from 'node:path';
import type { MindDB } from '@waggle/hive-mind-core';
import { FrameStore } from '@waggle/hive-mind-core';
import { SessionStore } from '@waggle/hive-mind-core';
const FILE_INDEX_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS file_index (
file_path TEXT PRIMARY KEY,
frame_id INTEGER NOT NULL,
mime_type TEXT,
size_bytes INTEGER NOT NULL DEFAULT 0,
content_hash TEXT NOT NULL,
indexed_at TEXT NOT NULL DEFAULT (datetime('now'))
)
`;
const FILE_INDEX_IDX_SQL = `
CREATE INDEX IF NOT EXISTS idx_file_index_frame ON file_index (frame_id)
`;
const INDEX_SESSION_GOP_ID = 'session:file-index';
const INDEX_SESSION_PROJECT = 'file-indexer';
/** Maximum bytes of file content we store in a single frame. Beyond this, the
* content is truncated with a marker. Keeps mind size bounded on large dumps. */
export const MAX_CONTENT_BYTES = 64 * 1024;
/** Extensions we index for v1. PDF/DOCX/XLSX land in Bucket 2. */
const INDEXABLE_EXTENSIONS = new Set(['.md', '.markdown', '.txt']);
export interface FileIndexRow {
filePath: string;
frameId: number;
mimeType: string | null;
sizeBytes: number;
contentHash: string;
indexedAt: string;
}
export type FileIndexResult =
| { skipped: true; reason: 'unsupported_format' | 'unchanged' | 'empty' }
| { skipped: false; frameId: number; truncated: boolean };
export class FileIndexer {
private readonly db: MindDB;
private readonly frames: FrameStore;
private readonly sessions: SessionStore;
private cachedGopId: string | null = null;
constructor(db: MindDB) {
this.db = db;
this.frames = new FrameStore(db);
this.sessions = new SessionStore(db);
this.ensureTable();
}
private ensureTable(): void {
const raw = this.db.getDatabase();
const existsRow = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='file_index'",
).get();
if (existsRow) return;
raw.prepare(FILE_INDEX_TABLE_SQL).run();
raw.prepare(FILE_INDEX_IDX_SQL).run();
}
/** True iff the path's extension is one we index in v1. */
static shouldIndex(filePath: string): boolean {
const ext = path.extname(filePath).toLowerCase();
return INDEXABLE_EXTENSIONS.has(ext);
}
private ensureSession(): string {
if (this.cachedGopId) return this.cachedGopId;
const session = this.sessions.ensure(
INDEX_SESSION_GOP_ID,
INDEX_SESSION_PROJECT,
'Auto-indexed workspace files',
);
this.cachedGopId = session.gop_id;
return session.gop_id;
}
/**
* Index a file. Idempotent on unchanged content (same hash → skip). On
* overwrite with new content, deletes the old frame and inserts a fresh one.
*/
indexFile(filePath: string, content: Buffer, mime?: string): FileIndexResult {
if (!FileIndexer.shouldIndex(filePath)) {
return { skipped: true, reason: 'unsupported_format' };
}
if (content.length === 0) {
// Still remove any stale index so an emptied file doesn't leave an
// orphan frame pointing at old content.
this.removeFile(filePath);
return { skipped: true, reason: 'empty' };
}
const hash = createHash('sha256').update(content).digest('hex');
const raw = this.db.getDatabase();
const existingRow = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(filePath) as
| Record<string, unknown>
| undefined;
if (existingRow && existingRow.content_hash === hash) {
return { skipped: true, reason: 'unchanged' };
}
const normalized = this.normalizeText(content);
const truncated = content.length > MAX_CONTENT_BYTES;
const frameBody = this.buildFrameContent(filePath, normalized, truncated, mime);
const gopId = this.ensureSession();
// Atomicity contract (L-20 BLOCKER-1 fix): createIFrame → (conditional
// old-frame delete) → UPDATE/INSERT on file_index must commit as one unit.
// If any step throws mid-callback, better-sqlite3 rolls back the new frame
// and the index-row write, leaving the table in its pre-call state. Without
// this, a crash between the delete and the UPDATE would leave file_index
// pointing at a deleted frame_id (dangling) or an uncommitted new frame
// orphaned in the frames table.
const mutate = raw.transaction(() => {
const frame = this.frames.createIFrame(gopId, frameBody, 'normal', 'system');
if (existingRow) {
// Overwrite path: delete the old frame before we swap the row, unless
// the dedup coalesced two paths onto the same frame (leave that frame
// alone in that case — the other path still references it).
const oldFrameId = Number(existingRow.frame_id);
if (oldFrameId !== frame.id) {
const otherRef = raw.prepare('SELECT 1 FROM file_index WHERE frame_id = ? AND file_path != ? LIMIT 1').get(oldFrameId, filePath);
if (!otherRef) {
this.frames.delete(oldFrameId);
}
}
raw.prepare(`
UPDATE file_index SET
frame_id = ?,
mime_type = ?,
size_bytes = ?,
content_hash = ?,
indexed_at = datetime('now')
WHERE file_path = ?
`).run(frame.id, mime ?? null, content.length, hash, filePath);
} else {
raw.prepare(`
INSERT INTO file_index (file_path, frame_id, mime_type, size_bytes, content_hash)
VALUES (?, ?, ?, ?, ?)
`).run(filePath, frame.id, mime ?? null, content.length, hash);
}
return frame.id;
});
const frameId = mutate();
return { skipped: false, frameId, truncated };
}
/** Remove a file's frame + index row. Returns true if anything was removed. */
removeFile(filePath: string): boolean {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT frame_id FROM file_index WHERE file_path = ?').get(filePath) as
| { frame_id: number }
| undefined;
if (!row) return false;
const otherRef = raw.prepare('SELECT 1 FROM file_index WHERE frame_id = ? AND file_path != ? LIMIT 1').get(row.frame_id, filePath);
if (!otherRef) {
this.frames.delete(row.frame_id);
}
raw.prepare('DELETE FROM file_index WHERE file_path = ?').run(filePath);
return true;
}
/**
* Update the recorded path for a moved file. The frame content itself still
* carries the old path in its header — that's acceptable for v1 since the
* frame body is already the file's own content (not a description). A later
* reader sees the stale header only if they dig into raw frame content; the
* index table, which is what every other code path queries, is correct.
*/
moveFile(from: string, to: string): boolean {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(from) as
| Record<string, unknown>
| undefined;
if (!row) return false;
// If `to` already indexed, drop its entry (file was replaced by the move).
raw.prepare('DELETE FROM file_index WHERE file_path = ?').run(to);
raw.prepare('UPDATE file_index SET file_path = ?, indexed_at = datetime(\'now\') WHERE file_path = ?').run(to, from);
return true;
}
getRow(filePath: string): FileIndexRow | null {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(filePath) as
| Record<string, unknown>
| undefined;
if (!row) return null;
return {
filePath: row.file_path as string,
frameId: row.frame_id as number,
mimeType: (row.mime_type as string | null) ?? null,
sizeBytes: row.size_bytes as number,
contentHash: row.content_hash as string,
indexedAt: row.indexed_at as string,
};
}
listAll(): FileIndexRow[] {
const raw = this.db.getDatabase();
const rows = raw.prepare('SELECT * FROM file_index ORDER BY indexed_at DESC').all() as Record<string, unknown>[];
return rows.map(r => ({
filePath: r.file_path as string,
frameId: r.frame_id as number,
mimeType: (r.mime_type as string | null) ?? null,
sizeBytes: r.size_bytes as number,
contentHash: r.content_hash as string,
indexedAt: r.indexed_at as string,
}));
}
private normalizeText(content: Buffer): string {
// UTF-8 decode. Malformed bytes become U+FFFD; acceptable for index use.
const full = content.toString('utf-8');
if (Buffer.byteLength(full, 'utf-8') <= MAX_CONTENT_BYTES) return full;
// Truncate in character-safe chunks. Walk down until we're under the byte
// budget; avoids cutting a multi-byte character in half.
let candidate = full.slice(0, MAX_CONTENT_BYTES);
while (Buffer.byteLength(candidate, 'utf-8') > MAX_CONTENT_BYTES && candidate.length > 0) {
candidate = candidate.slice(0, candidate.length - 1);
}
return candidate;
}
private buildFrameContent(filePath: string, body: string, truncated: boolean, mime?: string): string {
const header = `[FILE: ${filePath}${mime ? ` · ${mime}` : ''}]`;
const suffix = truncated ? `\n\n[…truncated at ${MAX_CONTENT_BYTES} bytes for indexing]` : '';
return `${header}\n\n${body}${suffix}`;
}
}

View File

@@ -0,0 +1,639 @@
/**
* FileStore — workspace file storage abstraction.
*
* Provides a clean interface for reading/writing workspace files that abstracts
* over the underlying storage backend. Two implementations:
*
* - LocalFileStore: manages files in ~/.waggle/workspaces/{id}/files/ (virtual storage)
* - LinkedDirStore: reads/writes to workspace.directory (linked to external folder)
* - S3FileStore: MinIO/S3-backed storage for team/cloud deployments (lazy SDK import)
*
* All operations enforce path traversal protection — files cannot escape
* the workspace boundary.
*/
import fs from 'node:fs';
import path from 'node:path';
import { glob } from 'glob';
// Type-only import: erased at compile time, so it does not force @aws-sdk/client-s3
// (a devDependency) into the runtime bundle for local-only users.
import type { S3Client } from '@aws-sdk/client-s3';
// ── Interface ───────────────────────────────────────────────────────
export interface FileEntry {
name: string;
path: string; // relative to workspace root
size: number;
modified: string; // ISO timestamp
isDirectory: boolean;
}
export interface StorageInfo {
usedBytes: number;
fileCount: number;
storageType: 'virtual' | 'linked';
}
export interface FileStore {
// Read
readFile(relativePath: string): Promise<Buffer>;
listFiles(directory?: string): Promise<FileEntry[]>;
searchFiles(pattern: string): Promise<FileEntry[]>;
// Write
writeFile(relativePath: string, content: Buffer | string): Promise<void>;
deleteFile(relativePath: string): Promise<void>;
moveFile(from: string, to: string): Promise<void>;
// Meta
getStorageInfo(): Promise<StorageInfo>;
getRootPath(): string;
getStorageType(): 'virtual' | 'linked';
}
// ── Path safety ─────────────────────────────────────────────────────
/**
* Directory segments that, anywhere in a path, almost always hold secrets.
* Matched (normalized, case-insensitive) against each path segment.
*/
const SENSITIVE_DIR_SEGMENTS = new Set([
'.ssh', '.aws', '.gnupg', '.gpg', '.docker', '.kube', '.azure', '.terraform', '.terraform.d',
]);
/** Exact basenames (normalized) that are secret material. */
const SENSITIVE_BASENAMES = new Set([
'id_rsa', 'id_dsa', 'id_ecdsa', 'id_ed25519', 'authorized_keys', 'known_hosts',
'.netrc', '.pgpass', '.npmrc', '.pypirc', '.git-credentials',
'credentials', 'credentials.json', 'service-account.json',
'terraform.tfstate', 'terraform.tfstate.backup',
]);
/** Extensions that are (almost always) private-key material. */
const SENSITIVE_EXTENSIONS = new Set(['.pem']);
/** Backup/copy suffixes — strip and re-test the base (id_rsa.bak → id_rsa). */
const BACKUP_SUFFIX_RE = /\.(bak|old|backup|orig|copy|save|swp)$/i;
/** `.env` files are secrets — but the documented, checked-in templates are not. */
const ENV_TEMPLATE_ALLOW = new Set(['.env.example', '.env.sample', '.env.template', '.env.dist', '.env.defaults']);
/**
* Normalize one path segment to the name the OS will actually open: lower-case
* (case-insensitive FS), strip a Windows NTFS alternate-data-stream suffix
* (`id_rsa::$DATA` → `id_rsa`) and any trailing dots/spaces (`id_rsa.`, `.env `
* → the base) which Windows silently removes when opening.
*/
function normalizeSegment(seg: string): string {
return seg.toLowerCase().replace(/::.*$/, '').replace(/[. ]+$/, '');
}
function isSensitiveBase(base: string): boolean {
if (SENSITIVE_BASENAMES.has(base)) return true;
const dot = base.lastIndexOf('.');
if (dot > 0 && SENSITIVE_EXTENSIONS.has(base.slice(dot))) return true;
if (base === '.env' || base.startsWith('.env.')) return !ENV_TEMPLATE_ALLOW.has(base);
return false;
}
/**
* True when a path points at well-known secret material (SSH/GPG keys, cloud
* credentials, dotenv files, terraform state, …). Used to deny reads/writes
* inside LINKED external folders so an agent given a project directory cannot
* exfiltrate or clobber the user's secrets.
*
* It is a BLOCKLIST (defense-in-depth), not a sandbox: it raises the bar against
* obvious secrets but cannot enumerate every secret a home dir holds. Conservative
* on extensions (only *.pem) to avoid denying legitimate files. Path-separator
* agnostic; segments are normalized for case + Windows ADS/trailing-char tricks.
*/
export function isSensitiveFilePath(relativePath: string): boolean {
const segments = relativePath.replace(/\\/g, '/').split('/').map(normalizeSegment).filter(Boolean);
if (segments.length === 0) return false;
for (const seg of segments) {
if (SENSITIVE_DIR_SEGMENTS.has(seg)) return true;
}
const base = segments[segments.length - 1];
if (isSensitiveBase(base)) return true;
if (BACKUP_SUFFIX_RE.test(base) && isSensitiveBase(base.replace(BACKUP_SUFFIX_RE, ''))) return true;
return false;
}
interface ResolveSafeOptions {
/** Reject paths flagged by isSensitiveFilePath (linked external dirs only). */
denySensitive?: boolean;
}
/** realpathSync, falling back to the input if it can't be resolved (e.g. not created yet). */
function safeRealpath(p: string): string {
try { return fs.realpathSync(p); } catch { return p; }
}
/** Walk up to the deepest ancestor of `p` that exists on disk. */
function deepestExisting(p: string): string {
let cur = p;
while (cur !== path.dirname(cur) && !fs.existsSync(cur)) cur = path.dirname(cur);
return cur;
}
/**
* Resolve `relativePath` under `root` and assert it cannot escape the boundary.
*
* Two layers:
* 1. LEXICAL segment-boundary containment — the resolved path is the root or
* sits under `root + sep` (a bare `startsWith(root)` wrongly admits a sibling
* like `${root}-evil/secret`).
* 2. SYMLINK-aware containment — `path.resolve` is lexical but `fs.*` follows
* symlinks, so we realpath the deepest EXISTING ancestor of the target and
* re-check it is still under the realpath'd root. This blocks a benign-named
* symlink that points outside the boundary, while still permitting in-root
* symlinks (e.g. monorepo package links). The sensitive-file deny then runs
* on BOTH the lexical and the real in-root path, so an in-root symlink to a
* secret (`alias → ./.env`) cannot launder it past a benign basename.
*/
function resolveSafe(root: string, relativePath: string, opts: ResolveSafeOptions = {}): string {
const resolvedRoot = path.resolve(root);
const resolved = path.resolve(resolvedRoot, relativePath);
if (resolved !== resolvedRoot && !resolved.startsWith(resolvedRoot + path.sep)) {
throw new Error(`Path traversal denied: ${relativePath}`);
}
// Symlink-aware containment only applies once the root exists on disk — if it
// doesn't, nothing inside it exists to be a symlink, and walking the deepest
// existing ancestor above the (not-yet-created) root would compare against an
// unrelated real dir (e.g. an OS temp-dir symlink). Lexical containment holds.
const realRoot = safeRealpath(resolvedRoot);
let realTarget = realRoot;
if (fs.existsSync(resolvedRoot)) {
realTarget = safeRealpath(deepestExisting(resolved));
if (realTarget !== realRoot && !realTarget.startsWith(realRoot + path.sep)) {
throw new Error(`Path traversal denied (symlink): ${relativePath}`);
}
}
if (opts.denySensitive) {
const lexicalRel = path.relative(resolvedRoot, resolved);
const realRel = path.relative(realRoot, realTarget);
if (isSensitiveFilePath(lexicalRel) || isSensitiveFilePath(realRel)) {
throw new Error(`Access to sensitive file denied: ${relativePath}`);
}
}
return resolved;
}
/** Keep only glob matches that resolve back inside `root` (glob `../` patterns can escape cwd). */
function containGlobMatches(root: string, matches: string[]): string[] {
const resolvedRoot = path.resolve(root);
return matches.filter(m => {
const abs = path.resolve(resolvedRoot, m);
return abs === resolvedRoot || abs.startsWith(resolvedRoot + path.sep);
});
}
/**
* Convert a `*`/`?` glob to a RegExp that is SAFE against catastrophic
* backtracking (ReDoS). Every regex metacharacter is escaped first, so the only
* specials left are the two wildcards we re-introduce (`.*`, `.`) — neither can
* nest a quantifier, so a hostile pattern like `(a+)+x` becomes a harmless
* literal. Used by S3FileStore.searchFiles where the pattern is caller-controlled.
* Unanchored (substring match) to preserve the prior behavior.
*/
function globToRegExp(pattern: string): RegExp {
const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const body = escaped.replace(/\\\*/g, '.*').replace(/\\\?/g, '.');
return new RegExp(body, 'i');
}
/**
* Reject an S3 key relative-path that would escape its workspace prefix. S3 keys
* are literal, but MinIO (path-style, filesystem-backed) can normalize `..`, so a
* `..` segment could cross into a sibling workspace's prefix. Makes the file
* header's "all operations enforce path traversal protection" true for S3 too.
*/
function assertSafeRelativeKey(relativePath: string): void {
const segments = relativePath.replace(/\\/g, '/').split('/');
if (segments.some(s => s === '..')) {
throw new Error(`Path traversal denied: ${relativePath}`);
}
}
function ensureDir(dirPath: string): void {
if (!fs.existsSync(dirPath)) {
fs.mkdirSync(dirPath, { recursive: true });
}
}
// ── LocalFileStore (virtual workspace storage) ──────────────────────
export class LocalFileStore implements FileStore {
private readonly root: string;
constructor(dataDir: string, workspaceId: string) {
this.root = path.join(dataDir, 'workspaces', workspaceId, 'files');
}
getRootPath(): string { return this.root; }
getStorageType(): 'virtual' { return 'virtual'; }
async readFile(relativePath: string): Promise<Buffer> {
const fullPath = resolveSafe(this.root, relativePath);
return fs.readFileSync(fullPath);
}
async listFiles(directory?: string): Promise<FileEntry[]> {
const dir = directory ? resolveSafe(this.root, directory) : this.root;
if (!fs.existsSync(dir)) return [];
const entries = fs.readdirSync(dir, { withFileTypes: true });
return entries.map(entry => {
const fullPath = path.join(dir, entry.name);
const stat = fs.statSync(fullPath);
return {
name: entry.name,
path: path.relative(this.root, fullPath).replace(/\\/g, '/'),
size: stat.size,
modified: stat.mtime.toISOString(),
isDirectory: entry.isDirectory(),
};
});
}
async searchFiles(pattern: string): Promise<FileEntry[]> {
if (!fs.existsSync(this.root)) return [];
const matches = await glob(pattern, {
cwd: this.root,
nodir: true,
ignore: ['node_modules/**', '.git/**'],
});
// A glob pattern with `../` can escape cwd — keep only matches inside root.
return containGlobMatches(this.root, matches).slice(0, 200).map(match => {
const fullPath = path.join(this.root, match);
try {
const stat = fs.statSync(fullPath);
return {
name: path.basename(match),
path: match.replace(/\\/g, '/'),
size: stat.size,
modified: stat.mtime.toISOString(),
isDirectory: false,
};
} catch {
return { name: path.basename(match), path: match, size: 0, modified: '', isDirectory: false };
}
});
}
async writeFile(relativePath: string, content: Buffer | string): Promise<void> {
const fullPath = resolveSafe(this.root, relativePath);
// Lazy directory creation
ensureDir(path.dirname(fullPath));
fs.writeFileSync(fullPath, content);
}
async deleteFile(relativePath: string): Promise<void> {
const fullPath = resolveSafe(this.root, relativePath);
if (fs.existsSync(fullPath)) {
const stat = fs.statSync(fullPath);
if (stat.isDirectory()) {
fs.rmSync(fullPath, { recursive: true });
} else {
fs.unlinkSync(fullPath);
}
}
}
async moveFile(from: string, to: string): Promise<void> {
const fromPath = resolveSafe(this.root, from);
const toPath = resolveSafe(this.root, to);
ensureDir(path.dirname(toPath));
fs.renameSync(fromPath, toPath);
}
async getStorageInfo(): Promise<StorageInfo> {
if (!fs.existsSync(this.root)) {
return { usedBytes: 0, fileCount: 0, storageType: 'virtual' };
}
let usedBytes = 0;
let fileCount = 0;
const walk = (dir: string) => {
try {
const entries = fs.readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
walk(fullPath);
} else {
try {
usedBytes += fs.statSync(fullPath).size;
fileCount++;
} catch { /* skip unreadable */ }
}
}
} catch { /* skip unreadable dirs */ }
};
walk(this.root);
return { usedBytes, fileCount, storageType: 'virtual' };
}
}
// ── LinkedDirStore (linked to external directory) ───────────────────
export class LinkedDirStore implements FileStore {
private readonly root: string;
constructor(directory: string) {
this.root = directory;
}
getRootPath(): string { return this.root; }
getStorageType(): 'linked' { return 'linked'; }
// Linked stores point at a REAL external folder (a code project, even the home
// dir), so every path op denies well-known secret files — the agent cannot
// read or clobber ~/.ssh, .env, cloud credentials, etc. (LocalFileStore is a
// sandboxed virtual dir and needs no such deny.)
private static readonly DENY = { denySensitive: true } as const;
async readFile(relativePath: string): Promise<Buffer> {
const fullPath = resolveSafe(this.root, relativePath, LinkedDirStore.DENY);
return fs.readFileSync(fullPath);
}
async listFiles(directory?: string): Promise<FileEntry[]> {
const dir = directory ? resolveSafe(this.root, directory, LinkedDirStore.DENY) : this.root;
if (!fs.existsSync(dir)) return [];
const entries = fs.readdirSync(dir, { withFileTypes: true });
return entries
.filter(e => !e.name.startsWith('.') && e.name !== 'node_modules')
// Don't disclose the existence/size/mtime of non-dot secrets (credentials.json,
// id_rsa, known_hosts, …) — symmetric with searchFiles + the read deny.
.filter(e => !isSensitiveFilePath(e.name))
.map(entry => {
const fullPath = path.join(dir, entry.name);
try {
const stat = fs.statSync(fullPath);
return {
name: entry.name,
path: path.relative(this.root, fullPath).replace(/\\/g, '/'),
size: stat.size,
modified: stat.mtime.toISOString(),
isDirectory: entry.isDirectory(),
};
} catch {
return { name: entry.name, path: entry.name, size: 0, modified: '', isDirectory: false };
}
});
}
async searchFiles(pattern: string): Promise<FileEntry[]> {
const matches = await glob(pattern, {
cwd: this.root,
nodir: true,
ignore: ['node_modules/**', '.git/**', 'dist/**', 'build/**', '**/.ssh/**', '**/.aws/**', '**/.gnupg/**'],
});
// Contain `../`-escaping globs to the root, then drop any secret a creative
// pattern still matched — search never discloses the existence/path of secrets.
return containGlobMatches(this.root, matches).filter(m => !isSensitiveFilePath(m)).slice(0, 200).map(match => {
const fullPath = path.join(this.root, match);
try {
const stat = fs.statSync(fullPath);
return {
name: path.basename(match),
path: match.replace(/\\/g, '/'),
size: stat.size,
modified: stat.mtime.toISOString(),
isDirectory: false,
};
} catch {
return { name: path.basename(match), path: match, size: 0, modified: '', isDirectory: false };
}
});
}
async writeFile(relativePath: string, content: Buffer | string): Promise<void> {
const fullPath = resolveSafe(this.root, relativePath, LinkedDirStore.DENY);
ensureDir(path.dirname(fullPath));
fs.writeFileSync(fullPath, content);
}
async deleteFile(relativePath: string): Promise<void> {
const fullPath = resolveSafe(this.root, relativePath, LinkedDirStore.DENY);
if (fs.existsSync(fullPath)) {
fs.unlinkSync(fullPath);
}
}
async moveFile(from: string, to: string): Promise<void> {
const fromPath = resolveSafe(this.root, from, LinkedDirStore.DENY);
const toPath = resolveSafe(this.root, to, LinkedDirStore.DENY);
ensureDir(path.dirname(toPath));
fs.renameSync(fromPath, toPath);
}
async getStorageInfo(): Promise<StorageInfo> {
if (!fs.existsSync(this.root)) {
return { usedBytes: 0, fileCount: 0, storageType: 'linked' };
}
let usedBytes = 0;
let fileCount = 0;
const walk = (dir: string, depth = 0) => {
if (depth > 5) return; // Don't walk too deep in linked dirs
try {
const entries = fs.readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
if (entry.name === 'node_modules' || entry.name === '.git') continue;
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
walk(fullPath, depth + 1);
} else {
try {
usedBytes += fs.statSync(fullPath).size;
fileCount++;
} catch { /* skip */ }
}
}
} catch { /* skip */ }
};
walk(this.root);
return { usedBytes, fileCount, storageType: 'linked' };
}
}
// ── S3FileStore (MinIO/S3 for team workspace storage) ───────────────
export interface S3Config {
endpoint: string;
bucket: string;
accessKey: string;
secretKey: string;
prefix: string; // e.g., "workspaces/{id}/"
region?: string;
}
export class S3FileStore implements FileStore {
private config: S3Config;
// Lazily constructed S3 client; the SDK is imported on first use to avoid
// bundling it for local-only users.
private client: S3Client | undefined;
constructor(config: S3Config) {
this.config = config;
}
private async getClient(): Promise<S3Client> {
if (this.client) return this.client;
const { S3Client } = await import('@aws-sdk/client-s3');
this.client = new S3Client({
endpoint: `http://${this.config.endpoint}`,
region: this.config.region ?? 'us-east-1',
credentials: {
accessKeyId: this.config.accessKey,
secretAccessKey: this.config.secretKey,
},
forcePathStyle: true, // Required for MinIO
});
return this.client;
}
getRootPath(): string { return `s3://${this.config.bucket}/${this.config.prefix}`; }
getStorageType(): 'virtual' | 'linked' { return 'virtual'; }
async readFile(relativePath: string): Promise<Buffer> {
assertSafeRelativeKey(relativePath);
const client = await this.getClient();
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
const key = this.config.prefix + relativePath;
const response = await client.send(new GetObjectCommand({
Bucket: this.config.bucket,
Key: key,
}));
const body = response.Body;
if (!body) throw new Error(`S3 object has no body: ${key}`);
const chunks: Uint8Array[] = [];
// In Node.js the S3 streaming body is an async-iterable readable stream.
for await (const chunk of body as AsyncIterable<Uint8Array>) chunks.push(chunk);
return Buffer.concat(chunks);
}
async writeFile(relativePath: string, content: Buffer | string): Promise<void> {
assertSafeRelativeKey(relativePath);
const client = await this.getClient();
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
const key = this.config.prefix + relativePath;
await client.send(new PutObjectCommand({
Bucket: this.config.bucket,
Key: key,
Body: typeof content === 'string' ? Buffer.from(content) : content,
}));
}
async deleteFile(relativePath: string): Promise<void> {
assertSafeRelativeKey(relativePath);
const client = await this.getClient();
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3');
const key = this.config.prefix + relativePath;
await client.send(new DeleteObjectCommand({
Bucket: this.config.bucket,
Key: key,
}));
}
async listFiles(directory?: string): Promise<FileEntry[]> {
if (directory) assertSafeRelativeKey(directory);
const client = await this.getClient();
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
const pfx = this.config.prefix + (directory ? directory + '/' : '');
const response = await client.send(new ListObjectsV2Command({
Bucket: this.config.bucket,
Prefix: pfx,
Delimiter: '/',
}));
const entries: FileEntry[] = [];
for (const obj of response.Contents ?? []) {
if (!obj.Key) continue;
const relPath = obj.Key.slice(this.config.prefix.length);
entries.push({
name: relPath.split('/').pop() ?? relPath,
path: relPath,
size: obj.Size ?? 0,
modified: obj.LastModified?.toISOString() ?? '',
isDirectory: false,
});
}
for (const cpfx of response.CommonPrefixes ?? []) {
if (!cpfx.Prefix) continue;
const relPath = cpfx.Prefix.slice(this.config.prefix.length).replace(/\/$/, '');
entries.push({
name: relPath.split('/').pop() ?? relPath,
path: relPath,
size: 0,
modified: '',
isDirectory: true,
});
}
return entries;
}
async searchFiles(pattern: string): Promise<FileEntry[]> {
const all = await this.listFiles();
const regex = globToRegExp(pattern); // ReDoS-safe glob→regex (pattern is caller-controlled)
return all.filter(f => regex.test(f.path) || regex.test(f.name));
}
async moveFile(from: string, to: string): Promise<void> {
assertSafeRelativeKey(from);
assertSafeRelativeKey(to);
const client = await this.getClient();
const { CopyObjectCommand, DeleteObjectCommand } = await import('@aws-sdk/client-s3');
const fromKey = this.config.prefix + from;
const toKey = this.config.prefix + to;
await client.send(new CopyObjectCommand({
Bucket: this.config.bucket,
CopySource: `${this.config.bucket}/${fromKey}`,
Key: toKey,
}));
await client.send(new DeleteObjectCommand({
Bucket: this.config.bucket,
Key: fromKey,
}));
}
async getStorageInfo(): Promise<StorageInfo> {
const client = await this.getClient();
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
const response = await client.send(new ListObjectsV2Command({
Bucket: this.config.bucket,
Prefix: this.config.prefix,
}));
let usedBytes = 0;
let fileCount = 0;
for (const obj of response.Contents ?? []) {
usedBytes += obj.Size ?? 0;
fileCount++;
}
return { usedBytes, fileCount, storageType: 'virtual' };
}
}
// ── Factory ─────────────────────────────────────────────────────────
/**
* Create the appropriate FileStore for a workspace.
* If s3Config is provided, use S3FileStore (team/cloud deployment).
* If the workspace has a linked directory, use LinkedDirStore.
* Otherwise, use LocalFileStore (virtual managed storage).
*/
export function createFileStore(
dataDir: string,
workspaceId: string,
linkedDirectory?: string,
s3Config?: S3Config,
): FileStore {
if (s3Config) return new S3FileStore(s3Config);
if (linkedDirectory && fs.existsSync(linkedDirectory)) {
return new LinkedDirStore(linkedDirectory);
}
return new LocalFileStore(dataDir, workspaceId);
}

159
packages/core/src/index.ts Normal file
View File

@@ -0,0 +1,159 @@
// @waggle/core — Waggle-specific orchestration layer.
//
// As of CC Sesija B 2026-04-30 (PM Q3 Plan A ratification), the substrate
// (mind/ + harvest/ + logger + injection-scanner) lives in @waggle/hive-mind-core
// and is distributed as Apache 2.0 OSS via subtree-split. This barrel re-exports
// substrate symbols for backward compatibility — existing consumers (packages/agent,
// apps/web, packages/server, etc.) keep their `import { ... } from '@waggle/core'`
// imports unchanged.
//
// Waggle-specific (non-extracted) modules below the substrate re-exports stay
// in this package: config, multi-mind, workspace orchestration, vault, telemetry,
// install-audit, cron-store, skill-hashes, team-sync, file-store, file-indexer,
// memory-import, optimization-log, compliance/.
// ── Substrate re-exports from @waggle/hive-mind-core (Apache 2.0 OSS) ──
export {
// Logger + injection scanner
createCoreLogger, type CoreLogger,
scanForInjection, type ScanResult,
// mind/ — memory substrate
MindDB, EmbeddingDimMismatchError,
type EmbeddingFingerprint, type FingerprintCheck,
IdentityLayer, type Identity,
AwarenessLayer, type AwarenessItem, type AwarenessCategory,
FrameStore, stripHmPrefix, hashFrameContent,
RawArchive, hashRaw, readArchiveUids, withArchiveUid, type RawArchiveRow, type ArchiveInput,
MindErasure, type EraseResult,
SuppressionStore, type SuppressedSubject,
type MemoryFrame, type FrameType, type Importance, type FrameSource,
SessionStore, type Session,
HybridSearch, type SearchResult,
// D1 (oss-drift triage, 2026-06-11) — chunk-level retrieval (flag-gated, default OFF)
chunkRetrievalEnabled, rechunkAllFrames, type RechunkResult,
chunkText, type ChunkOptions, type FrameChunk,
KnowledgeGraph, type Entity, type Relation, type ValidationSchema,
SCHEMA_SQL, VEC_TABLE_SQL, CHUNKS_VEC_TABLE_SQL, SCHEMA_VERSION,
vecTableSqlForDim, chunksVecTableSqlForDim,
computeRelevance,
computeTemporalScore,
computePopularityScore,
computeContextualScore,
computeImportanceScore,
SCORING_PROFILES,
type ScoringProfile,
type ScoringWeights,
type Embedder,
createLiteLLMEmbedder, type LiteLLMEmbedderConfig,
createInProcessEmbedder, normalizeDimensions, type InProcessEmbedderConfig,
createOllamaEmbedder, type OllamaEmbedderConfig,
createApiEmbedder, type ApiEmbedderConfig,
createEmbeddingProvider, EmbeddingQuotaExceededError, getMinimumTierForProvider,
type EmbeddingProviderConfig, type EmbeddingProviderStatus, type EmbeddingProviderType,
type EmbeddingProviderInstance, type EmbeddingQuotaStatus,
normalizeEntityName, findDuplicates, isNoiseName, isLikelyAcronym,
Ontology, validateEntity, type EntitySchema, type ValidationResult,
ImprovementSignalStore,
type ImprovementSignal, type ActionableSignal, type SignalCategory, type ActionableThresholds,
ExecutionTraceStore, EXECUTION_TRACES_TABLE_SQL,
type ExecutionTrace, type ParsedExecutionTrace, type TraceOutcome,
type TracePayload, type TraceToolCall, type TraceReasoningStep,
type StartTraceInput, type FinalizeTraceInput, type TraceQueryFilter,
EvolutionRunStore, EVOLUTION_RUNS_TABLE_SQL,
type EvolutionRun, type EvolutionRunStatus, type EvolutionRunTarget,
type CreateEvolutionRunInput, type EvolutionRunFilter,
reconcileIndexes, reconcileFtsIndex, reconcileVecIndex, cleanOrphanVectors, cleanOrphanFts,
type ReconcileResult,
ConceptTracker, CONCEPT_MASTERY_TABLE_SQL,
type ConceptEntry, type ConceptUpdate,
// recall-context — temporal rendering helpers (W4.1 production port)
TEMPORAL_GUIDANCE, toDatePrefix, renderDatedSnippet, referenceDate, renderReferenceDateLine,
parseDateWindow, type DateWindow,
resolveRelativeDate, type ResolvedDate,
createInProcessReranker, type Reranker, type InProcessRerankerConfig,
// harvest/ — universal memory ingestion pipeline
HarvestSourceStore,
HarvestRunStore, type HarvestRun, type HarvestRunStatus,
ChatGPTAdapter,
ClaudeAdapter,
ClaudeCodeAdapter,
GeminiAdapter,
UniversalAdapter,
MarkdownAdapter,
PlaintextAdapter,
UrlAdapter,
PdfAdapter,
HarvestPipeline, type LLMCallFn, type PipelineOptions,
extractMemoryLanes, writeMemoryLaneFrames,
MIND_FACT_PREFIX, MIND_EVENT_PREFIX, MIND_PROFILE_PREFIX,
type MemoryLaneExtraction, type ExtractedEvent, type ExtractedFact, type ExtractedProfile,
type WriteLaneFramesResult,
// D2 — LLM-based KG entity extraction (oss-drift triage, 2026-06-11)
extractKgEntities, writeKgEntities, KG_ENTITY_TYPES,
type KgEntity, type KgEntityType, type KgEntityExtraction, type WriteKgEntitiesResult,
// W4.6 — raw-turn storage + RAWDETAIL recall lane
writeRawTurnFrames, rawTurnHeader, parseRawTurnHeader, rawTurnConvKey,
MIND_RAWTURN_PREFIX, MAX_TURNS_PER_ITEM, RAWDETAIL_KILL_SWITCH,
type WriteRawTurnsResult, type ParsedRawTurnHeader,
fetchRawDetailLane, rawTurnBody, RAW_DETAIL_K,
type RawTurnHit, type RawDetailLaneOptions,
dedup, harvestSetHash,
HARVEST_FRAME_CONTENT_CAP,
type ImportSourceType, type ImportItemType, type UniversalImportItem, type DistilledKnowledge,
type HarvestPipelineResult, type HarvestSource, type SourceAdapter, type FilesystemAdapter,
type ClassifiedItem, type ExtractedContent, type KnowledgeProvenance,
// Plan A AMENDMENT 2026-04-30 — multi-workspace orchestration
MultiMind, type MultiMindSearchResult, type MindSource, type SearchScope,
MultiMindCache, type MultiMindCacheConfig,
WorkspaceManager, type WorkspaceConfig, type CreateWorkspaceOptions,
// Supersession (P) + bridge (B) frame producer (mono-parity 2026-07-05)
detectSupersessionChains, detectEntityGroups, applyConsolidation,
collectObservations, getCurrentValues,
type ConsolidationLlm, type Observation, type SupersessionChain,
type EntityGroup, type ConsolidationResult, type CollectObservationsOptions,
} from '@waggle/hive-mind-core';
// ── Waggle-specific orchestration (stays in @waggle/core) ──
export { WaggleConfig, type ProviderEntry, type TeamServerConfig } from './config.js';
export { needsMigration, migrateToMultiMind } from './migration.js';
export { TeamSync, frameToEntity, entityToSyncedFrame, type TeamSyncConfig, type SyncedFrame } from './team-sync.js';
export {
InstallAuditStore, INSTALL_AUDIT_TABLE_SQL,
type InstallAuditEntry, type RecordAuditInput,
type AuditAction, type AuditRiskLevel, type AuditTrustSource,
type AuditApprovalClass, type AuditInitiator, type AuditCapabilityType,
} from './install-audit.js';
export {
CronStore, CRON_SCHEDULES_TABLE_SQL, CRON_RUN_LEASES_TABLE_SQL, VALID_JOB_TYPES, cronExprError,
type CronSchedule, type CreateScheduleInput, type CronJobType,
type CronExecutionRow, type CronRunLeaseRow,
type PendingActionRow, type PendingActionStatus, type SavePendingActionInput,
} from './cron-store.js';
export { VaultStore, type VaultEntry } from './vault.js';
export { TelemetryStore, TelemetryCollector, TELEMETRY_EVENTS, type TelemetryEvent, type TelemetrySummary } from './telemetry.js';
export {
SkillHashStore, computeSkillHash, SKILL_HASHES_TABLE_SQL,
type SkillHash,
} from './skill-hashes.js';
export { processImport, parseChatGPTExport, parseClaudeExport, extractKnowledge } from './memory-import.js';
export { createFileStore, LocalFileStore, LinkedDirStore, S3FileStore, isSensitiveFilePath, type FileStore, type FileEntry, type StorageInfo, type S3Config } from './file-store.js';
export { FileIndexer, MAX_CONTENT_BYTES, type FileIndexRow, type FileIndexResult } from './file-indexer.js';
export type { ImportSource, ImportResult, ExtractedKnowledge, ParsedConversation, ConversationMessage } from './memory-import.js';
export {
OptimizationLogStore, OPTIMIZATION_LOG_TABLE_SQL,
type OptimizationLogEntry, type CreateOptimizationLogInput,
} from './optimization-log.js';
// ── Compliance (AI Act) — stays in @waggle/core (NOT extracted per .github/sync.md) ──
export { InteractionStore } from './compliance/interaction-store.js';
export { ComplianceStatusChecker } from './compliance/status-checker.js';
export { ReportGenerator, type ReportGeneratorDeps } from './compliance/report-generator.js';
export { ComplianceTemplateStore, KVARK_TEMPLATE_NAME } from './compliance/template-store.js';
export { TEMPLATE_RISK_MAP } from './compliance/types.js';
export type {
AIActRiskLevel, HumanAction, AIInteraction, RecordInteractionInput,
ComplianceStatus, ArticleStatus, AuditReport, AuditReportRequest,
ModelInventoryEntry, OversightLogEntry, HarvestProvenanceEntry,
ComplianceTemplate, ComplianceTemplateSections,
CreateComplianceTemplateInput, UpdateComplianceTemplateInput,
} from './compliance/types.js';

View File

@@ -0,0 +1,221 @@
/**
* Install Audit Store — persistent audit trail for capability install events.
*
* Records every install-relevant action (proposed, approved, installed, rejected,
* failed) so there is a verifiable history of what was installed, when, why,
* and by whom.
*
* Follows the same pattern as ImprovementSignalStore — operates on the .mind DB.
*/
import type { MindDB } from '@waggle/hive-mind-core';
import {
sqlInList, RISK_LEVELS, APPROVAL_CLASSES, AUDIT_ACTIONS,
AUDIT_CAPABILITY_TYPES, AUDIT_INITIATORS, TRUST_SOURCES,
} from '@waggle/shared';
// ── Types ──────────────────────────────────────────────────────────────
// P7/D15 A2b: the audit vocabulary is now canonical in @waggle/shared. The
// Audit*-prefixed names are kept as aliases (re-exported) so every downstream
// `import { AuditAction, ... } from '@waggle/core'` keeps working unchanged.
// All six sets were already byte-identical to the shared ones (incl. P5/D4's
// 'uninstalled'), so this is a pure structural re-point — no value change.
export type {
AuditAction, AuditCapabilityType, AuditInitiator,
RiskLevel as AuditRiskLevel, ApprovalClass as AuditApprovalClass, TrustSource as AuditTrustSource,
} from '@waggle/shared';
import type {
AuditAction, AuditCapabilityType, AuditInitiator,
RiskLevel as AuditRiskLevel, ApprovalClass as AuditApprovalClass, TrustSource as AuditTrustSource,
} from '@waggle/shared';
export interface InstallAuditEntry {
id: number;
timestamp: string;
capability_name: string;
capability_type: AuditCapabilityType;
source: string;
version: string | null;
risk_level: AuditRiskLevel;
trust_source: AuditTrustSource;
approval_class: AuditApprovalClass;
action: AuditAction;
initiator: AuditInitiator;
detail: string;
}
export interface RecordAuditInput {
capabilityName: string;
capabilityType: AuditCapabilityType;
source: string;
version?: string | null;
riskLevel: AuditRiskLevel;
trustSource: AuditTrustSource;
approvalClass: AuditApprovalClass;
action: AuditAction;
initiator: AuditInitiator;
detail?: string;
}
// ── Table DDL ──────────────────────────────────────────────────────────
// P7/D15 A3: the CHECK lists are generated from the canonical @waggle/shared
// arrays via sqlInList, so the SQLite constraint and the TS union can no longer
// drift (divergence #14 — the old comment admitted "drift silently crashes
// record()"). The mirror DDL in hive-mind-core/src/mind/schema.ts stays a
// standalone literal (it's the OSS substrate, §7.5) but is locked to these same
// canonical lists by the parity test in install-audit-check-parity.test.ts.
// P7/D15 #15: trust_source now also carries a CHECK (was unconstrained at the DB
// while the TS type claimed a closed set). Every historical value came from the
// typed AuditTrustSource (the pre-security-gate 6-set ⊂ the current 7-set), so
// the rebuild migration's row copy can never violate it.
export const INSTALL_AUDIT_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS install_audit (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp TEXT NOT NULL DEFAULT (datetime('now')),
capability_name TEXT NOT NULL,
capability_type TEXT NOT NULL CHECK (capability_type IN (${sqlInList(AUDIT_CAPABILITY_TYPES)})),
source TEXT NOT NULL,
version TEXT,
risk_level TEXT NOT NULL CHECK (risk_level IN (${sqlInList(RISK_LEVELS)})),
trust_source TEXT NOT NULL CHECK (trust_source IN (${sqlInList(TRUST_SOURCES)})),
approval_class TEXT NOT NULL CHECK (approval_class IN (${sqlInList(APPROVAL_CLASSES)})),
action TEXT NOT NULL CHECK (action IN (${sqlInList(AUDIT_ACTIONS)})),
initiator TEXT NOT NULL CHECK (initiator IN (${sqlInList(AUDIT_INITIATORS)})),
detail TEXT NOT NULL DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_audit_capability ON install_audit (capability_name, action);
CREATE INDEX IF NOT EXISTS idx_audit_timestamp ON install_audit (timestamp DESC);
`;
// ── Store ──────────────────────────────────────────────────────────────
export class InstallAuditStore {
private db: MindDB;
constructor(db: MindDB) {
this.db = db;
this.ensureTable();
}
private ensureTable(): void {
const raw = this.db.getDatabase();
const existing = raw.prepare(
"SELECT sql FROM sqlite_master WHERE type='table' AND name='install_audit'",
).get() as { sql: string } | undefined;
if (!existing) {
raw.exec(INSTALL_AUDIT_TABLE_SQL);
return;
}
// P5/D4 migration: pre-'uninstalled' installs carry a narrower action CHECK
// baked into the table DDL. SQLite can't ALTER a CHECK, so rebuild the table
// when the stored DDL lacks the new value. Idempotent — a no-op once migrated.
// #15: also rebuild when the stored DDL has no trust_source CHECK (the column
// was previously unconstrained). "CHECK (trust_source IN" is a safe sentinel —
// it appears nowhere else in this DDL.
const needsActionWiden = !existing.sql.includes("'uninstalled'");
const needsTrustSourceCheck = !existing.sql.includes('CHECK (trust_source IN');
if (needsActionWiden || needsTrustSourceCheck) {
this.rebuildForWidenedActionCheck(raw);
}
}
/**
* Rebuild install_audit with the widened `action` CHECK, preserving all rows.
* Classic SQLite 12-step table redefinition, wrapped in a transaction so a
* crash mid-rebuild leaves the original table intact.
*/
private rebuildForWidenedActionCheck(raw: ReturnType<MindDB['getDatabase']>): void {
const migrate = raw.transaction(() => {
raw.exec('ALTER TABLE install_audit RENAME TO install_audit_legacy');
// SQLite carries indexes along with RENAME (still named idx_audit_*), so
// INSTALL_AUDIT_TABLE_SQL's CREATE INDEX IF NOT EXISTS would no-op and the
// DROP below would take the indexes with the legacy table. Drop them first
// (mirrors hive-mind-core db.ts) so they get recreated on the new table.
raw.exec('DROP INDEX IF EXISTS idx_audit_capability');
raw.exec('DROP INDEX IF EXISTS idx_audit_timestamp');
raw.exec(INSTALL_AUDIT_TABLE_SQL);
raw.exec(`
INSERT INTO install_audit (
id, timestamp, capability_name, capability_type, source, version,
risk_level, trust_source, approval_class, action, initiator, detail
)
SELECT id, timestamp, capability_name, capability_type, source, version,
risk_level, trust_source, approval_class, action, initiator, detail
FROM install_audit_legacy
`);
raw.exec('DROP TABLE install_audit_legacy');
});
migrate();
}
/** Record an install audit event. */
record(input: RecordAuditInput): InstallAuditEntry {
const raw = this.db.getDatabase();
raw.prepare(`
INSERT INTO install_audit (
capability_name, capability_type, source, version,
risk_level, trust_source, approval_class, action, initiator, detail
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
input.capabilityName,
input.capabilityType,
input.source,
input.version ?? null,
input.riskLevel,
input.trustSource,
input.approvalClass,
input.action,
input.initiator,
input.detail ?? '',
);
// Return the inserted row
return raw.prepare(
'SELECT * FROM install_audit ORDER BY id DESC LIMIT 1',
).get() as InstallAuditEntry;
}
/** Get audit history for a specific capability. */
getByCapability(name: string): InstallAuditEntry[] {
return this.db.getDatabase().prepare(
'SELECT * FROM install_audit WHERE capability_name = ? ORDER BY id DESC',
).all(name) as InstallAuditEntry[];
}
/** Get audit history filtered by action type. */
getByAction(action: AuditAction): InstallAuditEntry[] {
return this.db.getDatabase().prepare(
'SELECT * FROM install_audit WHERE action = ? ORDER BY id DESC',
).all(action) as InstallAuditEntry[];
}
/** Get recent audit entries (most recent first). */
getRecent(limit: number = 20): InstallAuditEntry[] {
return this.db.getDatabase().prepare(
'SELECT * FROM install_audit ORDER BY id DESC LIMIT ?',
).all(limit) as InstallAuditEntry[];
}
/** Get recent entries for one capability type (most recent first) — backs
* the shared Extend-layer audit read (GET /api/extend/audit?type=, C18). */
getRecentByType(type: AuditCapabilityType, limit: number = 20): InstallAuditEntry[] {
return this.db.getDatabase().prepare(
'SELECT * FROM install_audit WHERE capability_type = ? ORDER BY id DESC LIMIT ?',
).all(type, limit) as InstallAuditEntry[];
}
/** Get all entries (for testing). */
getAll(): InstallAuditEntry[] {
return this.db.getDatabase().prepare(
'SELECT * FROM install_audit ORDER BY id ASC',
).all() as InstallAuditEntry[];
}
/** Clear all entries (for testing). */
clear(): void {
this.db.getDatabase().prepare('DELETE FROM install_audit').run();
}
}

View File

@@ -0,0 +1,292 @@
/**
* Memory Import — parse ChatGPT and Claude exports, extract knowledge.
*
* Flow: raw JSON → parse conversations → extract knowledge items → ImportResult
*/
export type ImportSource = 'chatgpt' | 'claude';
export interface ConversationMessage {
role: 'user' | 'assistant';
text: string;
timestamp?: string;
}
export interface ParsedConversation {
title: string;
messages: ConversationMessage[];
createdAt?: string;
source: ImportSource;
}
export interface ExtractedKnowledge {
type: 'decision' | 'fact' | 'preference' | 'topic';
content: string;
source: ImportSource;
conversationTitle: string;
importance: 'important' | 'normal';
}
export interface ImportResult {
source: ImportSource;
conversationsFound: number;
conversationsParsed: number;
knowledgeExtracted: ExtractedKnowledge[];
errors: string[];
}
// ── Raw export shapes (untrusted external JSON; all fields optional) ──
/** A ChatGPT export node carries an optional message in its `mapping` entry. */
interface ChatGPTMessage {
author?: { role?: string };
content?: { parts?: unknown[] };
create_time?: number;
}
interface ChatGPTNode {
message?: ChatGPTMessage;
}
interface ChatGPTConversation {
title?: string;
mapping?: Record<string, ChatGPTNode>;
create_time?: number;
}
interface ClaudeMessage {
sender?: string;
role?: string;
text?: string;
content?: string;
created_at?: string;
timestamp?: string;
}
interface ClaudeConversation {
name?: string;
title?: string;
chat_messages?: ClaudeMessage[];
messages?: ClaudeMessage[];
created_at?: string;
create_time?: string;
}
/** Pull the `conversations` array out of an export that may be the array itself. */
function extractConversations(json: unknown): unknown[] {
if (Array.isArray(json)) return json;
if (json && typeof json === 'object' && 'conversations' in json) {
const c = (json as { conversations?: unknown }).conversations;
if (Array.isArray(c)) return c;
}
return [];
}
// ── Parsers ───────────────────────────────────────
export function parseChatGPTExport(json: unknown): ParsedConversation[] {
const conversations = extractConversations(json);
return conversations.map((raw) => {
const conv = raw as ChatGPTConversation;
const title = conv.title || 'Untitled';
const messages: ConversationMessage[] = [];
// ChatGPT uses a mapping object with node IDs
if (conv.mapping && typeof conv.mapping === 'object') {
const nodes = Object.values(conv.mapping);
// Sort by create_time for chronological order
const sorted = nodes
.filter((n) => (n?.message?.content?.parts?.length ?? 0) > 0)
.sort((a, b) => (a.message?.create_time ?? 0) - (b.message?.create_time ?? 0));
for (const node of sorted) {
const msg = node.message;
if (!msg || !msg.author?.role) continue;
const role = msg.author.role === 'user' ? 'user' : 'assistant';
if (msg.author.role === 'system') continue; // Skip system messages
const text = (msg.content?.parts ?? [])
.filter((p): p is string => typeof p === 'string')
.join('\n')
.trim();
if (!text) continue;
messages.push({
role,
text,
timestamp: msg.create_time ? new Date(msg.create_time * 1000).toISOString() : undefined,
});
}
}
return {
title,
messages,
createdAt: conv.create_time ? new Date(conv.create_time * 1000).toISOString() : undefined,
source: 'chatgpt' as ImportSource,
};
}).filter(c => c.messages.length > 0);
}
export function parseClaudeExport(json: unknown): ParsedConversation[] {
const conversations = extractConversations(json);
return conversations.map((raw) => {
const conv = raw as ClaudeConversation;
const title = conv.name || conv.title || 'Untitled';
const messages: ConversationMessage[] = [];
const chatMessages = conv.chat_messages ?? conv.messages ?? [];
for (const msg of chatMessages) {
const role = (msg.sender === 'human' || msg.role === 'user') ? 'user' : 'assistant';
const text = (msg.text ?? msg.content ?? '').trim();
if (!text) continue;
messages.push({
role,
text,
timestamp: msg.created_at ?? msg.timestamp,
});
}
return {
title,
messages,
createdAt: conv.created_at ?? conv.create_time,
source: 'claude' as ImportSource,
};
}).filter(c => c.messages.length > 0);
}
// ── Knowledge Extraction ──────────────────────────
const DECISION_PATTERNS = [
/\b(?:decided|chose|going with|we'll use|confirmed|agreed|settled on|picked|selected)\b/i,
/\b(?:decision|choice):\s/i,
/\blet's go with\b/i,
];
const PREFERENCE_PATTERNS = [
/\b(?:I prefer|I like|I always|I never|I don't like|don't use|use .+ instead)\b/i,
/\b(?:my preference|I'd rather|please always|please never|from now on)\b/i,
/\b(?:my style is|I want you to|format it as)\b/i,
];
const FACT_PATTERNS = [
/\b(?:my name is|I work at|I'm a|I am a|my company|my team|my role|our product|our stack)\b/i,
/\b(?:we use|our tech|we're building|the project is|the codebase)\b/i,
];
export function extractKnowledge(conversations: ParsedConversation[]): ExtractedKnowledge[] {
const knowledge: ExtractedKnowledge[] = [];
const seen = new Set<string>(); // Deduplicate
for (const conv of conversations) {
// Extract conversation topic
if (conv.title && conv.title !== 'Untitled' && conv.title.length > 5) {
const topicKey = `topic:${conv.title.toLowerCase().trim()}`;
if (!seen.has(topicKey)) {
seen.add(topicKey);
knowledge.push({
type: 'topic',
content: `Conversation topic: ${conv.title}`,
source: conv.source,
conversationTitle: conv.title,
importance: 'normal',
});
}
}
// Scan user messages for decisions, preferences, facts
for (const msg of conv.messages) {
if (msg.role !== 'user') continue;
if (msg.text.length < 10 || msg.text.length > 500) continue; // Skip too short/long
const sentences = msg.text.split(/[.!?\n]+/).map(s => s.trim()).filter(s => s.length > 10);
for (const sentence of sentences) {
const contentKey = sentence.toLowerCase().slice(0, 80);
if (seen.has(contentKey)) continue;
// Check for decisions
if (DECISION_PATTERNS.some(p => p.test(sentence))) {
seen.add(contentKey);
knowledge.push({
type: 'decision',
content: `Decision: ${sentence}`,
source: conv.source,
conversationTitle: conv.title,
importance: 'important',
});
continue;
}
// Check for preferences
if (PREFERENCE_PATTERNS.some(p => p.test(sentence))) {
seen.add(contentKey);
knowledge.push({
type: 'preference',
content: `Preference: ${sentence}`,
source: conv.source,
conversationTitle: conv.title,
importance: 'important',
});
continue;
}
// Check for facts
if (FACT_PATTERNS.some(p => p.test(sentence))) {
seen.add(contentKey);
knowledge.push({
type: 'fact',
content: `${sentence}`,
source: conv.source,
conversationTitle: conv.title,
importance: 'normal',
});
continue;
}
}
}
}
// Cap at 100 items to avoid polluting memory
return knowledge.slice(0, 100);
}
// ── Main Import Function ──────────────────────────
export function processImport(jsonData: unknown, source: ImportSource): ImportResult {
const errors: string[] = [];
let conversations: ParsedConversation[];
try {
conversations = source === 'chatgpt'
? parseChatGPTExport(jsonData)
: parseClaudeExport(jsonData);
} catch (err: unknown) {
const message = err instanceof Error ? err.message : String(err);
return {
source,
conversationsFound: 0,
conversationsParsed: 0,
knowledgeExtracted: [],
errors: [`Parse error: ${message}`],
};
}
if (conversations.length === 0) {
errors.push('No conversations found in export');
}
const knowledge = extractKnowledge(conversations);
return {
source,
conversationsFound: conversations.length,
conversationsParsed: conversations.filter(c => c.messages.length > 0).length,
knowledgeExtracted: knowledge,
errors,
};
}

View File

@@ -0,0 +1,45 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
/**
* Check if a migration from default.mind to personal.mind is needed.
* Returns true if default.mind exists AND personal.mind does NOT exist.
*/
export function needsMigration(waggleDir: string): boolean {
const defaultMind = path.join(waggleDir, 'default.mind');
const personalMind = path.join(waggleDir, 'personal.mind');
return fs.existsSync(defaultMind) && !fs.existsSync(personalMind);
}
/**
* Migrate from the old single-mind layout (default.mind) to the
* M4 multi-mind layout (personal.mind + workspaces/).
*
* Steps when migration is needed:
* 1. Copy default.mind -> personal.mind
* 2. Rename default.mind -> default.mind.bak
* 3. Create workspaces/ directory
*
* This is a pure file operation -- no data transformation is performed.
*/
export function migrateToMultiMind(waggleDir: string): { migrated: boolean; message: string } {
if (!needsMigration(waggleDir)) {
return { migrated: false, message: 'No migration needed' };
}
const defaultMind = path.join(waggleDir, 'default.mind');
const personalMind = path.join(waggleDir, 'personal.mind');
const backupMind = path.join(waggleDir, 'default.mind.bak');
const workspacesDir = path.join(waggleDir, 'workspaces');
// 1. Copy default.mind -> personal.mind
fs.copyFileSync(defaultMind, personalMind);
// 2. Rename default.mind -> default.mind.bak
fs.renameSync(defaultMind, backupMind);
// 3. Create workspaces/ directory (idempotent)
fs.mkdirSync(workspacesDir, { recursive: true });
return { migrated: true, message: 'Migrated default.mind to personal.mind' };
}

View File

@@ -0,0 +1,154 @@
/**
* Optimization Log Store — SQLite persistence for GEPA prompt optimization data.
*
* Captures agent interaction metadata (system prompts, tools used, turn counts,
* correction signals) for background analysis by the PromptOptimizer.
*
* Follows the same lazy-table pattern as CronStore and InstallAuditStore —
* operates on the .mind DB with auto-table creation.
*/
import type { MindDB } from '@waggle/hive-mind-core';
// ── Types ──────────────────────────────────────────────────────────────
export interface OptimizationLogEntry {
id: number;
session_id: string;
workspace_id: string;
system_prompt: string;
tools_used: string; // JSON array
turn_count: number;
was_correction: number; // SQLite integer boolean
input_tokens: number;
output_tokens: number;
timestamp: string;
}
export interface CreateOptimizationLogInput {
sessionId: string;
workspaceId: string;
systemPrompt: string;
toolsUsed: string[];
turnCount: number;
wasCorrection: boolean;
inputTokens?: number;
outputTokens?: number;
}
// ── Table DDL ──────────────────────────────────────────────────────────
export const OPTIMIZATION_LOG_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS optimization_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
session_id TEXT NOT NULL,
workspace_id TEXT NOT NULL,
system_prompt TEXT NOT NULL,
tools_used TEXT NOT NULL DEFAULT '[]',
turn_count INTEGER NOT NULL DEFAULT 0,
was_correction INTEGER NOT NULL DEFAULT 0,
input_tokens INTEGER NOT NULL DEFAULT 0,
output_tokens INTEGER NOT NULL DEFAULT 0,
timestamp TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_optlog_workspace ON optimization_log (workspace_id, timestamp DESC);
CREATE INDEX IF NOT EXISTS idx_optlog_timestamp ON optimization_log (timestamp DESC);
`;
// ── Store ──────────────────────────────────────────────────────────────
export class OptimizationLogStore {
private db: MindDB;
constructor(db: MindDB) {
this.db = db;
this.ensureTable();
}
private ensureTable(): void {
const raw = this.db.getDatabase();
const exists = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='optimization_log'",
).get();
if (!exists) {
raw.exec(OPTIMIZATION_LOG_TABLE_SQL);
}
}
/** Insert a new optimization log entry capturing an agent interaction. */
insert(input: CreateOptimizationLogInput): OptimizationLogEntry {
const raw = this.db.getDatabase();
const result = raw.prepare(`
INSERT INTO optimization_log (session_id, workspace_id, system_prompt, tools_used, turn_count, was_correction, input_tokens, output_tokens)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`).run(
input.sessionId,
input.workspaceId,
input.systemPrompt,
JSON.stringify(input.toolsUsed),
input.turnCount,
input.wasCorrection ? 1 : 0,
input.inputTokens ?? 0,
input.outputTokens ?? 0,
);
return raw.prepare(
'SELECT * FROM optimization_log WHERE id = ?',
).get(result.lastInsertRowid) as OptimizationLogEntry;
}
/** Get recent optimization log entries, ordered by most recent first. */
getRecent(limit: number = 50): OptimizationLogEntry[] {
return this.db.getDatabase().prepare(
'SELECT * FROM optimization_log ORDER BY timestamp DESC, id DESC LIMIT ?',
).all(limit) as OptimizationLogEntry[];
}
/** Get log entries for a specific workspace. */
getByWorkspace(workspaceId: string, limit: number = 50): OptimizationLogEntry[] {
return this.db.getDatabase().prepare(
'SELECT * FROM optimization_log WHERE workspace_id = ? ORDER BY timestamp DESC, id DESC LIMIT ?',
).all(workspaceId, limit) as OptimizationLogEntry[];
}
/** Get aggregate stats: total entries, correction rate, avg turn count, total tokens. */
getStats(): { total: number; correctionRate: number; avgTurnCount: number; totalInputTokens: number; totalOutputTokens: number } {
const raw = this.db.getDatabase();
const row = raw.prepare(`
SELECT
COUNT(*) as total,
COALESCE(AVG(was_correction * 1.0), 0) as correction_rate,
COALESCE(AVG(turn_count), 0) as avg_turn_count,
COALESCE(SUM(input_tokens), 0) as total_input_tokens,
COALESCE(SUM(output_tokens), 0) as total_output_tokens
FROM optimization_log
`).get() as {
total: number;
correction_rate: number;
avg_turn_count: number;
total_input_tokens: number;
total_output_tokens: number;
};
return {
total: row.total,
correctionRate: row.correction_rate,
avgTurnCount: row.avg_turn_count,
totalInputTokens: row.total_input_tokens,
totalOutputTokens: row.total_output_tokens,
};
}
/** Delete entries older than the given number of days. */
pruneOlderThan(days: number): number {
const result = this.db.getDatabase().prepare(
`DELETE FROM optimization_log WHERE timestamp < datetime('now', '-' || ? || ' days')`,
).run(days);
return result.changes;
}
/** Clear all entries (for testing). */
clear(): void {
this.db.getDatabase().prepare('DELETE FROM optimization_log').run();
}
}

View File

@@ -0,0 +1,122 @@
/**
* Skill Hash Store — SHA-256 content hashing for skill update detection.
*
* Stores hashes of skill file content in the .mind DB. On server startup,
* compares current file hashes to stored hashes to detect changes.
* Follows the same ensureTable pattern as CronStore / InstallAuditStore.
*/
import * as crypto from 'node:crypto';
import type { MindDB } from '@waggle/hive-mind-core';
// ── Types ──────────────────────────────────────────────────────────────
export interface SkillHash {
name: string;
hash: string;
verified_at: string;
}
// ── Table DDL ──────────────────────────────────────────────────────────
export const SKILL_HASHES_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS skill_hashes (
name TEXT PRIMARY KEY,
hash TEXT NOT NULL,
verified_at TEXT NOT NULL DEFAULT (datetime('now'))
);
`;
// ── Helpers ────────────────────────────────────────────────────────────
/** Compute SHA-256 hex digest of skill content. */
export function computeSkillHash(content: string): string {
return crypto.createHash('sha256').update(content).digest('hex');
}
// ── Store ──────────────────────────────────────────────────────────────
export class SkillHashStore {
private db: MindDB;
constructor(db: MindDB) {
this.db = db;
this.ensureTable();
}
private ensureTable(): void {
const raw = this.db.getDatabase();
const exists = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='skill_hashes'",
).get();
if (!exists) {
raw.exec(SKILL_HASHES_TABLE_SQL);
}
}
/** Store or update hash for a skill. */
setHash(name: string, hash: string): void {
this.db.getDatabase().prepare(`
INSERT INTO skill_hashes (name, hash) VALUES (?, ?)
ON CONFLICT (name) DO UPDATE SET hash = ?, verified_at = datetime('now')
`).run(name, hash, hash);
}
/** Get stored hash for a skill. */
getHash(name: string): SkillHash | undefined {
return this.db.getDatabase().prepare(
'SELECT * FROM skill_hashes WHERE name = ?',
).get(name) as SkillHash | undefined;
}
/** Remove hash (when skill is deleted). */
removeHash(name: string): void {
this.db.getDatabase().prepare('DELETE FROM skill_hashes WHERE name = ?').run(name);
}
/**
* Check all skills against stored hashes.
* Returns lists of changed, added, and removed skill names.
*/
checkAll(currentSkills: Array<{ name: string; content: string }>): {
changed: string[];
added: string[];
removed: string[];
} {
const storedHashes = this.db.getDatabase().prepare(
'SELECT name, hash FROM skill_hashes',
).all() as SkillHash[];
const storedMap = new Map(storedHashes.map(h => [h.name, h.hash]));
const currentNames = new Set(currentSkills.map(s => s.name));
const changed: string[] = [];
const added: string[] = [];
for (const skill of currentSkills) {
const currentHash = computeSkillHash(skill.content);
const storedHash = storedMap.get(skill.name);
if (!storedHash) {
added.push(skill.name);
} else if (storedHash !== currentHash) {
changed.push(skill.name);
}
}
const removed = storedHashes
.filter(h => !currentNames.has(h.name))
.map(h => h.name);
return { changed, added, removed };
}
/** Mark a skill as verified (update hash to current content). */
verify(name: string, content: string): void {
this.setHash(name, computeSkillHash(content));
}
/** Clear all hashes (for testing). */
clear(): void {
this.db.getDatabase().prepare('DELETE FROM skill_hashes').run();
}
}

View File

@@ -0,0 +1,186 @@
/**
* TeamSync — Syncs memory frames between local .mind and team server.
*
* Uses the team server's entities API (entityType='memory_frame') for storage.
* Push-on-write: new frames are pushed to team server after local write.
* Pull-on-activate: frames are pulled from team server when workspace is activated.
* Attribution: each frame carries the author's userId and displayName.
*
* Sync protocol:
* - Frames are identified by a composite key: gopId + t (group-of-pictures ID + temporal index)
* - Pull uses ?type=memory_frame&since=<timestamp> to get only new frames
* - Push sends each frame as a teamEntity with entityType='memory_frame'
* - Last-write-wins at frame level (frames are append-mostly, conflicts are rare)
*/
import type { MemoryFrame, FrameType, Importance } from '@waggle/hive-mind-core';
import { createCoreLogger } from '@waggle/hive-mind-core';
const log = createCoreLogger('team-sync');
export interface TeamSyncConfig {
teamServerUrl: string;
teamSlug: string;
authToken: string;
userId: string;
displayName: string;
}
export interface SyncedFrame {
/** Server-side entity ID (UUID). */
remoteId: string;
gopId: string;
t: number;
frameType: FrameType;
content: string;
importance: Importance;
authorId: string;
authorName: string;
createdAt: string;
}
/**
* Convert a local MemoryFrame to the team server entity format.
*/
export function frameToEntity(frame: MemoryFrame, authorId: string, authorName: string) {
return {
entityType: 'memory_frame',
name: frame.gop_id,
properties: {
frameType: frame.frame_type,
t: frame.t,
baseFrameId: frame.base_frame_id,
content: frame.content,
importance: frame.importance,
authorId,
authorName,
localId: frame.id,
},
};
}
/**
* Convert a team server entity back to a SyncedFrame.
*/
export function entityToSyncedFrame(entity: {
id: string;
name: string;
properties: Record<string, unknown>;
createdAt: string;
}): SyncedFrame {
const props = entity.properties;
return {
remoteId: entity.id,
gopId: entity.name,
t: (props.t as number) ?? 0,
frameType: (props.frameType as FrameType) ?? 'I',
content: (props.content as string) ?? '',
importance: (props.importance as Importance) ?? 'normal',
authorId: (props.authorId as string) ?? '',
authorName: (props.authorName as string) ?? '',
createdAt: entity.createdAt,
};
}
/**
* TeamSync handles push/pull of memory frames to/from the team server.
* This is a stateless client — call pushFrame() after writing locally,
* call pullFrames() on workspace activation.
*/
export class TeamSync {
private config: TeamSyncConfig;
private lastSyncTimestamp: string | null = null;
constructor(config: TeamSyncConfig) {
this.config = config;
}
/**
* Push a single frame to the team server.
* Call this after writing a frame to the local workspace .mind.
*/
async pushFrame(frame: MemoryFrame): Promise<{ remoteId: string } | null> {
const entity = frameToEntity(frame, this.config.userId, this.config.displayName);
try {
const response = await fetch(
`${this.config.teamServerUrl}/api/teams/${this.config.teamSlug}/entities`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.config.authToken}`,
},
body: JSON.stringify(entity),
}
);
if (!response.ok) {
log.error(`Push failed: ${response.status} ${response.statusText}`);
return null;
}
const created = await response.json() as { id: string };
return { remoteId: created.id };
} catch (err) {
log.error('Push error:', err);
return null;
}
}
/**
* Pull all memory frames from the team server.
* Returns frames sorted by creation time (newest first).
* Optionally pass `since` to get only frames created after that timestamp.
*/
async pullFrames(since?: string): Promise<SyncedFrame[]> {
try {
const url = `${this.config.teamServerUrl}/api/teams/${this.config.teamSlug}/entities?type=memory_frame`;
// Note: `since` filtering would require server-side support. For now, pull all and filter client-side.
const response = await fetch(url, {
headers: {
'Authorization': `Bearer ${this.config.authToken}`,
},
});
if (!response.ok) {
log.error(`Pull failed: ${response.status} ${response.statusText}`);
return [];
}
const entities = await response.json() as Array<{
id: string;
name: string;
properties: Record<string, unknown>;
createdAt: string;
}>;
const frames = entities.map(entityToSyncedFrame);
// Client-side filter by timestamp if requested
if (since) {
return frames.filter(f => f.createdAt > since);
}
this.lastSyncTimestamp = new Date().toISOString();
return frames;
} catch (err) {
log.error('Pull error:', err);
return [];
}
}
/**
* Get the last sync timestamp (for incremental sync).
*/
getLastSyncTimestamp(): string | null {
return this.lastSyncTimestamp;
}
/**
* Set the last sync timestamp (e.g. loaded from persistent storage).
*/
setLastSyncTimestamp(ts: string): void {
this.lastSyncTimestamp = ts;
}
}

View File

@@ -0,0 +1,324 @@
/**
* TelemetryStore — local, privacy-first event tracking.
*
* All data stays in ~/.waggle/telemetry.db (SQLite).
* Default: OFF. User opts in via Settings toggle.
* No cloud reporting in M2 — data is queryable via local API only.
*
* NEVER tracked: message content, memory content, file paths,
* API keys, personal info, IP addresses, device IDs.
*/
import Database from 'better-sqlite3';
import type { Database as DatabaseType } from 'better-sqlite3';
import path from 'node:path';
import fs from 'node:fs';
/* ── Schema ── */
const TELEMETRY_SCHEMA = `
CREATE TABLE IF NOT EXISTS telemetry_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
event TEXT NOT NULL,
properties TEXT NOT NULL DEFAULT '{}',
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_telemetry_event ON telemetry_events (event, created_at);
CREATE INDEX IF NOT EXISTS idx_telemetry_date ON telemetry_events (created_at);
`;
/* ── Event constants ── */
export const TELEMETRY_EVENTS = {
// Onboarding funnel
ONBOARDING_START: 'onboarding_start',
ONBOARDING_STEP: 'onboarding_step',
ONBOARDING_COMPLETE: 'onboarding_complete',
ONBOARDING_SKIP: 'onboarding_skip',
// Session engagement
SESSION_START: 'session_start',
SESSION_END: 'session_end',
// Infrastructure
EMBEDDING_PROVIDER: 'embedding_provider',
LLM_PROVIDER: 'llm_provider',
// Feature usage
TEMPLATE_SELECTED: 'template_selected',
WORKSPACE_CREATED: 'workspace_created',
FIRST_AGENT_RESPONSE: 'first_agent_response',
SLASH_COMMAND_USED: 'slash_command_used',
// App lifecycle
APP_START: 'app_start',
APP_ERROR: 'app_error',
} as const;
/* ── Types ── */
export interface TelemetryEvent {
id: number;
event: string;
properties: Record<string, unknown>;
created_at: string;
}
export interface TelemetrySummary {
enabled: boolean;
totalEvents: number;
firstEvent: string | null;
lastEvent: string | null;
onboardingCompleted: boolean;
totalSessions: number;
embeddingProvider: string | null;
templatesUsed: string[];
eventBreakdown: Record<string, number>;
}
/* ── Store ── */
export class TelemetryStore {
private db: DatabaseType;
private enabled: boolean;
constructor(dataDir: string, enabled = false) {
this.enabled = enabled;
const dbPath = path.join(dataDir, 'telemetry.db');
this.db = new Database(dbPath);
this.db.pragma('journal_mode = WAL');
this.db.exec(TELEMETRY_SCHEMA);
}
/** Track an event. No-op if telemetry is disabled. */
track(event: string, properties?: Record<string, unknown>): void {
if (!this.enabled) return;
this.db.prepare(
'INSERT INTO telemetry_events (event, properties) VALUES (?, ?)'
).run(event, JSON.stringify(properties ?? {}));
}
/** Enable/disable telemetry at runtime. */
setEnabled(enabled: boolean): void {
this.enabled = enabled;
}
/** Whether telemetry is currently enabled. */
isEnabled(): boolean {
return this.enabled;
}
/** Get aggregated summary. */
getSummary(): TelemetrySummary {
const total = (this.db.prepare(
'SELECT COUNT(*) as cnt FROM telemetry_events'
).get() as { cnt: number }).cnt;
const first = this.db.prepare(
'SELECT MIN(created_at) as d FROM telemetry_events'
).get() as { d: string | null };
const last = this.db.prepare(
'SELECT MAX(created_at) as d FROM telemetry_events'
).get() as { d: string | null };
const onboarded = (this.db.prepare(
"SELECT COUNT(*) as cnt FROM telemetry_events WHERE event = 'onboarding_complete'"
).get() as { cnt: number }).cnt > 0;
const sessions = (this.db.prepare(
"SELECT COUNT(*) as cnt FROM telemetry_events WHERE event = 'session_start'"
).get() as { cnt: number }).cnt;
const embRow = this.db.prepare(
"SELECT properties FROM telemetry_events WHERE event = 'embedding_provider' ORDER BY created_at DESC LIMIT 1"
).get() as { properties: string } | undefined;
const embProvider = embRow
? (JSON.parse(embRow.properties) as Record<string, unknown>).provider as string ?? null
: null;
const tplRows = this.db.prepare(
"SELECT DISTINCT json_extract(properties, '$.templateId') as tid FROM telemetry_events WHERE event IN ('template_selected', 'workspace_created') AND json_extract(properties, '$.templateId') IS NOT NULL"
).all() as Array<{ tid: string }>;
const breakdownRows = this.db.prepare(
'SELECT event, COUNT(*) as cnt FROM telemetry_events GROUP BY event ORDER BY cnt DESC'
).all() as Array<{ event: string; cnt: number }>;
const breakdown: Record<string, number> = {};
for (const row of breakdownRows) {
breakdown[row.event] = row.cnt;
}
return {
enabled: this.enabled,
totalEvents: total,
firstEvent: first.d,
lastEvent: last.d,
onboardingCompleted: onboarded,
totalSessions: sessions,
embeddingProvider: embProvider,
templatesUsed: tplRows.map(r => r.tid),
eventBreakdown: breakdown,
};
}
/** Get raw events with optional filters. */
getEvents(options?: { event?: string; since?: string; until?: string; limit?: number }): TelemetryEvent[] {
const conditions: string[] = [];
const params: unknown[] = [];
if (options?.event) {
conditions.push('event = ?');
params.push(options.event);
}
if (options?.since) {
conditions.push('created_at >= ?');
params.push(options.since);
}
if (options?.until) {
conditions.push('created_at <= ?');
params.push(options.until);
}
const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
const limit = options?.limit ?? 100;
const rows = this.db.prepare(
`SELECT id, event, properties, created_at FROM telemetry_events ${where} ORDER BY created_at DESC LIMIT ?`
).all(...params, limit) as Array<{ id: number; event: string; properties: string; created_at: string }>;
return rows.map(r => ({
id: r.id,
event: r.event,
properties: JSON.parse(r.properties) as Record<string, unknown>,
created_at: r.created_at,
}));
}
/** Delete all telemetry data (user right-to-delete). */
clear(): { deleted: number } {
const result = this.db.prepare('DELETE FROM telemetry_events').run();
return { deleted: result.changes };
}
/** Close the database connection. */
close(): void {
this.db.close();
}
}
/* ── Collector ── */
const COLLECTOR_SCHEMA = `
CREATE TABLE IF NOT EXISTS collector_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
category TEXT NOT NULL,
name TEXT NOT NULL,
count INTEGER NOT NULL DEFAULT 1,
date TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_collector_date ON collector_events (date);
CREATE UNIQUE INDEX IF NOT EXISTS idx_collector_name_date ON collector_events (name, date);
`;
/**
* TelemetryCollector — higher-level daily-aggregated event tracker.
*
* Records tool usage, commands, errors, and capability gaps by day.
* Privacy-safe: only category + name + count, no content or PII.
*/
export class TelemetryCollector {
private db: DatabaseType;
private enabled: boolean;
private dataDir: string;
constructor(dataDir: string, enabled = false) {
this.dataDir = dataDir;
this.enabled = enabled;
const dbPath = path.join(dataDir, 'telemetry.db');
this.db = new Database(dbPath);
this.db.pragma('journal_mode = WAL');
this.db.exec(COLLECTOR_SCHEMA);
}
recordToolUse(toolName: string): void {
if (!this.enabled) return;
this.upsertEvent('tool', toolName);
}
recordCommand(command: string): void {
if (!this.enabled) return;
this.upsertEvent('command', command);
}
recordError(errorName: string): void {
if (!this.enabled) return;
this.upsertEvent('error', errorName);
}
recordCapabilityGap(capability: string): void {
if (!this.enabled) return;
this.upsertEvent('capability_gap', capability);
}
recordSession(durationMs: number, interactionCount: number): void {
if (!this.enabled) return;
const today = new Date().toISOString().split('T')[0];
this.db.prepare(
`INSERT INTO collector_events (category, name, count, date) VALUES ('session', 'duration_total_ms', ?, ?)
ON CONFLICT(name, date) DO UPDATE SET count = count + excluded.count`
).run(durationMs, today);
this.db.prepare(
`INSERT INTO collector_events (category, name, count, date) VALUES ('session', 'interaction_count', ?, ?)
ON CONFLICT(name, date) DO UPDATE SET count = count + excluded.count`
).run(interactionCount, today);
this.db.prepare(
`INSERT INTO collector_events (category, name, count, date) VALUES ('session', 'session_count', 1, ?)
ON CONFLICT(name, date) DO UPDATE SET count = count + 1`
).run(today);
}
private upsertEvent(category: string, name: string): void {
const today = new Date().toISOString().split('T')[0];
this.db.prepare(
`INSERT INTO collector_events (category, name, count, date) VALUES (?, ?, 1, ?)
ON CONFLICT(name, date) DO UPDATE SET count = count + 1`
).run(category, name, today);
}
getReport(days = 7): {
totalEvents: number;
events: Array<{ name: string; count: number; category: string; date: string }>;
dateRange: { from: string | null };
} {
const since = new Date();
since.setDate(since.getDate() - days);
const sinceStr = since.toISOString().split('T')[0];
const rows = this.db.prepare(
`SELECT name, SUM(count) as total_count, category, date
FROM collector_events WHERE date >= ?
GROUP BY name, category, date ORDER BY date DESC, name`
).all(sinceStr) as Array<{ name: string; total_count: number; category: string; date: string }>;
const totalEvents = rows.reduce((sum, r) => sum + r.total_count, 0);
const events = rows.map(r => ({ name: r.name, count: r.total_count, category: r.category, date: r.date }));
const firstRow = this.db.prepare(
'SELECT MIN(date) as first_date FROM collector_events'
).get() as { first_date: string | null };
return { totalEvents, events, dateRange: { from: firstRow.first_date } };
}
/** Write events to telemetry.json for external consumption. */
flush(): void {
const rows = this.db.prepare(
'SELECT name, count, category, date FROM collector_events ORDER BY date DESC, name'
).all() as Array<{ name: string; count: number; category: string; date: string }>;
fs.writeFileSync(
path.join(this.dataDir, 'telemetry.json'),
JSON.stringify(rows, null, 2),
);
}
setEnabled(enabled: boolean): void { this.enabled = enabled; }
isEnabled(): boolean { return this.enabled; }
close(): void { this.db.close(); }
}

299
packages/core/src/vault.ts Normal file
View File

@@ -0,0 +1,299 @@
/**
* VaultStore — encrypted local secrets store.
*
* Encrypts secrets using AES-256-GCM (Node.js built-in crypto).
* Stores encrypted data in vault.json with a machine-local key file.
* Each entry is independently encrypted so individual secrets can be
* updated without re-encrypting everything.
*/
import * as crypto from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { execFileSync } from 'node:child_process';
import { createCoreLogger } from '@waggle/hive-mind-core';
const log = createCoreLogger('vault');
const ALGORITHM = 'aes-256-gcm';
const KEY_LENGTH = 32;
const IV_LENGTH = 16;
export interface VaultEntry {
name: string;
value: string;
metadata?: Record<string, unknown>;
updatedAt: string;
}
interface VaultRecord {
encrypted: string;
metadata?: Record<string, unknown>;
updatedAt: string;
}
export class VaultStore {
private dataDir: string;
private vaultPath: string;
private keyPath: string;
private encryptionKey: Buffer;
constructor(dataDir: string) {
this.dataDir = dataDir;
this.vaultPath = path.join(dataDir, 'vault.json');
this.keyPath = path.join(dataDir, '.vault-key');
// Ensure data directory exists
if (!fs.existsSync(this.dataDir)) {
fs.mkdirSync(this.dataDir, { recursive: true });
}
this.encryptionKey = this.ensureKey();
}
/** Ensure the encryption key exists. Generate if missing. */
private ensureKey(): Buffer {
if (fs.existsSync(this.keyPath)) {
const key = Buffer.from(fs.readFileSync(this.keyPath, 'utf-8').trim(), 'hex');
if (key.length !== KEY_LENGTH) {
throw new Error(
`Vault key file is corrupted — expected ${KEY_LENGTH} bytes, got ${key.length}. Delete ${this.keyPath} to regenerate.`
);
}
return key;
}
const key = crypto.randomBytes(KEY_LENGTH);
fs.writeFileSync(this.keyPath, key.toString('hex'), { mode: 0o600 });
// Review Critical #1: On Windows, restrict key file access to current user only.
// Previously used `require('node:child_process')` inline which fails silently under
// ESM (`type: module` in the sidecar) — the try/catch swallowed the ReferenceError
// and every Windows install left the vault key with no ACL restriction.
// Now imported statically at the top of the file; the try/catch only covers actual
// icacls failures (e.g. icacls.exe not on PATH in a minimal Windows image).
if (process.platform === 'win32') {
try {
// Resolve the current user via whoami (safer than process.env.USERNAME
// which can be absent or spoofed in containerized/scripted setups)
const currentUser = execFileSync('whoami', { encoding: 'utf-8' }).trim();
execFileSync('icacls', [
this.keyPath,
'/inheritance:r',
'/grant:r',
`${currentUser}:F`,
], { stdio: 'ignore' });
} catch (err) {
log.warn('Could not restrict key file permissions via icacls — vault key may be readable by other users', err);
}
}
return key;
}
/** Encrypt a plaintext string. Returns iv:authTag:ciphertext (all hex). */
private encrypt(plaintext: string): string {
const iv = crypto.randomBytes(IV_LENGTH);
const cipher = crypto.createCipheriv(ALGORITHM, this.encryptionKey, iv);
const encrypted = Buffer.concat([cipher.update(plaintext, 'utf-8'), cipher.final()]);
const authTag = cipher.getAuthTag();
return `${iv.toString('hex')}:${authTag.toString('hex')}:${encrypted.toString('hex')}`;
}
/** Decrypt an encoded string (iv:authTag:ciphertext, all hex). */
private decrypt(encoded: string): string {
const parts = encoded.split(':');
if (parts.length !== 3) {
throw new Error('Vault entry format invalid — expected iv:authTag:ciphertext');
}
const [ivHex, authTagHex, ciphertextHex] = parts;
const iv = Buffer.from(ivHex, 'hex');
const authTag = Buffer.from(authTagHex, 'hex');
const ciphertext = Buffer.from(ciphertextHex, 'hex');
const decipher = crypto.createDecipheriv(ALGORITHM, this.encryptionKey, iv);
decipher.setAuthTag(authTag);
return decipher.update(ciphertext) + decipher.final('utf-8');
}
/** Read the vault file (encrypted entries). */
private readVault(): Record<string, VaultRecord> {
if (!fs.existsSync(this.vaultPath)) return {};
try {
// Review Critical #3: use a null-prototype object so a malicious `__proto__`
// key in vault.json cannot pollute Object.prototype when accessed. Matches
// the `name` validation at the route layer.
const parsed = JSON.parse(fs.readFileSync(this.vaultPath, 'utf-8'));
return Object.assign(Object.create(null), parsed);
} catch (err) {
// M5: corrupt vault.json — back up the corrupt file so data isn't silently lost
log.error('Failed to parse vault.json — backing up corrupt file', err);
const bakPath = this.vaultPath + '.bak';
try { fs.copyFileSync(this.vaultPath, bakPath); } catch { /* best effort */ }
return {};
}
}
/** Write the vault file atomically (write to .tmp, then replace). */
private writeVault(vault: Record<string, VaultRecord>): void {
const tmpPath = this.vaultPath + '.tmp';
const data = JSON.stringify(vault, null, 2);
fs.writeFileSync(tmpPath, data, { mode: 0o600 });
try {
try {
fs.renameSync(tmpPath, this.vaultPath);
} catch {
// M6: On Windows, rename fails with EPERM when target exists.
// Delete target first, then rename — preserves atomicity better than direct write.
try { fs.unlinkSync(this.vaultPath); } catch { /* target may not exist */ }
try {
fs.renameSync(tmpPath, this.vaultPath);
} catch (renameErr) {
// Last resort: direct write (non-atomic) — log the degradation
log.error('Atomic vault write failed, falling back to direct write', renameErr);
fs.writeFileSync(this.vaultPath, data, { mode: 0o600 });
}
}
} finally {
// M12: clean up .tmp if it still exists after any failure path
try { if (fs.existsSync(tmpPath)) fs.unlinkSync(tmpPath); } catch { /* best effort */ }
}
}
/** Set a secret (encrypts value, stores metadata alongside). Uses atomic write. */
set(name: string, value: string, metadata?: Record<string, unknown>): void {
const vault = this.readVault();
vault[name] = {
encrypted: this.encrypt(value),
metadata,
updatedAt: new Date().toISOString(),
};
this.writeVault(vault);
}
/**
* Async set — delegates to the synchronous set().
* Review Critical #2: the previous implementation used a promise chain
* (.writeLock.then(...)) which deferred the actual write to a microtask.
* A sync set() between the chain call and microtask execution would be
* overwritten when the deferred write fired. Since all vault I/O is
* synchronous (fs.readFileSync/writeFileSync), the lock is unnecessary —
* each callback runs to completion before the next microtask. Delegating
* to the sync method eliminates the interleaving window entirely.
*/
async setAsync(name: string, value: string, metadata?: Record<string, unknown>): Promise<void> {
this.set(name, value, metadata);
}
/** Get a decrypted secret by name. Returns null if not found or decryption fails. */
get(name: string): VaultEntry | null {
const vault = this.readVault();
const entry = vault[name];
if (!entry) return null;
try {
return {
name,
value: this.decrypt(entry.encrypted),
metadata: entry.metadata,
updatedAt: entry.updatedAt,
};
} catch {
return null; // Decryption failed (key mismatch, corrupted)
}
}
/** Delete a secret. Returns true if it existed. Uses atomic write. */
delete(name: string): boolean {
const vault = this.readVault();
if (!vault[name]) return false;
delete vault[name];
this.writeVault(vault);
return true;
}
/**
* Async delete — delegates to the synchronous delete().
* Review Critical #2: same rationale as setAsync — sync I/O means no
* interleaving risk. The previous promise-chain approach deferred the
* actual delete, creating a window where a sync caller's write could
* be overwritten.
*/
async deleteAsync(name: string): Promise<boolean> {
return this.delete(name);
}
/** List secret names (without values). */
list(): Array<{ name: string; metadata?: Record<string, unknown>; updatedAt: string }> {
const vault = this.readVault();
return Object.entries(vault).map(([name, entry]) => ({
name,
metadata: entry.metadata,
updatedAt: entry.updatedAt,
}));
}
/** Check if a secret exists. */
has(name: string): boolean {
const vault = this.readVault();
return name in vault;
}
/** Set a connector credential with typed metadata */
setConnectorCredential(connectorId: string, credential: {
type: 'api_key' | 'oauth2' | 'bearer' | 'basic';
value: string;
refreshToken?: string;
expiresAt?: string;
scopes?: string[];
}): void {
this.set(`connector:${connectorId}`, credential.value, {
credentialType: credential.type,
expiresAt: credential.expiresAt,
scopes: credential.scopes,
});
// Store refresh token as a separate encrypted entry (never in plaintext metadata)
if (credential.refreshToken) {
this.set(`connector:${connectorId}:refresh`, credential.refreshToken);
} else {
// Clear any previously stored refresh token if not provided
this.delete(`connector:${connectorId}:refresh`);
}
}
/** Get a connector credential with typed metadata */
getConnectorCredential(connectorId: string): {
value: string;
type: string;
refreshToken?: string;
expiresAt?: string;
scopes?: string[];
isExpired: boolean;
} | null {
const entry = this.get(`connector:${connectorId}`);
if (!entry) return null;
const expiresAt = entry.metadata?.expiresAt as string | undefined;
// Retrieve refresh token from its own encrypted entry
const refreshEntry = this.get(`connector:${connectorId}:refresh`);
return {
value: entry.value,
type: (entry.metadata?.credentialType as string) ?? 'api_key',
// Review Major #7: removed dead plaintext metadata fallback that contradicted
// the "never in plaintext metadata" security contract from setConnectorCredential.
refreshToken: refreshEntry?.value,
expiresAt,
scopes: entry.metadata?.scopes as string[] | undefined,
isExpired: expiresAt ? new Date(expiresAt) < new Date() : false,
};
}
/** Migrate plaintext providers from config.json to vault. Returns count migrated. */
migrateFromConfig(config: { providers?: Record<string, { apiKey: string; models?: string[]; baseUrl?: string }> }): number {
if (!config.providers) return 0;
let migrated = 0;
for (const [name, provider] of Object.entries(config.providers)) {
if (provider.apiKey && !this.has(name)) {
this.set(name, provider.apiKey, {
models: provider.models,
baseUrl: provider.baseUrl,
});
migrated++;
}
}
return migrated;
}
}

View File

@@ -0,0 +1,254 @@
/**
* ComplianceTemplateStore unit tests (M-03)
*
* Covers CRUD + section merge semantics + default section fill-in + risk-class CHECK
* constraint + deletion. Following HarvestRunStore test shape (in-memory MindDB).
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { MindDB } from '@waggle/hive-mind-core';
import { ComplianceTemplateStore, KVARK_TEMPLATE_NAME } from '../../src/compliance/template-store.js';
import type { ComplianceTemplateSections } from '../../src/compliance/types.js';
const ALL_ON: ComplianceTemplateSections = {
interactions: true,
oversight: true,
models: true,
provenance: true,
riskAssessment: true,
fria: true,
};
const ALL_OFF: ComplianceTemplateSections = {
interactions: false,
oversight: false,
models: false,
provenance: false,
riskAssessment: false,
fria: false,
};
describe('ComplianceTemplateStore', () => {
let db: MindDB;
let store: ComplianceTemplateStore;
beforeEach(() => {
db = new MindDB(':memory:');
store = new ComplianceTemplateStore(db);
});
afterEach(() => {
db.close();
});
describe('create', () => {
it('persists a template with all fields', () => {
const t = store.create({
name: 'KVARK enterprise',
description: 'Full AI Act package for enterprise on-prem',
sections: ALL_ON,
riskClassification: 'high-risk',
orgName: 'KVARK Sovereign',
footerText: 'Confidential — internal only',
});
expect(t.id).toBeGreaterThan(0);
expect(t.name).toBe('KVARK enterprise');
expect(t.description).toBe('Full AI Act package for enterprise on-prem');
expect(t.sections).toEqual(ALL_ON);
expect(t.riskClassification).toBe('high-risk');
expect(t.orgName).toBe('KVARK Sovereign');
expect(t.footerText).toBe('Confidential — internal only');
expect(t.createdAt).toBeTruthy();
expect(t.updatedAt).toBeTruthy();
});
it('defaults optional fields to null', () => {
const t = store.create({ name: 'Bare', sections: ALL_OFF });
expect(t.description).toBeNull();
expect(t.riskClassification).toBeNull();
expect(t.orgName).toBeNull();
expect(t.footerText).toBeNull();
});
it('trims whitespace from name', () => {
const t = store.create({ name: ' Spaced ', sections: ALL_OFF });
expect(t.name).toBe('Spaced');
});
it('rejects empty name', () => {
expect(() => store.create({ name: ' ', sections: ALL_OFF })).toThrow(/name is required/i);
});
it('fills in missing section keys with defaults', () => {
const t = store.create({
name: 'Partial',
// Only interactions specified; others should fall back to DEFAULT_SECTIONS.
sections: { interactions: false } as ComplianceTemplateSections,
});
expect(t.sections.interactions).toBe(false);
expect(t.sections.oversight).toBe(true); // default
expect(t.sections.fria).toBe(false); // default
});
});
describe('getById', () => {
it('returns null for missing id', () => {
expect(store.getById(999)).toBeNull();
});
it('round-trips a created template', () => {
const a = store.create({ name: 'A', sections: ALL_ON, riskClassification: 'limited' });
const b = store.getById(a.id);
expect(b).not.toBeNull();
expect(b?.name).toBe('A');
expect(b?.sections).toEqual(ALL_ON);
expect(b?.riskClassification).toBe('limited');
});
});
describe('list', () => {
it('returns empty array when none exist', () => {
expect(store.list()).toEqual([]);
});
it('returns templates newest-updated first', async () => {
store.create({ name: 'Old', sections: ALL_OFF });
await new Promise(resolve => setTimeout(resolve, 1100)); // datetime('now') is second-resolution
store.create({ name: 'New', sections: ALL_OFF });
const list = store.list();
expect(list).toHaveLength(2);
expect(list[0].name).toBe('New');
expect(list[1].name).toBe('Old');
});
});
describe('update', () => {
it('returns null for missing id', () => {
expect(store.update(999, { name: 'nope' })).toBeNull();
});
it('updates only the fields provided', () => {
const t = store.create({
name: 'Orig',
description: 'desc',
sections: ALL_ON,
orgName: 'Acme',
});
const updated = store.update(t.id, { name: 'Renamed' });
expect(updated?.name).toBe('Renamed');
expect(updated?.description).toBe('desc'); // preserved
expect(updated?.sections).toEqual(ALL_ON); // preserved
expect(updated?.orgName).toBe('Acme'); // preserved
});
it('clears a field when explicitly set to null', () => {
const t = store.create({
name: 'T',
sections: ALL_ON,
orgName: 'To remove',
footerText: 'also remove',
});
const updated = store.update(t.id, { orgName: null, footerText: null });
expect(updated?.orgName).toBeNull();
expect(updated?.footerText).toBeNull();
});
it('replaces sections wholesale', () => {
const t = store.create({ name: 'T', sections: ALL_ON });
const updated = store.update(t.id, { sections: ALL_OFF });
expect(updated?.sections).toEqual(ALL_OFF);
});
it('rejects an empty rename', () => {
const t = store.create({ name: 'T', sections: ALL_OFF });
expect(() => store.update(t.id, { name: ' ' })).toThrow(/name is required/i);
});
it('bumps updated_at', async () => {
const t = store.create({ name: 'T', sections: ALL_OFF });
await new Promise(resolve => setTimeout(resolve, 1100));
const updated = store.update(t.id, { name: 'T2' });
expect(updated?.updatedAt).not.toBe(t.updatedAt);
});
});
describe('delete', () => {
it('returns false for missing id', () => {
expect(store.delete(999)).toBe(false);
});
it('removes an existing row and returns true', () => {
const t = store.create({ name: 'T', sections: ALL_OFF });
expect(store.delete(t.id)).toBe(true);
expect(store.getById(t.id)).toBeNull();
});
});
describe('risk_classification CHECK constraint', () => {
it('accepts all four AIActRiskLevel values', () => {
for (const level of ['minimal', 'limited', 'high-risk', 'unacceptable'] as const) {
const t = store.create({ name: `T-${level}`, sections: ALL_OFF, riskClassification: level });
expect(t.riskClassification).toBe(level);
}
});
it('rejects an invalid risk class at the SQL layer', () => {
expect(() =>
store.create({
name: 'Bad',
sections: ALL_OFF,
riskClassification: 'super-dangerous' as never,
}),
).toThrow(); // better-sqlite3 surfaces the CHECK violation
});
});
describe('seedKvarkTemplateIfMissing (M-06)', () => {
it('creates the KVARK template on first run', () => {
const seeded = store.seedKvarkTemplateIfMissing();
expect(seeded).not.toBeNull();
expect(seeded!.name).toBe(KVARK_TEMPLATE_NAME);
expect(seeded!.riskClassification).toBe('high-risk');
expect(seeded!.sections.fria).toBe(true);
expect(seeded!.orgName).toContain('KVARK');
expect(seeded!.footerText).toContain('sovereign');
});
it('is idempotent — second call returns null', () => {
store.seedKvarkTemplateIfMissing();
expect(store.seedKvarkTemplateIfMissing()).toBeNull();
// Only one KVARK row exists.
const all = store.list().filter(t => t.name === KVARK_TEMPLATE_NAME);
expect(all).toHaveLength(1);
});
it('does not seed when the user already created a same-named template', () => {
store.create({
name: KVARK_TEMPLATE_NAME,
sections: ALL_OFF,
});
expect(store.seedKvarkTemplateIfMissing()).toBeNull();
});
});
describe('mergeSections (static)', () => {
it('unions template + runtime flags', () => {
const template: ComplianceTemplateSections = { ...ALL_OFF, interactions: true, fria: true };
const runtime: ComplianceTemplateSections = { ...ALL_OFF, oversight: true };
const merged = ComplianceTemplateStore.mergeSections(template, runtime);
expect(merged.interactions).toBe(true); // from template
expect(merged.oversight).toBe(true); // from runtime
expect(merged.fria).toBe(true); // from template
expect(merged.models).toBe(false);
});
it('never hides a section the runtime requested (union semantics)', () => {
const merged = ComplianceTemplateStore.mergeSections(ALL_OFF, ALL_ON);
expect(merged).toEqual(ALL_ON);
});
it('never hides a section the template requested', () => {
const merged = ComplianceTemplateStore.mergeSections(ALL_ON, ALL_OFF);
expect(merged).toEqual(ALL_ON);
});
});
});

View File

@@ -0,0 +1,245 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { WaggleConfig, type ProviderEntry, type TeamServerConfig } from '../src/config.js';
describe('WaggleConfig', () => {
const tempDirs: string[] = [];
function makeTempDir(): string {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-config-test-'));
tempDirs.push(dir);
return dir;
}
afterEach(() => {
for (const dir of tempDirs) {
fs.rmSync(dir, { recursive: true, force: true });
}
tempDirs.length = 0;
});
it('creates config directory if missing', () => {
const base = makeTempDir();
const configDir = path.join(base, 'nested', '.waggle');
new WaggleConfig(configDir);
expect(fs.existsSync(configDir)).toBe(true);
});
it('returns default config when no file exists', () => {
const configDir = makeTempDir();
const config = new WaggleConfig(configDir);
expect(config.getDefaultModel()).toBe('claude-sonnet-4-6');
expect(config.getProviders()).toEqual({});
expect(config.getMindPath()).toBe(path.join(configDir, 'default.mind'));
});
it('saves and loads provider config', () => {
const configDir = makeTempDir();
const config = new WaggleConfig(configDir);
const provider: ProviderEntry = {
apiKey: 'sk-test-key',
models: ['claude-sonnet-4-6', 'claude-haiku-3'],
baseUrl: 'https://api.anthropic.com',
};
config.setProvider('anthropic', provider);
config.save();
// Load fresh instance from same directory
const config2 = new WaggleConfig(configDir);
const providers = config2.getProviders();
expect(providers['anthropic']).toEqual(provider);
expect(providers['anthropic'].apiKey).toBe('sk-test-key');
expect(providers['anthropic'].models).toHaveLength(2);
});
it('sets and gets default model', () => {
const configDir = makeTempDir();
const config = new WaggleConfig(configDir);
config.setDefaultModel('gpt-4o');
config.save();
const config2 = new WaggleConfig(configDir);
expect(config2.getDefaultModel()).toBe('gpt-4o');
});
it('returns mind file path', () => {
const configDir = makeTempDir();
const config = new WaggleConfig(configDir);
expect(config.getMindPath()).toBe(path.join(configDir, 'default.mind'));
});
it('removes a provider', () => {
const configDir = makeTempDir();
const config = new WaggleConfig(configDir);
config.setProvider('anthropic', { apiKey: 'key1', models: ['m1'] });
config.setProvider('openai', { apiKey: 'key2', models: ['m2'] });
config.removeProvider('anthropic');
config.save();
const config2 = new WaggleConfig(configDir);
const providers = config2.getProviders();
expect(providers['anthropic']).toBeUndefined();
expect(providers['openai']).toBeDefined();
});
it('returns config directory path', () => {
const configDir = makeTempDir();
const config = new WaggleConfig(configDir);
expect(config.getConfigDir()).toBe(configDir);
});
describe('team server config', () => {
it('returns null when no team server configured', () => {
const configDir = makeTempDir();
const config = new WaggleConfig(configDir);
expect(config.getTeamServer()).toBeNull();
expect(config.isTeamConnected()).toBe(false);
});
it('sets and gets team server config', () => {
const configDir = makeTempDir();
const config = new WaggleConfig(configDir);
const teamConfig: TeamServerConfig = {
url: 'https://team.waggle.dev',
token: 'clerk-jwt-token',
userId: 'user-123',
displayName: 'Marko',
};
config.setTeamServer(teamConfig);
config.save();
const config2 = new WaggleConfig(configDir);
const loaded = config2.getTeamServer();
expect(loaded).toEqual(teamConfig);
expect(config2.isTeamConnected()).toBe(true);
});
it('clears team server config', () => {
const configDir = makeTempDir();
const config = new WaggleConfig(configDir);
config.setTeamServer({ url: 'https://team.waggle.dev' });
config.clearTeamServer();
config.save();
const config2 = new WaggleConfig(configDir);
expect(config2.getTeamServer()).toBeNull();
expect(config2.isTeamConnected()).toBe(false);
});
it('persists team server through save/load cycle', () => {
const configDir = makeTempDir();
const config = new WaggleConfig(configDir);
config.setTeamServer({ url: 'https://example.com', userId: 'u1' });
config.save();
const config2 = new WaggleConfig(configDir);
expect(config2.getTeamServer()!.url).toBe('https://example.com');
expect(config2.getTeamServer()!.userId).toBe('u1');
});
});
describe('governed CLI config', () => {
it('normalizes, deduplicates, and persists the CLI allowlist', () => {
const configDir = makeTempDir();
const config = new WaggleConfig(configDir);
config.setCliAllowlist([' node ', 'NODE', '', 'git']);
expect(config.getCliAllowlist()).toEqual(['node', 'git']);
config.save();
const config2 = new WaggleConfig(configDir);
expect(config2.getCliAllowlist()).toEqual(['node', 'git']);
});
});
describe('Model Pilot config fields', () => {
let tmpDir: string;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-config-pilot-'));
});
afterEach(() => {
fs.rmSync(tmpDir, { recursive: true, force: true });
});
it('returns null for fallbackModel when not set', () => {
const config = new WaggleConfig(tmpDir);
expect(config.getFallbackModel()).toBeNull();
});
it('persists fallbackModel', () => {
const config = new WaggleConfig(tmpDir);
config.setFallbackModel('qwen/qwen3.6-plus:free');
config.save();
const config2 = new WaggleConfig(tmpDir);
expect(config2.getFallbackModel()).toBe('qwen/qwen3.6-plus:free');
});
it('returns null for budgetModel when not set', () => {
const config = new WaggleConfig(tmpDir);
expect(config.getBudgetModel()).toBeNull();
});
it('persists budgetModel', () => {
const config = new WaggleConfig(tmpDir);
config.setBudgetModel('deepseek/deepseek-chat-v3-0324:free');
config.save();
const config2 = new WaggleConfig(tmpDir);
expect(config2.getBudgetModel()).toBe('deepseek/deepseek-chat-v3-0324:free');
});
it('returns 0.8 as default budgetThreshold', () => {
const config = new WaggleConfig(tmpDir);
expect(config.getBudgetThreshold()).toBe(0.8);
});
it('persists budgetThreshold', () => {
const config = new WaggleConfig(tmpDir);
config.setBudgetThreshold(0.6);
config.save();
const config2 = new WaggleConfig(tmpDir);
expect(config2.getBudgetThreshold()).toBe(0.6);
});
it('clearFallbackModel removes the field', () => {
const config = new WaggleConfig(tmpDir);
config.setFallbackModel('test-model');
config.save();
config.clearFallbackModel();
config.save();
const config2 = new WaggleConfig(tmpDir);
expect(config2.getFallbackModel()).toBeNull();
});
it('returns 90 as default maxIterations', () => {
const config = new WaggleConfig(tmpDir);
expect(config.getMaxIterations()).toBe(90);
});
it('persists maxIterations', () => {
const config = new WaggleConfig(tmpDir);
config.setMaxIterations(50);
config.save();
const config2 = new WaggleConfig(tmpDir);
expect(config2.getMaxIterations()).toBe(50);
});
});
});

View File

@@ -0,0 +1,268 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { MindDB } from '@waggle/hive-mind-core';
import { CronStore, type CreateScheduleInput, type SavePendingActionInput } from '../src/cron-store.js';
describe('CronStore', () => {
let tmpDir: string;
let db: MindDB;
let store: CronStore;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-cron-'));
db = new MindDB(path.join(tmpDir, 'test.mind'));
store = new CronStore(db);
});
afterEach(() => {
db.close();
fs.rmSync(tmpDir, { recursive: true, force: true });
});
function makeInput(overrides?: Partial<CreateScheduleInput>): CreateScheduleInput {
return {
name: 'Daily backup',
cronExpr: '0 9 * * *', // every day at 9am
jobType: 'memory_consolidation',
...overrides,
};
}
it('create with valid cron computes next_run_at in the future', () => {
const schedule = store.create(makeInput());
expect(schedule.id).toBeGreaterThan(0);
expect(schedule.name).toBe('Daily backup');
expect(schedule.cron_expr).toBe('0 9 * * *');
expect(schedule.job_type).toBe('memory_consolidation');
expect(schedule.enabled).toBe(1);
expect(schedule.next_run_at).toBeTruthy();
// next_run_at should be in the future
expect(new Date(schedule.next_run_at!).getTime()).toBeGreaterThan(Date.now());
});
it('create with invalid cron expression throws', () => {
expect(() => store.create(makeInput({ cronExpr: 'not a cron' }))).toThrow();
});
it('create agent_task without workspaceId throws', () => {
expect(() =>
store.create(makeInput({ jobType: 'agent_task' })),
).toThrow(/workspace/i);
});
it('create loop job type round-trips with its job_config', () => {
const schedule = store.create(makeInput({
jobType: 'loop',
jobConfig: { prompt: 'Summarize what changed in this workspace.' },
}));
expect(schedule.job_type).toBe('loop');
expect(JSON.parse(schedule.job_config).prompt).toBe('Summarize what changed in this workspace.');
});
it('create loop without workspaceId succeeds (loops can run on the personal mind)', () => {
const schedule = store.create(makeInput({ jobType: 'loop' }));
expect(schedule.job_type).toBe('loop');
expect(schedule.workspace_id).toBeNull();
});
it('create agent_task with workspaceId succeeds', () => {
const schedule = store.create(makeInput({
jobType: 'agent_task',
workspaceId: 'ws-123',
}));
expect(schedule.job_type).toBe('agent_task');
expect(schedule.workspace_id).toBe('ws-123');
});
it('list returns schedules ordered by name', () => {
store.create(makeInput({ name: 'Zebra task' }));
store.create(makeInput({ name: 'Alpha task' }));
store.create(makeInput({ name: 'Middle task' }));
const list = store.list();
expect(list).toHaveLength(3);
expect(list[0].name).toBe('Alpha task');
expect(list[1].name).toBe('Middle task');
expect(list[2].name).toBe('Zebra task');
});
it('getById returns the schedule when found', () => {
const created = store.create(makeInput({ name: 'Findable' }));
const found = store.getById(created.id);
expect(found).toBeDefined();
expect(found!.name).toBe('Findable');
});
it('getById returns undefined when not found', () => {
const found = store.getById(99999);
expect(found).toBeUndefined();
});
it('update changes name and enabled', () => {
const created = store.create(makeInput());
store.update(created.id, { name: 'Renamed', enabled: false });
const updated = store.getById(created.id)!;
expect(updated.name).toBe('Renamed');
expect(updated.enabled).toBe(0);
});
it('update cronExpr recomputes next_run_at', () => {
const created = store.create(makeInput({ cronExpr: '0 9 * * *' }));
const originalNext = created.next_run_at;
// Change to every minute — next_run_at should change
store.update(created.id, { cronExpr: '*/1 * * * *' });
const updated = store.getById(created.id)!;
expect(updated.cron_expr).toBe('*/1 * * * *');
expect(updated.next_run_at).toBeTruthy();
// The new next_run_at should differ (different schedule)
// Both should be valid ISO dates
expect(new Date(updated.next_run_at!).getTime()).toBeGreaterThan(0);
});
it('delete removes the schedule', () => {
const created = store.create(makeInput());
store.delete(created.id);
expect(store.getById(created.id)).toBeUndefined();
});
it('getDue returns only enabled past-due schedules', () => {
const raw = db.getDatabase();
// Create two schedules: one past-due, one future
store.create(makeInput({ name: 'Future job' }));
// Manually insert a past-due schedule
raw.prepare(`
INSERT INTO cron_schedules (name, cron_expr, job_type, job_config, enabled, next_run_at, created_at)
VALUES (?, ?, ?, ?, 1, datetime('now', '-1 hour'), datetime('now'))
`).run('Past due job', '0 9 * * *', 'memory_consolidation', '{}');
const due = store.getDue();
expect(due).toHaveLength(1);
expect(due[0].name).toBe('Past due job');
});
it('getDue excludes disabled schedules', () => {
const raw = db.getDatabase();
// Insert a past-due but disabled schedule
raw.prepare(`
INSERT INTO cron_schedules (name, cron_expr, job_type, job_config, enabled, next_run_at, created_at)
VALUES (?, ?, ?, ?, 0, datetime('now', '-1 hour'), datetime('now'))
`).run('Disabled job', '0 9 * * *', 'memory_consolidation', '{}');
const due = store.getDue();
expect(due).toHaveLength(0);
});
it('markRun updates last_run_at and recomputes next_run_at', () => {
const created = store.create(makeInput({ cronExpr: '0 9 * * *' }));
expect(created.last_run_at).toBeNull();
store.markRun(created.id);
const updated = store.getById(created.id)!;
expect(updated.last_run_at).toBeTruthy();
expect(updated.next_run_at).toBeTruthy();
// next_run_at should be in the future
expect(new Date(updated.next_run_at!).getTime()).toBeGreaterThan(Date.now());
});
it('pruneExecutionHistory deletes only rows older than the cutoff', () => {
const created = store.create(makeInput());
// A fresh row (executed_at = now) must survive the prune.
store.recordExecution(created.id, created.name, { success: true });
// A back-dated row beyond the 30-day retention must go.
db.getDatabase().prepare(`
INSERT INTO cron_execution_history (schedule_id, schedule_name, executed_at, success)
VALUES (?, ?, datetime('now', '-40 days'), 1)
`).run(created.id, created.name);
expect(store.getExecutionHistory(created.id)).toHaveLength(2);
const deleted = store.pruneExecutionHistory(30);
expect(deleted).toBe(1);
const remaining = store.getExecutionHistory(created.id);
expect(remaining).toHaveLength(1);
expect(remaining[0].success).toBe(1);
});
it('countExecutionsToday counts only today, only this schedule (#17 daily cap)', () => {
const created = store.create(makeInput());
const other = store.create(makeInput({ name: 'Other job' }));
store.recordExecution(created.id, created.name, { success: true });
store.recordExecution(created.id, created.name, { success: false });
store.recordExecution(other.id, other.name, { success: true });
// Yesterday's row must not count.
db.getDatabase().prepare(`
INSERT INTO cron_execution_history (schedule_id, schedule_name, executed_at, success)
VALUES (?, ?, datetime('now', '-1 day'), 1)
`).run(created.id, created.name);
expect(store.countExecutionsToday(created.id)).toBe(2);
expect(store.countExecutionsToday(other.id)).toBe(1);
});
describe('pending_actions (L2 held-action queue)', () => {
function held(over?: Partial<SavePendingActionInput>): SavePendingActionInput {
return {
id: 'pa-1', workspaceId: null, source: 'loop:1', toolName: 'send_email',
argsJson: JSON.stringify({ to: 'x@y.z', subject: 'hi' }), summary: 'Send follow-up',
riskLevel: 'medium', approvalClass: 'elevated', ...over,
};
}
it('saves and lists held actions, filtered by status', () => {
store.savePendingAction(held());
store.savePendingAction(held({ id: 'pa-2', toolName: 'write_file' }));
const heldRows = store.listPendingActions('held');
expect(heldRows).toHaveLength(2);
expect(heldRows.map(r => r.tool_name).sort()).toEqual(['send_email', 'write_file']);
expect(store.listPendingActions('executed')).toHaveLength(0);
});
it('claimPendingAction is an atomic idempotency gate (double-claim no-ops)', () => {
store.savePendingAction(held());
const first = store.claimPendingAction('pa-1', 'approved', '2026-06-29T10:00:00Z');
expect(first?.status).toBe('approved');
expect(first?.decided_at).toBe('2026-06-29T10:00:00Z');
// A second claim (double-approve / approve-after-deny) wins nothing.
expect(store.claimPendingAction('pa-1', 'approved', '2026-06-29T10:05:00Z')).toBeUndefined();
expect(store.claimPendingAction('pa-1', 'denied', '2026-06-29T10:05:00Z')).toBeUndefined();
// Unknown id → undefined.
expect(store.claimPendingAction('nope', 'approved', '2026-06-29T10:00:00Z')).toBeUndefined();
});
it('updatePendingActionResult records the terminal outcome after a claim', () => {
store.savePendingAction(held());
store.claimPendingAction('pa-1', 'approved', '2026-06-29T10:00:00Z');
store.updatePendingActionResult('pa-1', { status: 'executed', resultSummary: 'sent', executedAt: '2026-06-29T10:01:00Z' });
const row = store.getPendingAction('pa-1');
expect(row?.status).toBe('executed');
expect(row?.result_summary).toBe('sent');
expect(row?.executed_at).toBe('2026-06-29T10:01:00Z');
});
it('persists held actions across a DB reopen (durable queue)', () => {
store.savePendingAction(held());
db.close();
db = new MindDB(path.join(tmpDir, 'test.mind'));
store = new CronStore(db);
const rows = store.listPendingActions('held');
expect(rows).toHaveLength(1);
expect(rows[0].id).toBe('pa-1');
});
it('expireStalePendingActions flips past-due held rows to expired', () => {
store.savePendingAction(held({ expiresAt: '2000-01-01T00:00:00Z' })); // long past
store.savePendingAction(held({ id: 'pa-2', expiresAt: '2999-01-01T00:00:00Z' })); // future
expect(store.expireStalePendingActions()).toBe(1);
expect(store.listPendingActions('held').map(r => r.id)).toEqual(['pa-2']);
expect(store.listPendingActions('expired').map(r => r.id)).toEqual(['pa-1']);
});
});
});

View File

@@ -0,0 +1,267 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import Database from 'better-sqlite3';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { createEmbeddingProvider, EmbeddingQuotaExceededError, getMinimumTierForProvider } from '@waggle/hive-mind-core';
import { TierError, TIER_CAPABILITIES } from '@waggle/shared';
describe('Embedding Provider — Tier & Quota Enforcement', () => {
let tmpDir: string;
let db: InstanceType<typeof Database>;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-embed-test-'));
db = new Database(path.join(tmpDir, 'quota.db'));
});
afterEach(() => {
db.close();
fs.rmSync(tmpDir, { recursive: true, force: true });
});
describe('Tier enforcement on provider selection', () => {
it('FREE user requesting litellm throws TierError (litellm stays Team-only)', async () => {
// Solo (FREE) unlocks BYO cloud embeddings (voyage/openai) but NOT the
// managed litellm router — that stays a paid-tier provider.
await expect(
createEmbeddingProvider({
provider: 'litellm',
userTier: 'FREE',
quotaDb: db,
})
).rejects.toThrow(TierError);
});
it('FREE user requesting inprocess succeeds', async () => {
// inprocess may fail to load ONNX in test env, but should NOT throw TierError
try {
await createEmbeddingProvider({
provider: 'inprocess',
userTier: 'FREE',
quotaDb: db,
});
} catch (err) {
// If it fails, it should NOT be a TierError — it should be a probe failure
expect(err).not.toBeInstanceOf(TierError);
}
});
it('FREE user requesting voyage does not throw TierError (Solo unlocks BYO cloud embeddings)', async () => {
// voyage will fail to connect (no real API), but should NOT throw TierError —
// Solo now allows the voyage/openai providers directly.
const provider = await createEmbeddingProvider({
provider: 'auto',
userTier: 'FREE',
quotaDb: db,
});
// Should fall back to mock (no real providers in test), but no TierError
expect(provider.getActiveProvider()).toBeDefined();
});
it('auto mode surfaces only key-backed providers for FREE tier', async () => {
const provider = await createEmbeddingProvider({
provider: 'auto',
userTier: 'FREE',
quotaDb: db,
});
// FREE allows inprocess, mock, ollama + cloud (voyage/openai); cloud is
// skipped here without API keys, so only local providers + mock surface.
const status = provider.getStatus();
for (const p of status.availableProviders) {
expect(['inprocess', 'mock', 'ollama']).toContain(p);
}
});
});
describe('Quota enforcement', () => {
// All current tiers have unlimited quotas (-1), so we patch FREE
// to a finite quota for these tests to exercise the quota mechanism.
const originalQuota = TIER_CAPABILITIES.FREE.embeddingQuotaPerMonth;
beforeEach(() => {
(TIER_CAPABILITIES.FREE as { embeddingQuotaPerMonth: number }).embeddingQuotaPerMonth = 500;
});
afterEach(() => {
(TIER_CAPABILITIES.FREE as { embeddingQuotaPerMonth: number }).embeddingQuotaPerMonth = originalQuota;
});
it('FREE user at 499 embeddings succeeds', async () => {
const provider = await createEmbeddingProvider({
provider: 'mock',
userTier: 'FREE',
userId: 'test-user',
quotaDb: db,
});
// Pre-fill 499 embeddings
const ym = `${new Date().getFullYear()}-${String(new Date().getMonth() + 1).padStart(2, '0')}`;
db.prepare('INSERT INTO embedding_usage (user_id, year_month, count, updated_at) VALUES (?, ?, 499, ?)').run('test-user', ym, Date.now());
// 500th should succeed (499 + 1 = 500 = quota)
const result = await provider.embed('test text');
expect(result).toBeInstanceOf(Float32Array);
});
it('FREE user at 500 embeddings throws EmbeddingQuotaExceededError', async () => {
const provider = await createEmbeddingProvider({
provider: 'mock',
userTier: 'FREE',
userId: 'test-user',
quotaDb: db,
});
// Pre-fill to quota limit
const ym = `${new Date().getFullYear()}-${String(new Date().getMonth() + 1).padStart(2, '0')}`;
db.prepare('INSERT INTO embedding_usage (user_id, year_month, count, updated_at) VALUES (?, ?, 500, ?)').run('test-user', ym, Date.now());
await expect(provider.embed('test text')).rejects.toThrow(EmbeddingQuotaExceededError);
});
it('TEAMS user with unlimited quota never throws quota error', async () => {
const provider = await createEmbeddingProvider({
provider: 'mock',
userTier: 'TEAMS',
userId: 'test-user',
quotaDb: db,
});
// Even with high usage, should succeed (TEAMS has -1 = unlimited)
const ym = `${new Date().getFullYear()}-${String(new Date().getMonth() + 1).padStart(2, '0')}`;
db.prepare('INSERT INTO embedding_usage (user_id, year_month, count, updated_at) VALUES (?, ?, 999999, ?)').run('test-user', ym, Date.now());
const result = await provider.embed('test text');
expect(result).toBeInstanceOf(Float32Array);
});
});
describe('getQuotaStatus', () => {
const originalQuota = TIER_CAPABILITIES.FREE.embeddingQuotaPerMonth;
beforeEach(() => {
(TIER_CAPABILITIES.FREE as { embeddingQuotaPerMonth: number }).embeddingQuotaPerMonth = 500;
});
afterEach(() => {
(TIER_CAPABILITIES.FREE as { embeddingQuotaPerMonth: number }).embeddingQuotaPerMonth = originalQuota;
});
it('returns correct percentage for FREE user', async () => {
const provider = await createEmbeddingProvider({
provider: 'mock',
userTier: 'FREE',
userId: 'test-user',
quotaDb: db,
});
// Use 250 of 500 quota
const ym = `${new Date().getFullYear()}-${String(new Date().getMonth() + 1).padStart(2, '0')}`;
db.prepare('INSERT INTO embedding_usage (user_id, year_month, count, updated_at) VALUES (?, ?, 250, ?)').run('test-user', ym, Date.now());
const status = provider.getQuotaStatus();
expect(status.tier).toBe('FREE');
expect(status.quota).toBe(500);
expect(status.used).toBe(250);
expect(status.remaining).toBe(250);
expect(status.percentage).toBe(50);
expect(status.resetsAt).toBeTruthy();
});
it('returns unlimited for TEAMS tier', async () => {
const provider = await createEmbeddingProvider({
provider: 'mock',
userTier: 'TEAMS',
userId: 'test-user',
quotaDb: db,
});
const status = provider.getQuotaStatus();
expect(status.tier).toBe('TEAMS');
expect(status.quota).toBe(-1);
expect(status.remaining).toBe(-1);
expect(status.percentage).toBe(0);
});
});
describe('WAGGLE_EVAL_MODE tier bypass (PA v5 §11.3)', () => {
// Defensive cleanup: never leak the flag across tests. If a prior run
// crashed mid-test, this block restores a known-clean baseline.
beforeEach(() => { delete process.env.WAGGLE_EVAL_MODE; });
afterEach(() => { delete process.env.WAGGLE_EVAL_MODE; });
it('without WAGGLE_EVAL_MODE: FREE + litellm still throws TierError (control)', async () => {
// Explicit sanity check that the normal gate is still live — baseline
// for the bypass test below. litellm is the provider FREE still lacks.
expect(process.env.WAGGLE_EVAL_MODE).toBeUndefined();
await expect(
createEmbeddingProvider({
provider: 'litellm',
userTier: 'FREE',
quotaDb: db,
})
).rejects.toThrow(TierError);
});
it('with WAGGLE_EVAL_MODE=1: FREE + litellm no longer throws TierError', async () => {
process.env.WAGGLE_EVAL_MODE = '1';
try {
await createEmbeddingProvider({
provider: 'litellm',
userTier: 'FREE',
quotaDb: db,
});
} catch (err) {
// Probe failure (no real voyage backend in tests) is fine — just not
// a TierError. The point: the tier gate is bypassed.
expect(err).not.toBeInstanceOf(TierError);
}
});
it('with WAGGLE_EVAL_MODE=1 and no userTier set: behaves as if unenforced', async () => {
process.env.WAGGLE_EVAL_MODE = '1';
const provider = await createEmbeddingProvider({
provider: 'auto',
quotaDb: db,
});
// Falls back to mock with no tier skip/probe; provider is constructed.
expect(provider.getActiveProvider()).toBeDefined();
});
it('only activates when env value is exactly "1" (defensive literal match)', async () => {
// Guard against accidental truthy-but-not-"1" values. Harness must use
// "1" exactly per §11.3.
for (const bad of ['true', 'yes', '0', '']) {
process.env.WAGGLE_EVAL_MODE = bad;
await expect(
createEmbeddingProvider({
provider: 'litellm',
userTier: 'FREE',
quotaDb: db,
})
).rejects.toThrow(TierError);
delete process.env.WAGGLE_EVAL_MODE;
}
});
});
describe('getMinimumTierForProvider', () => {
it('inprocess requires TRIAL (first tier that allows it)', () => {
expect(getMinimumTierForProvider('inprocess')).toBe('TRIAL');
});
it('voyage requires TRIAL (TRIAL unlocks all providers)', () => {
// TRIAL is the first tier in TIERS array and has all providers
expect(getMinimumTierForProvider('voyage')).toBe('TRIAL');
});
it('litellm requires TRIAL', () => {
// TRIAL is the first tier with litellm (all unlocked)
expect(getMinimumTierForProvider('litellm')).toBe('TRIAL');
});
it('mock requires TRIAL', () => {
expect(getMinimumTierForProvider('mock')).toBe('TRIAL');
});
});
});

View File

@@ -0,0 +1,267 @@
/**
* FileIndexer unit tests (L-20)
*
* Covers: format gate, indexing, overwrite semantics, move, remove, truncation,
* shared-content dedup safety, and the underlying file_index table.
*/
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { MindDB } from '@waggle/hive-mind-core';
import { FrameStore } from '@waggle/hive-mind-core';
import { FileIndexer, MAX_CONTENT_BYTES } from '../src/file-indexer.js';
describe('FileIndexer', () => {
let db: MindDB;
let indexer: FileIndexer;
beforeEach(() => {
db = new MindDB(':memory:');
indexer = new FileIndexer(db);
});
afterEach(() => {
db.close();
});
describe('shouldIndex', () => {
it('accepts markdown + text', () => {
expect(FileIndexer.shouldIndex('/notes/a.md')).toBe(true);
expect(FileIndexer.shouldIndex('/notes/a.markdown')).toBe(true);
expect(FileIndexer.shouldIndex('/notes/a.txt')).toBe(true);
});
it('ignores case differences in the extension', () => {
expect(FileIndexer.shouldIndex('/a.MD')).toBe(true);
expect(FileIndexer.shouldIndex('/a.TXT')).toBe(true);
});
it('rejects formats deferred to Bucket 2', () => {
expect(FileIndexer.shouldIndex('/a.pdf')).toBe(false);
expect(FileIndexer.shouldIndex('/a.docx')).toBe(false);
expect(FileIndexer.shouldIndex('/a.xlsx')).toBe(false);
});
it('rejects files with no extension', () => {
expect(FileIndexer.shouldIndex('/a')).toBe(false);
expect(FileIndexer.shouldIndex('/README')).toBe(false);
});
});
describe('indexFile', () => {
it('returns unsupported_format for non-indexable extensions', () => {
const result = indexer.indexFile('/a.pdf', Buffer.from('PDF content'));
expect(result.skipped).toBe(true);
if (result.skipped) expect(result.reason).toBe('unsupported_format');
});
it('indexes a markdown file and creates a backing frame', () => {
const content = Buffer.from('# Hello\n\nThis is a note.');
const result = indexer.indexFile('/notes/hello.md', content, 'text/markdown');
expect(result.skipped).toBe(false);
if (!result.skipped) {
expect(result.frameId).toBeGreaterThan(0);
expect(result.truncated).toBe(false);
const frames = new FrameStore(db);
const frame = frames.getById(result.frameId);
expect(frame).toBeTruthy();
expect(frame!.content).toContain('# Hello');
expect(frame!.content).toContain('[FILE: /notes/hello.md');
expect(frame!.content).toContain('text/markdown');
expect(frame!.source).toBe('system');
}
});
it('records the index row with hash + size + mime', () => {
const content = Buffer.from('hello world');
indexer.indexFile('/a.txt', content, 'text/plain');
const row = indexer.getRow('/a.txt');
expect(row).toBeTruthy();
expect(row!.sizeBytes).toBe(content.length);
expect(row!.mimeType).toBe('text/plain');
expect(row!.contentHash).toMatch(/^[a-f0-9]{64}$/);
expect(row!.indexedAt).toBeTruthy();
});
it('returns unchanged when re-indexed with identical content', () => {
const content = Buffer.from('same bytes');
const first = indexer.indexFile('/a.md', content);
expect(first.skipped).toBe(false);
const second = indexer.indexFile('/a.md', content);
expect(second.skipped).toBe(true);
if (second.skipped) expect(second.reason).toBe('unchanged');
});
it('swaps the frame on content change (overwrite path)', () => {
const first = indexer.indexFile('/a.md', Buffer.from('original'));
const second = indexer.indexFile('/a.md', Buffer.from('updated'));
expect(first.skipped).toBe(false);
expect(second.skipped).toBe(false);
if (!first.skipped && !second.skipped) {
expect(second.frameId).not.toBe(first.frameId);
// Old frame is gone.
const frames = new FrameStore(db);
expect(frames.getById(first.frameId)).toBeUndefined();
// New frame exists and row points at it.
expect(frames.getById(second.frameId)).toBeTruthy();
const row = indexer.getRow('/a.md');
expect(row!.frameId).toBe(second.frameId);
}
});
it('truncates content over MAX_CONTENT_BYTES and sets the truncated flag', () => {
const giant = Buffer.alloc(MAX_CONTENT_BYTES + 5000, 0x41); // lots of 'A'
const result = indexer.indexFile('/big.md', giant);
expect(result.skipped).toBe(false);
if (!result.skipped) {
expect(result.truncated).toBe(true);
const frames = new FrameStore(db);
const frame = frames.getById(result.frameId);
expect(frame!.content).toContain('[…truncated');
}
});
it('treats an empty file as remove-if-present', () => {
indexer.indexFile('/a.md', Buffer.from('something'));
expect(indexer.getRow('/a.md')).toBeTruthy();
const emptyResult = indexer.indexFile('/a.md', Buffer.from(''));
expect(emptyResult.skipped).toBe(true);
if (emptyResult.skipped) expect(emptyResult.reason).toBe('empty');
expect(indexer.getRow('/a.md')).toBeNull();
});
it('rolls back atomically when a mutation throws mid-overwrite (L-20 BLOCKER-1)', () => {
// Index a file, then simulate a crash during the overwrite path by
// making frames.delete throw. The whole transaction (new frame +
// old-frame delete + file_index UPDATE) must roll back together. Table
// state after the throw must match state before the throw.
const first = indexer.indexFile('/a.md', Buffer.from('original'));
expect(first.skipped).toBe(false);
if (first.skipped) return;
const originalFrameId = first.frameId;
const raw = db.getDatabase();
const framesBefore = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
const indexBefore = (raw.prepare('SELECT COUNT(*) as c FROM file_index').get() as { c: number }).c;
// Inject crash mid-transaction: the old-frame delete step throws.
const framesProp = (indexer as unknown as { frames: FrameStore }).frames;
const deleteSpy = vi.spyOn(framesProp, 'delete').mockImplementation(() => {
throw new Error('simulated crash mid-overwrite');
});
try {
expect(() => indexer.indexFile('/a.md', Buffer.from('updated'))).toThrow('simulated crash mid-overwrite');
} finally {
deleteSpy.mockRestore();
}
// Rollback invariants:
// 1. Frame-table row count unchanged (new frame not committed).
// 2. Old frame still present (delete was rolled back).
// 3. file_index row count unchanged + row still points at original frame.
const framesAfter = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
const indexAfter = (raw.prepare('SELECT COUNT(*) as c FROM file_index').get() as { c: number }).c;
expect(framesAfter).toBe(framesBefore);
expect(indexAfter).toBe(indexBefore);
const framesStore = new FrameStore(db);
expect(framesStore.getById(originalFrameId)).toBeTruthy();
expect(indexer.getRow('/a.md')!.frameId).toBe(originalFrameId);
});
});
describe('removeFile', () => {
it('returns false when the path is not indexed', () => {
expect(indexer.removeFile('/not-indexed.md')).toBe(false);
});
it('deletes the frame + row when indexed', () => {
const result = indexer.indexFile('/a.md', Buffer.from('goodbye'));
expect(result.skipped).toBe(false);
if (!result.skipped) {
expect(indexer.removeFile('/a.md')).toBe(true);
expect(indexer.getRow('/a.md')).toBeNull();
const frames = new FrameStore(db);
expect(frames.getById(result.frameId)).toBeUndefined();
}
});
it('two paths with identical bodies get distinct frames (path is in the header)', () => {
// The header `[FILE: <path>]` is part of the frame content, so two files
// with the same body but different paths produce different frame hashes.
// This is intentional: a file is a file, not just its bytes.
const body = Buffer.from('shared body');
const first = indexer.indexFile('/dir-a/shared.md', body);
const second = indexer.indexFile('/dir-b/shared.md', body);
expect(first.skipped).toBe(false);
expect(second.skipped).toBe(false);
if (!first.skipped && !second.skipped) {
expect(second.frameId).not.toBe(first.frameId);
// Removing one file removes its own frame + row without touching the other.
indexer.removeFile('/dir-a/shared.md');
const frames = new FrameStore(db);
expect(frames.getById(first.frameId)).toBeUndefined();
expect(frames.getById(second.frameId)).toBeTruthy();
expect(indexer.getRow('/dir-b/shared.md')).toBeTruthy();
}
});
});
describe('moveFile', () => {
it('updates the recorded path + keeps the frame', () => {
const res = indexer.indexFile('/old.md', Buffer.from('body'));
expect(res.skipped).toBe(false);
if (!res.skipped) {
expect(indexer.moveFile('/old.md', '/new.md')).toBe(true);
expect(indexer.getRow('/old.md')).toBeNull();
const row = indexer.getRow('/new.md');
expect(row!.frameId).toBe(res.frameId);
const frames = new FrameStore(db);
expect(frames.getById(res.frameId)).toBeTruthy();
}
});
it('returns false when the source is not indexed', () => {
expect(indexer.moveFile('/missing.md', '/somewhere.md')).toBe(false);
});
it('overwrites a pre-existing destination index row', () => {
indexer.indexFile('/src.md', Buffer.from('src-body'));
indexer.indexFile('/dst.md', Buffer.from('dst-body'));
expect(indexer.moveFile('/src.md', '/dst.md')).toBe(true);
expect(indexer.getRow('/src.md')).toBeNull();
expect(indexer.getRow('/dst.md')).toBeTruthy();
});
});
describe('listAll', () => {
it('returns an empty array with no index rows', () => {
expect(indexer.listAll()).toEqual([]);
});
it('returns rows newest-indexed first', async () => {
indexer.indexFile('/a.md', Buffer.from('a'));
await new Promise(resolve => setTimeout(resolve, 1100));
indexer.indexFile('/b.md', Buffer.from('b'));
const all = indexer.listAll();
expect(all).toHaveLength(2);
expect(all[0].filePath).toBe('/b.md');
expect(all[1].filePath).toBe('/a.md');
});
});
describe('constructor side effects', () => {
it('ensureTable is idempotent (second instance on same DB is OK)', () => {
indexer.indexFile('/a.md', Buffer.from('x'));
const indexer2 = new FileIndexer(db);
expect(indexer2.getRow('/a.md')).toBeTruthy();
});
});
});

View File

@@ -0,0 +1,175 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { LocalFileStore, LinkedDirStore, isSensitiveFilePath } from '../src/file-store.js';
let tmp: string;
beforeEach(() => { tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-fsguard-')); });
afterEach(() => { fs.rmSync(tmp, { recursive: true, force: true }); });
describe('isSensitiveFilePath', () => {
it('flags SSH/GPG/cloud secret directories and key basenames', () => {
for (const p of ['.ssh/id_rsa', '.ssh/config', '.aws/credentials', '.gnupg/secring.gpg',
'id_rsa', 'id_ed25519', '.netrc', '.pgpass', '.git-credentials', 'credentials.json',
'project/nested/.ssh/known_hosts']) {
expect(isSensitiveFilePath(p), p).toBe(true);
}
});
it('flags dotenv files but NOT their checked-in templates', () => {
expect(isSensitiveFilePath('.env')).toBe(true);
expect(isSensitiveFilePath('config/.env.production')).toBe(true);
expect(isSensitiveFilePath('.env.example')).toBe(false);
expect(isSensitiveFilePath('.env.template')).toBe(false);
});
it('is case-insensitive and path-separator agnostic', () => {
expect(isSensitiveFilePath('.SSH/ID_RSA')).toBe(true);
expect(isSensitiveFilePath('.ssh\\id_rsa')).toBe(true);
});
it('does NOT flag ordinary files (incl. public keys)', () => {
for (const p of ['', 'readme.md', 'src/index.ts', 'config.json', 'id_rsa.pub', '.environment', 'data/credentials-form.tsx']) {
expect(isSensitiveFilePath(p), p).toBe(false);
}
});
it('defeats Windows ADS + trailing dot/space normalization tricks', () => {
for (const p of ['id_rsa::$DATA', '.env::$DATA', 'id_rsa.', '.env ', '.NPMRC ']) {
expect(isSensitiveFilePath(p), p).toBe(true);
}
});
it('flags backup copies of secrets but not ordinary backups', () => {
expect(isSensitiveFilePath('credentials.bak')).toBe(true);
expect(isSensitiveFilePath('id_rsa.old')).toBe(true);
expect(isSensitiveFilePath('.npmrc.backup')).toBe(true);
expect(isSensitiveFilePath('readme.bak')).toBe(false);
});
it('flags extended secret classes (pem keys, authorized_keys, cloud, terraform)', () => {
for (const p of ['deploy/secret.pem', 'authorized_keys', 'known_hosts', '.azure/accessTokens.json',
'.terraform/x', 'service-account.json', 'terraform.tfstate', 'infra/terraform.tfstate.backup']) {
expect(isSensitiveFilePath(p), p).toBe(true);
}
});
});
describe('resolveSafe containment (via LocalFileStore)', () => {
const store = () => new LocalFileStore(tmp, 'ws1'); // root = tmp/workspaces/ws1/files
it('reads/writes a normal in-root path', async () => {
const s = store();
await s.writeFile('notes/todo.txt', 'hi');
expect((await s.readFile('notes/todo.txt')).toString()).toBe('hi');
});
it('denies a classic ../ escape', async () => {
await expect(store().readFile('../../../etc/passwd')).rejects.toThrow(/traversal denied/i);
});
it('denies a SIBLING-prefix escape (the startsWith bug)', async () => {
// root is .../ws1/files; this resolves to a sibling .../ws1/files-evil which a
// bare startsWith(root) check WRONGLY admitted. Segment-boundary check rejects it.
await expect(store().writeFile('../files-evil/loot.txt', 'x')).rejects.toThrow(/traversal denied/i);
});
});
describe('LinkedDirStore sensitive-file deny (external folder)', () => {
function seedLinked(): string {
const dir = path.join(tmp, 'project');
fs.mkdirSync(path.join(dir, '.ssh'), { recursive: true });
fs.mkdirSync(path.join(dir, '.aws'), { recursive: true });
fs.writeFileSync(path.join(dir, '.ssh', 'id_rsa'), 'PRIVATE KEY');
fs.writeFileSync(path.join(dir, '.aws', 'credentials'), '[default]\naws_secret=xxx');
fs.writeFileSync(path.join(dir, '.env'), 'SECRET=1');
fs.writeFileSync(path.join(dir, '.env.example'), 'SECRET=');
fs.writeFileSync(path.join(dir, 'README.md'), 'hello');
fs.writeFileSync(path.join(dir, 'credentials.json'), '{"token":"x"}');
fs.writeFileSync(path.join(dir, 'data.json'), '{"ok":true}');
return dir;
}
it('denies reading SSH keys, cloud creds, and .env', async () => {
const s = new LinkedDirStore(seedLinked());
await expect(s.readFile('.ssh/id_rsa')).rejects.toThrow(/sensitive file denied/i);
await expect(s.readFile('.aws/credentials')).rejects.toThrow(/sensitive file denied/i);
await expect(s.readFile('.env')).rejects.toThrow(/sensitive file denied/i);
});
it('allows a normal file and the .env template', async () => {
const s = new LinkedDirStore(seedLinked());
expect((await s.readFile('README.md')).toString()).toBe('hello');
expect((await s.readFile('.env.example')).toString()).toBe('SECRET=');
});
it('denies writing/clobbering and moving a secret file', async () => {
const s = new LinkedDirStore(seedLinked());
await expect(s.writeFile('.ssh/authorized_keys', 'attacker-key')).rejects.toThrow(/sensitive file denied/i);
await expect(s.moveFile('README.md', '.env')).rejects.toThrow(/sensitive file denied/i);
});
it('searchFiles never discloses a non-dot secret (credentials.json)', async () => {
const s = new LinkedDirStore(seedLinked());
const names = (await s.searchFiles('*.json')).map(f => f.name);
expect(names).toContain('data.json');
expect(names).not.toContain('credentials.json');
});
it('listFiles hides non-dot secrets (credentials.json) but keeps normal files', async () => {
const s = new LinkedDirStore(seedLinked());
const names = (await s.listFiles()).map(f => f.name);
expect(names).toContain('README.md');
expect(names).not.toContain('credentials.json');
});
it('searchFiles cannot escape the root via a ../ glob pattern', async () => {
const outside = path.join(tmp, 'outside'); fs.mkdirSync(outside, { recursive: true });
fs.writeFileSync(path.join(outside, 'loot.txt'), 'x');
const s = new LinkedDirStore(seedLinked());
const results = await s.searchFiles('../**/*');
expect(results.every(f => !f.path.includes('..'))).toBe(true);
expect(results.map(f => f.name)).not.toContain('loot.txt');
});
it('still denies a ../ escape out of the linked root', async () => {
const s = new LinkedDirStore(seedLinked());
await expect(s.readFile('../../secret.txt')).rejects.toThrow(/traversal denied/i);
});
});
describe('LinkedDirStore symlink containment (the CRITICAL escape)', () => {
// Symlink creation can fail without privilege (esp. Windows file symlinks) — the
// fix still applies; these tests self-skip when the env cannot create the link.
function trySymlink(target: string, link: string, type: 'junction' | 'file' | 'dir'): boolean {
try { fs.symlinkSync(target, link, type); return true; } catch { return false; }
}
it('denies reading through a junction/symlink that escapes the root', async () => {
const outside = path.join(tmp, 'outside'); fs.mkdirSync(outside, { recursive: true });
fs.writeFileSync(path.join(outside, 'secret.txt'), 'TOPSECRET');
const root = path.join(tmp, 'proj'); fs.mkdirSync(root, { recursive: true });
if (!trySymlink(outside, path.join(root, 'escape'), 'junction')) return;
const s = new LinkedDirStore(root);
await expect(s.readFile('escape/secret.txt')).rejects.toThrow(/traversal denied/i);
});
it('still allows a symlink that stays inside the root (monorepo-style link)', async () => {
const root = path.join(tmp, 'proj2'); fs.mkdirSync(path.join(root, 'real'), { recursive: true });
fs.writeFileSync(path.join(root, 'real', 'data.txt'), 'OK');
if (!trySymlink(path.join(root, 'real'), path.join(root, 'alias'), 'junction')) return;
const s = new LinkedDirStore(root);
expect((await s.readFile('alias/data.txt')).toString()).toBe('OK');
});
it('denies a symlink that launders an in-root secret past a benign name', async () => {
const root = path.join(tmp, 'proj3'); fs.mkdirSync(root, { recursive: true });
fs.writeFileSync(path.join(root, '.env'), 'SECRET=1');
if (!trySymlink(path.join(root, '.env'), path.join(root, 'notes'), 'file')) return;
const s = new LinkedDirStore(root);
await expect(s.readFile('notes')).rejects.toThrow(/sensitive file denied/i);
});
});
describe('LocalFileStore is sandboxed — no sensitive-file deny', () => {
it('allows a .env in the virtual workspace (the agent\'s own scratch)', async () => {
const s = new LocalFileStore(tmp, 'ws2');
await s.writeFile('.env', 'LOCAL=1');
expect((await s.readFile('.env')).toString()).toBe('LOCAL=1');
});
});

View File

@@ -0,0 +1,266 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { S3FileStore, type S3Config } from '../src/file-store.js';
// ── Mock @aws-sdk/client-s3 ────────────────────────────────────────
const mockSend = vi.fn();
type CommandInput = Record<string, unknown>;
class MockS3Client {
send = mockSend;
constructor(_config: unknown) {}
}
vi.mock('@aws-sdk/client-s3', () => {
return {
S3Client: MockS3Client,
GetObjectCommand: class { [k: string]: unknown; _type = 'GetObject'; constructor(input: CommandInput) { Object.assign(this, input); } },
PutObjectCommand: class { [k: string]: unknown; _type = 'PutObject'; constructor(input: CommandInput) { Object.assign(this, input); } },
DeleteObjectCommand: class { [k: string]: unknown; _type = 'DeleteObject'; constructor(input: CommandInput) { Object.assign(this, input); } },
ListObjectsV2Command: class { [k: string]: unknown; _type = 'ListObjects'; constructor(input: CommandInput) { Object.assign(this, input); } },
CopyObjectCommand: class { [k: string]: unknown; _type = 'CopyObject'; constructor(input: CommandInput) { Object.assign(this, input); } },
};
});
// ── Test setup ──────────────────────────────────────────────────────
const testConfig: S3Config = {
endpoint: 'minio:9000',
bucket: 'waggle-files',
accessKey: 'waggle',
secretKey: 'waggle_s3_prod',
prefix: 'workspaces/ws-123/',
};
describe('S3FileStore', () => {
let store: S3FileStore;
beforeEach(() => {
vi.clearAllMocks();
store = new S3FileStore(testConfig);
});
// ── Meta ────────────────────────────────────────────────────────
it('getStorageType returns virtual', () => {
expect(store.getStorageType()).toBe('virtual');
});
it('getRootPath returns s3:// URL', () => {
expect(store.getRootPath()).toBe('s3://waggle-files/workspaces/ws-123/');
});
// ── writeFile ───────────────────────────────────────────────────
it('writeFile sends PutObjectCommand with correct bucket and key', async () => {
mockSend.mockResolvedValueOnce({});
await store.writeFile('docs/notes.md', 'hello world');
expect(mockSend).toHaveBeenCalledTimes(1);
const cmd = mockSend.mock.calls[0][0];
expect(cmd._type).toBe('PutObject');
expect(cmd.Bucket).toBe('waggle-files');
expect(cmd.Key).toBe('workspaces/ws-123/docs/notes.md');
expect(cmd.Body).toEqual(Buffer.from('hello world'));
});
it('writeFile accepts Buffer content', async () => {
mockSend.mockResolvedValueOnce({});
const buf = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
await store.writeFile('image.png', buf);
const cmd = mockSend.mock.calls[0][0];
expect(cmd.Body).toBe(buf);
});
// ── readFile ────────────────────────────────────────────────────
it('readFile sends GetObjectCommand and returns Buffer', async () => {
const chunks = [Buffer.from('chunk1'), Buffer.from('chunk2')];
const asyncIterable = {
async *[Symbol.asyncIterator]() {
for (const c of chunks) yield c;
},
};
mockSend.mockResolvedValueOnce({ Body: asyncIterable });
const result = await store.readFile('docs/notes.md');
expect(mockSend).toHaveBeenCalledTimes(1);
const cmd = mockSend.mock.calls[0][0];
expect(cmd._type).toBe('GetObject');
expect(cmd.Bucket).toBe('waggle-files');
expect(cmd.Key).toBe('workspaces/ws-123/docs/notes.md');
expect(Buffer.isBuffer(result)).toBe(true);
expect(result.toString()).toBe('chunk1chunk2');
});
// ── deleteFile ──────────────────────────────────────────────────
it('deleteFile sends DeleteObjectCommand', async () => {
mockSend.mockResolvedValueOnce({});
await store.deleteFile('old-file.txt');
expect(mockSend).toHaveBeenCalledTimes(1);
const cmd = mockSend.mock.calls[0][0];
expect(cmd._type).toBe('DeleteObject');
expect(cmd.Bucket).toBe('waggle-files');
expect(cmd.Key).toBe('workspaces/ws-123/old-file.txt');
});
// ── listFiles ───────────────────────────────────────────────────
it('listFiles sends ListObjectsV2Command and returns FileEntry[]', async () => {
mockSend.mockResolvedValueOnce({
Contents: [
{ Key: 'workspaces/ws-123/readme.md', Size: 1024, LastModified: new Date('2025-06-01T00:00:00Z') },
{ Key: 'workspaces/ws-123/src/index.ts', Size: 512, LastModified: new Date('2025-06-02T00:00:00Z') },
],
CommonPrefixes: [
{ Prefix: 'workspaces/ws-123/docs/' },
],
});
const entries = await store.listFiles();
expect(mockSend).toHaveBeenCalledTimes(1);
const cmd = mockSend.mock.calls[0][0];
expect(cmd._type).toBe('ListObjects');
expect(cmd.Bucket).toBe('waggle-files');
expect(cmd.Prefix).toBe('workspaces/ws-123/');
expect(cmd.Delimiter).toBe('/');
expect(entries).toHaveLength(3);
// File entries
expect(entries[0]).toEqual({
name: 'readme.md',
path: 'readme.md',
size: 1024,
modified: '2025-06-01T00:00:00.000Z',
isDirectory: false,
});
expect(entries[1]).toEqual({
name: 'index.ts',
path: 'src/index.ts',
size: 512,
modified: '2025-06-02T00:00:00.000Z',
isDirectory: false,
});
// Directory entry
expect(entries[2]).toEqual({
name: 'docs',
path: 'docs',
size: 0,
modified: '',
isDirectory: true,
});
});
it('listFiles with directory argument appends to prefix', async () => {
mockSend.mockResolvedValueOnce({ Contents: [], CommonPrefixes: [] });
await store.listFiles('src');
const cmd = mockSend.mock.calls[0][0];
expect(cmd.Prefix).toBe('workspaces/ws-123/src/');
});
it('listFiles handles empty response', async () => {
mockSend.mockResolvedValueOnce({});
const entries = await store.listFiles();
expect(entries).toEqual([]);
});
// ── moveFile ────────────────────────────────────────────────────
it('moveFile copies then deletes', async () => {
mockSend.mockResolvedValueOnce({}); // CopyObject
mockSend.mockResolvedValueOnce({}); // DeleteObject
await store.moveFile('old.txt', 'new.txt');
expect(mockSend).toHaveBeenCalledTimes(2);
const copyCmd = mockSend.mock.calls[0][0];
expect(copyCmd._type).toBe('CopyObject');
expect(copyCmd.CopySource).toBe('waggle-files/workspaces/ws-123/old.txt');
expect(copyCmd.Key).toBe('workspaces/ws-123/new.txt');
const deleteCmd = mockSend.mock.calls[1][0];
expect(deleteCmd._type).toBe('DeleteObject');
expect(deleteCmd.Key).toBe('workspaces/ws-123/old.txt');
});
// ── searchFiles ─────────────────────────────────────────────────
it('searchFiles filters listFiles results by pattern', async () => {
mockSend.mockResolvedValueOnce({
Contents: [
{ Key: 'workspaces/ws-123/readme.md', Size: 100, LastModified: new Date() },
{ Key: 'workspaces/ws-123/notes.txt', Size: 200, LastModified: new Date() },
{ Key: 'workspaces/ws-123/data.md', Size: 50, LastModified: new Date() },
],
CommonPrefixes: [],
});
const results = await store.searchFiles('*.md');
expect(results).toHaveLength(2);
expect(results[0].name).toBe('readme.md');
expect(results[1].name).toBe('data.md');
});
it('searchFiles is ReDoS-safe with a hostile regex pattern', async () => {
mockSend.mockResolvedValueOnce({
Contents: [{ Key: 'workspaces/ws-123/' + 'a'.repeat(40) + 'X', Size: 1, LastModified: new Date() }],
CommonPrefixes: [],
});
const start = Date.now();
// Under the old `new RegExp(pattern…)` this built /(a+)+$/i and catastrophically
// backtracked on the key. Now the metachars are escaped → a literal match → instant.
const results = await store.searchFiles('(a+)+$');
expect(Date.now() - start).toBeLessThan(1000);
expect(results).toHaveLength(0); // matched literally; no key contains "(a+)+$"
});
// ── key traversal protection ────────────────────────────────────
it('rejects ../ traversal in every op BEFORE any S3 call', async () => {
await expect(store.readFile('../ws-456/secret')).rejects.toThrow(/traversal denied/i);
await expect(store.writeFile('../ws-456/x', 'y')).rejects.toThrow(/traversal denied/i);
await expect(store.deleteFile('../../etc/passwd')).rejects.toThrow(/traversal denied/i);
await expect(store.moveFile('ok', '../ws-456/b')).rejects.toThrow(/traversal denied/i);
await expect(store.listFiles('../ws-456')).rejects.toThrow(/traversal denied/i);
expect(mockSend).not.toHaveBeenCalled();
});
it('allows a normal nested key', async () => {
mockSend.mockResolvedValueOnce({});
await store.writeFile('docs/sub/file.md', 'x');
expect(mockSend.mock.calls[0][0].Key).toBe('workspaces/ws-123/docs/sub/file.md');
});
// ── getStorageInfo ──────────────────────────────────────────────
it('getStorageInfo sums sizes from S3 listing', async () => {
mockSend.mockResolvedValueOnce({
Contents: [
{ Key: 'workspaces/ws-123/a.txt', Size: 100 },
{ Key: 'workspaces/ws-123/b.txt', Size: 250 },
{ Key: 'workspaces/ws-123/c.txt', Size: 50 },
],
});
const info = await store.getStorageInfo();
expect(info).toEqual({
usedBytes: 400,
fileCount: 3,
storageType: 'virtual',
});
});
});

View File

@@ -0,0 +1,44 @@
/**
* P7/D15 A3 — drift-lock for the install_audit CHECK constraints.
*
* The table is declared in TWO places: install-audit.ts (core, generated from
* the canonical @waggle/shared arrays) and hive-mind-core/src/mind/schema.ts
* (the OSS substrate, a standalone literal). They MUST produce identical CHECK
* lists or auditStore.record() crashes on one path. This test pins both to the
* single canonical source, so a drift in either fails CI instead of production.
*/
import { describe, it, expect } from 'vitest';
import {
sqlInList, RISK_LEVELS, APPROVAL_CLASSES, AUDIT_ACTIONS,
AUDIT_CAPABILITY_TYPES, AUDIT_INITIATORS, TRUST_SOURCES,
} from '@waggle/shared';
import { SCHEMA_SQL } from '@waggle/hive-mind-core';
import { INSTALL_AUDIT_TABLE_SQL } from '../src/install-audit.js';
const COLUMNS = [
{ col: 'capability_type', values: AUDIT_CAPABILITY_TYPES },
{ col: 'risk_level', values: RISK_LEVELS },
{ col: 'trust_source', values: TRUST_SOURCES },
{ col: 'approval_class', values: APPROVAL_CLASSES },
{ col: 'action', values: AUDIT_ACTIONS },
{ col: 'initiator', values: AUDIT_INITIATORS },
] as const;
describe('install_audit CHECK parity (A3)', () => {
for (const { col, values } of COLUMNS) {
const expected = `${col} IN (${sqlInList(values)})`;
it(`core install-audit DDL pins ${col} to the canonical list`, () => {
expect(INSTALL_AUDIT_TABLE_SQL).toContain(expected);
});
it(`OSS substrate schema.ts pins ${col} to the canonical list`, () => {
expect(SCHEMA_SQL).toContain(expected);
});
}
it('both DDLs accept the P5/D4 uninstalled action', () => {
expect(INSTALL_AUDIT_TABLE_SQL).toContain("'uninstalled'");
expect(SCHEMA_SQL).toContain("'uninstalled'");
});
});

View File

@@ -0,0 +1,479 @@
/**
* InstallAuditStore tests.
*
* Relocated here from packages/hive-mind-core/tests/mind/ — commit 05c9ec3
* ("relocate substrate tests") moved this test to hive-mind-core, but the
* source (`install-audit.ts`) stayed in @waggle/core and imports MindDB FROM
* @waggle/hive-mind-core. hive-mind-core cannot depend back on @waggle/core
* (dependency inversion + breaks the OSS parity model), so the relocated test
* imported a non-existent `../../src/install-audit.js` and the entire suite
* silently failed at collection — masking the FIX-3 CHECK-drift regression.
* Its correct home is alongside the source, in @waggle/core.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { MindDB } from '@waggle/hive-mind-core';
import { InstallAuditStore, type RecordAuditInput } from '../src/install-audit.js';
describe('InstallAuditStore', () => {
let tmpDir: string;
let db: MindDB;
let store: InstallAuditStore;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-audit-'));
db = new MindDB(path.join(tmpDir, 'test.mind'));
store = new InstallAuditStore(db);
});
afterEach(() => {
db.close();
fs.rmSync(tmpDir, { recursive: true, force: true });
});
function makeInput(overrides?: Partial<RecordAuditInput>): RecordAuditInput {
return {
capabilityName: 'risk-assessment',
capabilityType: 'skill',
source: 'starter-pack',
riskLevel: 'low',
trustSource: 'starter_pack',
approvalClass: 'standard',
action: 'installed',
initiator: 'agent',
detail: 'Installed successfully',
...overrides,
};
}
it("records an 'uninstalled' action (P5/D4 — capability removal trail)", () => {
const entry = store.record(makeInput({ action: 'uninstalled', initiator: 'agent', detail: 'deleted by agent' }));
expect(entry.action).toBe('uninstalled');
expect(store.getByAction('uninstalled')).toHaveLength(1);
});
it("migrates a legacy-CHECK table to accept 'uninstalled' (P5/D4)", () => {
const tmp2 = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-audit-legacy-'));
const legacyDb = new MindDB(path.join(tmp2, 'legacy.mind'));
const raw = legacyDb.getDatabase();
// Simulate a pre-P5 install: drop the migrated table and recreate it with
// the OLD narrower action CHECK (no 'uninstalled'), seeding one row.
raw.exec('DROP TABLE IF EXISTS install_audit');
raw.exec(`
CREATE TABLE install_audit (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp TEXT NOT NULL DEFAULT (datetime('now')),
capability_name TEXT NOT NULL,
capability_type TEXT NOT NULL CHECK (capability_type IN ('native','skill','plugin','mcp','connector','marketplace')),
source TEXT NOT NULL,
version TEXT,
risk_level TEXT NOT NULL CHECK (risk_level IN ('low','medium','high','critical')),
trust_source TEXT NOT NULL,
approval_class TEXT NOT NULL CHECK (approval_class IN ('standard','elevated','critical','blocked')),
action TEXT NOT NULL CHECK (action IN ('proposed','approved','installed','rejected','failed','blocked')),
initiator TEXT NOT NULL CHECK (initiator IN ('agent','user','system')),
detail TEXT NOT NULL DEFAULT ''
);
`);
raw.prepare(`INSERT INTO install_audit
(capability_name, capability_type, source, risk_level, trust_source, approval_class, action, initiator, detail)
VALUES ('legacy-skill','skill','starter-pack','low','starter_pack','standard','installed','user','pre-migration row')`).run();
// Constructing the store triggers ensureTable() → rebuild migration.
const migrated = new InstallAuditStore(legacyDb);
// Pre-existing row survives the rebuild.
expect(migrated.getByCapability('legacy-skill')).toHaveLength(1);
// The widened CHECK now accepts 'uninstalled' (would throw on a stale table).
const entry = migrated.record(makeInput({ capabilityName: 'legacy-skill', action: 'uninstalled', detail: 'removed' }));
expect(entry.action).toBe('uninstalled');
// Review #1: the rebuild must NOT drop the declared indexes (SQLite RENAME
// carries index names to the legacy table; without an explicit DROP INDEX
// they get destroyed with it, leaving the audit table index-less).
const idx = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='install_audit' AND name LIKE 'idx_audit_%'",
).all() as Array<{ name: string }>;
expect(idx.map(i => i.name).sort()).toEqual(['idx_audit_capability', 'idx_audit_timestamp']);
// #15: the rebuild also added the trust_source CHECK (the legacy table had
// trust_source unconstrained). The migrated DDL now carries it, and the
// pre-migration row (trust_source 'starter_pack') survived because every
// historical value is in the canonical 7-set.
const ddl = raw.prepare(
"SELECT sql FROM sqlite_master WHERE type='table' AND name='install_audit'",
).get() as { sql: string };
expect(ddl.sql).toContain('CHECK (trust_source IN');
// A bogus trust_source is now rejected at the DB (was previously accepted).
expect(() => raw.prepare(
`INSERT INTO install_audit
(capability_name, capability_type, source, risk_level, trust_source, approval_class, action, initiator, detail)
VALUES ('x','skill','s','low','BOGUS_SOURCE','standard','installed','user','')`,
).run()).toThrow();
legacyDb.close();
fs.rmSync(tmp2, { recursive: true, force: true });
});
it('records and retrieves an audit entry', () => {
const entry = store.record(makeInput());
expect(entry.id).toBeGreaterThan(0);
expect(entry.capability_name).toBe('risk-assessment');
expect(entry.action).toBe('installed');
expect(entry.risk_level).toBe('low');
expect(entry.trust_source).toBe('starter_pack');
expect(entry.timestamp).toBeTruthy();
});
it('records multiple events for same capability', () => {
store.record(makeInput({ action: 'proposed' }));
store.record(makeInput({ action: 'approved' }));
store.record(makeInput({ action: 'installed' }));
const history = store.getByCapability('risk-assessment');
expect(history).toHaveLength(3);
expect(history[0].action).toBe('installed');
expect(history[2].action).toBe('proposed');
});
it('queries by action type', () => {
store.record(makeInput({ capabilityName: 'draft-memo', action: 'installed' }));
store.record(makeInput({ capabilityName: 'code-review', action: 'proposed' }));
store.record(makeInput({ capabilityName: 'brainstorm', action: 'installed' }));
const installed = store.getByAction('installed');
expect(installed).toHaveLength(2);
expect(installed.map(e => e.capability_name).sort()).toEqual(['brainstorm', 'draft-memo']);
});
it('retrieves recent entries in descending order', () => {
store.record(makeInput({ capabilityName: 'first', action: 'proposed' }));
store.record(makeInput({ capabilityName: 'second', action: 'installed' }));
store.record(makeInput({ capabilityName: 'third', action: 'failed' }));
const recent = store.getRecent(2);
expect(recent).toHaveLength(2);
expect(recent[0].capability_name).toBe('third');
expect(recent[1].capability_name).toBe('second');
});
it('preserves all fields round-trip', () => {
const entry = store.record(makeInput({
capabilityName: 'test-skill',
capabilityType: 'plugin',
source: 'third-party',
version: '1.2.3',
riskLevel: 'high',
trustSource: 'third_party_unverified',
approvalClass: 'critical',
action: 'failed',
initiator: 'user',
detail: 'Permission denied by user',
}));
expect(entry.capability_name).toBe('test-skill');
expect(entry.capability_type).toBe('plugin');
expect(entry.source).toBe('third-party');
expect(entry.version).toBe('1.2.3');
expect(entry.risk_level).toBe('high');
expect(entry.trust_source).toBe('third_party_unverified');
expect(entry.approval_class).toBe('critical');
expect(entry.action).toBe('failed');
expect(entry.initiator).toBe('user');
expect(entry.detail).toBe('Permission denied by user');
});
it('handles null version', () => {
const entry = store.record(makeInput({ version: null }));
expect(entry.version).toBeNull();
});
it('handles empty detail', () => {
const entry = store.record(makeInput({ detail: undefined }));
expect(entry.detail).toBe('');
});
it('getAll returns entries in insertion order', () => {
store.record(makeInput({ capabilityName: 'a' }));
store.record(makeInput({ capabilityName: 'b' }));
store.record(makeInput({ capabilityName: 'c' }));
const all = store.getAll();
expect(all).toHaveLength(3);
expect(all[0].capability_name).toBe('a');
expect(all[2].capability_name).toBe('c');
});
it('clear removes all entries', () => {
store.record(makeInput({ capabilityName: 'a' }));
store.record(makeInput({ capabilityName: 'b' }));
expect(store.getAll()).toHaveLength(2);
store.clear();
expect(store.getAll()).toHaveLength(0);
});
it('creates table lazily on pre-existing databases', () => {
const store2 = new InstallAuditStore(db);
const entry = store2.record(makeInput({ capabilityName: 'from-second-store' }));
expect(entry.capability_name).toBe('from-second-store');
});
it('records failed validation as audit event', () => {
const entry = store.record(makeInput({
action: 'failed',
riskLevel: 'low',
trustSource: 'unknown',
detail: 'Skill "nonexistent" not found in the starter pack.',
}));
expect(entry.action).toBe('failed');
expect(entry.detail).toContain('not found');
});
// P0-005/FIX-3: install_audit CHECK constraints drifted behind their TS
// type unions. Once marketplace FTS search returned candidates,
// acquire_capability recommended a `type:'marketplace'` capability and
// auditStore.record() crashed with "CHECK constraint failed: capability_type
// IN ('native','skill','plugin','mcp')", throwing the whole tool and
// dead-ending the agent. These lock the CHECK <-> type-union alignment.
it('records a marketplace-type capability (capability_type CHECK widened)', () => {
const entry = store.record(makeInput({ capabilityType: 'marketplace', source: 'marketplace' }));
expect(entry.capability_type).toBe('marketplace');
});
it('records a connector-type capability', () => {
const entry = store.record(makeInput({ capabilityType: 'connector', source: 'connector' }));
expect(entry.capability_type).toBe('connector');
});
it("records the 'blocked' action and 'blocked' approval_class (sibling CHECK drift)", () => {
const entry = store.record(makeInput({ action: 'blocked', approvalClass: 'blocked' }));
expect(entry.action).toBe('blocked');
expect(entry.approval_class).toBe('blocked');
});
// M2 (UX-Refactor Phase 4 / C15): risk_level CHECK lacked 'critical' while
// the TS union had it — marketplace.ts's CRITICAL-block audit write was
// silently rejected (throw swallowed by `catch {}`). This locks the widened
// CHECK on fresh databases.
it("records riskLevel 'critical' (M2: risk_level CHECK widened)", () => {
const entry = store.record(makeInput({
riskLevel: 'critical',
approvalClass: 'blocked',
action: 'blocked',
trustSource: 'security-gate',
detail: 'SecurityGate blocked: CRITICAL findings',
}));
expect(entry.risk_level).toBe('critical');
expect(entry.action).toBe('blocked');
});
// C18: type-filtered read backing GET /api/extend/audit?type=
it('getRecentByType filters by capability_type, most recent first', () => {
store.record(makeInput({ capabilityName: 'a-skill', capabilityType: 'skill' }));
store.record(makeInput({ capabilityName: 'a-server', capabilityType: 'mcp' }));
store.record(makeInput({ capabilityName: 'b-server', capabilityType: 'mcp' }));
store.record(makeInput({ capabilityName: 'a-conn', capabilityType: 'connector' }));
const mcps = store.getRecentByType('mcp');
expect(mcps).toHaveLength(2);
expect(mcps[0].capability_name).toBe('b-server');
expect(mcps[1].capability_name).toBe('a-server');
expect(store.getRecentByType('mcp', 1)).toHaveLength(1);
expect(store.getRecentByType('native')).toHaveLength(0);
});
});
describe('InstallAuditStore — legacy CHECK migration', () => {
let tmpDir: string;
let dbPath: string;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-audit-mig-'));
dbPath = path.join(tmpDir, 'legacy.mind');
});
afterEach(() => {
fs.rmSync(tmpDir, { recursive: true, force: true });
});
const OLD_DDL = `CREATE TABLE install_audit (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp TEXT NOT NULL DEFAULT (datetime('now')),
capability_name TEXT NOT NULL,
capability_type TEXT NOT NULL CHECK (capability_type IN ('native', 'skill', 'plugin', 'mcp')),
source TEXT NOT NULL,
version TEXT,
risk_level TEXT NOT NULL CHECK (risk_level IN ('low', 'medium', 'high')),
trust_source TEXT NOT NULL,
approval_class TEXT NOT NULL CHECK (approval_class IN ('standard', 'elevated', 'critical')),
action TEXT NOT NULL CHECK (action IN ('proposed', 'approved', 'installed', 'rejected', 'failed')),
initiator TEXT NOT NULL CHECK (initiator IN ('agent', 'user', 'system')),
detail TEXT NOT NULL DEFAULT ''
)`;
it('rebuilds a legacy install_audit table (narrow CHECK) and preserves rows', () => {
// 1. Create the mind DB, then forcibly downgrade install_audit to the
// historical 4-value CHECK with one legacy row — simulating a real
// user .mind created before connector/marketplace existed.
{
const seed = new MindDB(dbPath);
const raw = seed.getDatabase();
raw.prepare('DROP TABLE IF EXISTS install_audit').run();
raw.prepare(OLD_DDL).run();
raw.prepare(`INSERT INTO install_audit
(capability_name, capability_type, source, risk_level, trust_source, approval_class, action, initiator, detail)
VALUES ('legacy-skill','skill','starter-pack','low','starter_pack','standard','installed','agent','pre-migration row')`).run();
seed.close();
}
// 2. Reopen — runMigrations() must rebuild install_audit with the widened CHECK.
const db = new MindDB(dbPath);
const store = new InstallAuditStore(db);
const legacy = store.getByCapability('legacy-skill');
expect(legacy).toHaveLength(1);
expect(legacy[0].detail).toBe('pre-migration row');
expect(() => store.record({
capabilityName: 'filesystem', capabilityType: 'marketplace', source: 'marketplace',
riskLevel: 'medium', trustSource: 'unknown', approvalClass: 'standard',
action: 'proposed', initiator: 'agent', detail: 'Proposed for need: read external files',
})).not.toThrow();
const fs2 = store.getByCapability('filesystem');
expect(fs2[0].capability_type).toBe('marketplace');
db.close();
});
// M2 (UX-Refactor Phase 4 / C15): the FIX-3-era DDL had every list widened
// EXCEPT risk_level — the exact shape real .minds created between FIX-3
// (2026-05-17) and Phase 4 are in. Only the new "'low', 'medium', 'high',
// 'critical'" sentinel triggers this rebuild ('critical' alone appears in
// approval_class, so it is NOT the key).
const FIX3_ERA_DDL = `CREATE TABLE install_audit (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp TEXT NOT NULL DEFAULT (datetime('now')),
capability_name TEXT NOT NULL,
capability_type TEXT NOT NULL CHECK (capability_type IN ('native', 'skill', 'plugin', 'mcp', 'connector', 'marketplace')),
source TEXT NOT NULL,
version TEXT,
risk_level TEXT NOT NULL CHECK (risk_level IN ('low', 'medium', 'high')),
trust_source TEXT NOT NULL,
approval_class TEXT NOT NULL CHECK (approval_class IN ('standard', 'elevated', 'critical', 'blocked')),
action TEXT NOT NULL CHECK (action IN ('proposed', 'approved', 'installed', 'rejected', 'failed', 'blocked')),
initiator TEXT NOT NULL CHECK (initiator IN ('agent', 'user', 'system')),
detail TEXT NOT NULL DEFAULT ''
)`;
it("M2: rebuilds a FIX-3-era table (risk_level missing 'critical') and preserves rows", () => {
{
const seed = new MindDB(dbPath);
const raw = seed.getDatabase();
raw.prepare('DROP TABLE IF EXISTS install_audit').run();
raw.prepare(FIX3_ERA_DDL).run();
raw.prepare(`INSERT INTO install_audit
(capability_name, capability_type, source, risk_level, trust_source, approval_class, action, initiator, detail)
VALUES ('legacy-mcp','mcp','marketplace','high','security-gate','blocked','blocked','system','pre-M2 row')`).run();
// Sanity: the legacy CHECK really rejects 'critical' (the live bug)
expect(() => raw.prepare(`INSERT INTO install_audit
(capability_name, capability_type, source, risk_level, trust_source, approval_class, action, initiator, detail)
VALUES ('x','mcp','marketplace','critical','security-gate','blocked','blocked','system','')`).run()
).toThrow(/CHECK/);
seed.close();
}
// Reopen — runMigrations() must rebuild keyed on the M2 sentinel.
const db = new MindDB(dbPath);
const store = new InstallAuditStore(db);
const legacy = store.getByCapability('legacy-mcp');
expect(legacy).toHaveLength(1);
expect(legacy[0].detail).toBe('pre-M2 row');
expect(legacy[0].risk_level).toBe('high');
const critical = store.record({
capabilityName: 'evil-pkg', capabilityType: 'marketplace', source: 'marketplace',
riskLevel: 'critical', trustSource: 'security-gate', approvalClass: 'blocked',
action: 'blocked', initiator: 'system', detail: 'SecurityGate blocked: CRITICAL',
});
expect(critical.risk_level).toBe('critical');
db.close();
// Idempotence: a second reopen must NOT rebuild again (rows + ids stable).
const db2 = new MindDB(dbPath);
const store2 = new InstallAuditStore(db2);
const all = store2.getAll();
expect(all).toHaveLength(2);
expect(all.map((e) => e.capability_name)).toEqual(['legacy-mcp', 'evil-pkg']);
// The rebuilt DDL carries the sentinel, so a third store write still works.
expect(() => store2.record({
capabilityName: 'again', capabilityType: 'mcp', source: 'mcp',
riskLevel: 'critical', trustSource: 'security-gate', approvalClass: 'blocked',
action: 'blocked', initiator: 'system',
})).not.toThrow();
db2.close();
});
// The rebuild now runs in ONE transaction, so a crash mid-rebuild rolls
// back — but DBs damaged by a PRE-transactional crashed rebuild exist in the
// wild with all rows stranded in install_audit__mig_old. These lock the
// recovery the FIX-3 comment always promised but never performed.
const LEGACY_ROW_INSERT = (table: string) => `INSERT INTO ${table}
(capability_name, capability_type, source, risk_level, trust_source, approval_class, action, initiator, detail)
VALUES ('stranded','mcp','marketplace','high','security-gate','blocked','blocked','system','pre-crash row')`;
it('recovers rows stranded by a crash between RENAME and recreate (install_audit missing)', () => {
{
const seed = new MindDB(dbPath);
const raw = seed.getDatabase();
raw.prepare('DROP TABLE IF EXISTS install_audit').run();
raw.prepare(FIX3_ERA_DDL).run();
raw.prepare(LEGACY_ROW_INSERT('install_audit')).run();
// Simulate the pre-transactional crash: renamed aside, then process died
// before SCHEMA_SQL recreated install_audit.
raw.prepare('ALTER TABLE install_audit RENAME TO install_audit__mig_old').run();
seed.close();
}
const db = new MindDB(dbPath);
const store = new InstallAuditStore(db);
// Rows restored AND the rebuild completed (the restored table had the
// FIX-3-era DDL, so the M2 sentinel re-triggered the rebuild)
const rows = store.getByCapability('stranded');
expect(rows).toHaveLength(1);
expect(rows[0].detail).toBe('pre-crash row');
expect(() => store.record({
capabilityName: 'post-recovery', capabilityType: 'mcp', source: 'mcp',
riskLevel: 'critical', trustSource: 'security-gate', approvalClass: 'blocked',
action: 'blocked', initiator: 'system',
})).not.toThrow();
// No stale __mig_old left to be destroyed by a future rebuild
const leftover = db.getDatabase().prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='install_audit__mig_old'",
).get();
expect(leftover).toBeUndefined();
db.close();
});
it('recovers rows stranded by a crash between recreate and copy-back (fresh empty install_audit)', () => {
{
const seed = new MindDB(dbPath); // creates the CURRENT empty install_audit
const raw = seed.getDatabase();
raw.prepare(FIX3_ERA_DDL.replace('CREATE TABLE install_audit', 'CREATE TABLE install_audit__mig_old')).run();
raw.prepare(LEGACY_ROW_INSERT('install_audit__mig_old')).run();
seed.close();
}
const db = new MindDB(dbPath);
const store = new InstallAuditStore(db);
const rows = store.getByCapability('stranded');
expect(rows).toHaveLength(1);
expect(rows[0].detail).toBe('pre-crash row');
const leftover = db.getDatabase().prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='install_audit__mig_old'",
).get();
expect(leftover).toBeUndefined();
db.close();
});
});

View File

@@ -0,0 +1,162 @@
import { describe, it, expect, vi } from 'vitest';
import { createLiteLLMEmbedder } from '@waggle/hive-mind-core';
describe('createLiteLLMEmbedder', () => {
const baseConfig = {
litellmUrl: 'http://localhost:4000/v1',
litellmApiKey: 'sk-test',
model: 'text-embedding',
dimensions: 8,
};
function mockFetchOk(data: unknown) {
return vi.fn().mockResolvedValue({
ok: true,
json: async () => data,
} as unknown as Response);
}
it('exposes the configured dimensions', () => {
const embedder = createLiteLLMEmbedder({ ...baseConfig, dimensions: 256, fetch: mockFetchOk({}) });
expect(embedder.dimensions).toBe(256);
});
it('defaults dimensions to 1024', () => {
const embedder = createLiteLLMEmbedder({
litellmUrl: 'http://localhost:4000/v1',
fetch: mockFetchOk({}),
});
expect(embedder.dimensions).toBe(1024);
});
it('embed() calls the correct endpoint with Bearer auth', async () => {
const fakeFetch = mockFetchOk({
data: [{ embedding: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] }],
});
const embedder = createLiteLLMEmbedder({ ...baseConfig, fetch: fakeFetch });
const result = await embedder.embed('hello world');
expect(fakeFetch).toHaveBeenCalledOnce();
const [url, options] = fakeFetch.mock.calls[0];
expect(url).toBe('http://localhost:4000/v1/embeddings');
expect(options.method).toBe('POST');
expect(options.headers['Authorization']).toBe('Bearer sk-test');
expect(options.headers['Content-Type']).toBe('application/json');
const body = JSON.parse(options.body);
expect(body.model).toBe('text-embedding');
expect(body.input).toBe('hello world');
expect(result).toBeInstanceOf(Float32Array);
expect(result.length).toBe(8);
expect(result[0]).toBeCloseTo(0.1);
});
it('embed() strips trailing /v1 to avoid double path', async () => {
const fakeFetch = mockFetchOk({
data: [{ embedding: [1, 2, 3, 4, 5, 6, 7, 8] }],
});
const embedder = createLiteLLMEmbedder({
...baseConfig,
litellmUrl: 'http://localhost:4000/v1',
fetch: fakeFetch,
});
await embedder.embed('test');
const [url] = fakeFetch.mock.calls[0];
expect(url).toBe('http://localhost:4000/v1/embeddings');
});
it('embedBatch() returns multiple Float32Arrays', async () => {
const fakeFetch = mockFetchOk({
data: [
{ embedding: [1, 2, 3, 4, 5, 6, 7, 8] },
{ embedding: [8, 7, 6, 5, 4, 3, 2, 1] },
],
});
const embedder = createLiteLLMEmbedder({ ...baseConfig, fetch: fakeFetch });
const results = await embedder.embedBatch(['hello', 'world']);
expect(results).toHaveLength(2);
expect(results[0]).toBeInstanceOf(Float32Array);
expect(results[1]).toBeInstanceOf(Float32Array);
expect(results[0][0]).toBe(1);
expect(results[1][0]).toBe(8);
// Should send array as input
const body = JSON.parse(fakeFetch.mock.calls[0][1].body);
expect(body.input).toEqual(['hello', 'world']);
});
it('embedBatch() returns empty array for empty input', async () => {
const fakeFetch = vi.fn();
const embedder = createLiteLLMEmbedder({ ...baseConfig, fetch: fakeFetch });
const results = await embedder.embedBatch([]);
expect(results).toEqual([]);
expect(fakeFetch).not.toHaveBeenCalled();
});
it('throws on API error when fallbackToMock is false', async () => {
const fakeFetch = vi.fn().mockResolvedValue({
ok: false,
status: 500,
text: async () => 'Internal Server Error',
} as unknown as Response);
const embedder = createLiteLLMEmbedder({ ...baseConfig, fetch: fakeFetch, fallbackToMock: false });
await expect(embedder.embed('test')).rejects.toThrow('LiteLLM embeddings error (500)');
});
it('falls back to mock on API error when fallbackToMock is true', async () => {
const fakeFetch = vi.fn().mockResolvedValue({
ok: false,
status: 500,
text: async () => 'error',
} as unknown as Response);
const embedder = createLiteLLMEmbedder({ ...baseConfig, fetch: fakeFetch, fallbackToMock: true });
const result = await embedder.embed('hello');
expect(result).toBeInstanceOf(Float32Array);
expect(result.length).toBe(8);
// Verify it's the deterministic mock: 'h' = 104, (104 - 128) / 128 = -0.1875
expect(result[0]).toBeCloseTo(-0.1875);
});
it('falls back to mock on network error when fallbackToMock is true', async () => {
const fakeFetch = vi.fn().mockRejectedValue(new Error('ECONNREFUSED'));
const embedder = createLiteLLMEmbedder({ ...baseConfig, fetch: fakeFetch, fallbackToMock: true });
const result = await embedder.embed('hi');
expect(result).toBeInstanceOf(Float32Array);
expect(result.length).toBe(8);
});
it('throws on network error when fallbackToMock is false', async () => {
const fakeFetch = vi.fn().mockRejectedValue(new Error('ECONNREFUSED'));
const embedder = createLiteLLMEmbedder({ ...baseConfig, fetch: fakeFetch });
await expect(embedder.embed('test')).rejects.toThrow('ECONNREFUSED');
});
it('omits Authorization header when no API key is provided', async () => {
const fakeFetch = mockFetchOk({
data: [{ embedding: [1, 2, 3, 4, 5, 6, 7, 8] }],
});
const embedder = createLiteLLMEmbedder({
litellmUrl: 'http://localhost:4000/v1',
dimensions: 8,
fetch: fakeFetch,
});
await embedder.embed('test');
const [, options] = fakeFetch.mock.calls[0];
expect(options.headers['Authorization']).toBeUndefined();
});
});

View File

@@ -0,0 +1,340 @@
import { describe, it, expect } from 'vitest';
import { parseChatGPTExport, parseClaudeExport, extractKnowledge, processImport } from '../src/memory-import';
describe('ChatGPT Export Parser', () => {
it('parses conversations with mapping structure', () => {
const data = [
{
title: 'Test Chat',
create_time: 1709000000,
mapping: {
'node1': {
message: {
author: { role: 'user' },
content: { parts: ['Hello, I need help with React'] },
create_time: 1709000001,
},
},
'node2': {
message: {
author: { role: 'assistant' },
content: { parts: ['Sure, I can help with React!'] },
create_time: 1709000002,
},
},
},
},
];
const result = parseChatGPTExport(data);
expect(result).toHaveLength(1);
expect(result[0].title).toBe('Test Chat');
expect(result[0].messages).toHaveLength(2);
expect(result[0].messages[0].role).toBe('user');
expect(result[0].messages[0].text).toBe('Hello, I need help with React');
expect(result[0].source).toBe('chatgpt');
});
it('skips system messages', () => {
const data = [
{
title: 'Test',
mapping: {
'sys': { message: { author: { role: 'system' }, content: { parts: ['System prompt'] } } },
'user': { message: { author: { role: 'user' }, content: { parts: ['Hi'] }, create_time: 1 } },
},
},
];
const result = parseChatGPTExport(data);
expect(result[0].messages).toHaveLength(1);
expect(result[0].messages[0].role).toBe('user');
});
it('handles empty/missing conversations gracefully', () => {
expect(parseChatGPTExport(null)).toEqual([]);
expect(parseChatGPTExport({})).toEqual([]);
expect(parseChatGPTExport([])).toEqual([]);
});
it('sorts messages chronologically by create_time', () => {
const data = [
{
title: 'Order Test',
mapping: {
'late': {
message: {
author: { role: 'user' },
content: { parts: ['Second message'] },
create_time: 200,
},
},
'early': {
message: {
author: { role: 'user' },
content: { parts: ['First message'] },
create_time: 100,
},
},
},
},
];
const result = parseChatGPTExport(data);
expect(result[0].messages[0].text).toBe('First message');
expect(result[0].messages[1].text).toBe('Second message');
});
it('filters out conversations with no messages', () => {
const data = [
{ title: 'Empty', mapping: {} },
{
title: 'Has Messages',
mapping: {
'n1': { message: { author: { role: 'user' }, content: { parts: ['Hello'] }, create_time: 1 } },
},
},
];
const result = parseChatGPTExport(data);
expect(result).toHaveLength(1);
expect(result[0].title).toBe('Has Messages');
});
it('handles wrapped format with conversations key', () => {
const data = {
conversations: [
{
title: 'Wrapped',
mapping: {
'n1': { message: { author: { role: 'user' }, content: { parts: ['Test'] }, create_time: 1 } },
},
},
],
};
const result = parseChatGPTExport(data);
expect(result).toHaveLength(1);
expect(result[0].title).toBe('Wrapped');
});
});
describe('Claude Export Parser', () => {
it('parses conversations with chat_messages', () => {
const data = [
{
name: 'Claude Chat',
created_at: '2024-03-01T10:00:00Z',
chat_messages: [
{ sender: 'human', text: 'What is TypeScript?', created_at: '2024-03-01T10:00:01Z' },
{ sender: 'assistant', text: 'TypeScript is a superset of JavaScript.', created_at: '2024-03-01T10:00:05Z' },
],
},
];
const result = parseClaudeExport(data);
expect(result).toHaveLength(1);
expect(result[0].title).toBe('Claude Chat');
expect(result[0].messages).toHaveLength(2);
expect(result[0].messages[0].role).toBe('user');
expect(result[0].source).toBe('claude');
});
it('handles empty exports gracefully', () => {
expect(parseClaudeExport(null)).toEqual([]);
expect(parseClaudeExport([])).toEqual([]);
});
it('handles alternative field names (title, messages, role, content)', () => {
const data = [
{
title: 'Alt Format',
messages: [
{ role: 'user', content: 'Hello from alt format', timestamp: '2024-03-01T10:00:00Z' },
],
},
];
const result = parseClaudeExport(data);
expect(result).toHaveLength(1);
expect(result[0].title).toBe('Alt Format');
expect(result[0].messages[0].text).toBe('Hello from alt format');
});
});
describe('Knowledge Extraction', () => {
it('extracts decisions from user messages', () => {
const convs = [{
title: 'Tech Discussion',
messages: [
{ role: 'user' as const, text: 'I decided to use React for the frontend and Node.js for the backend' },
{ role: 'assistant' as const, text: 'Great choice!' },
],
source: 'chatgpt' as const,
}];
const knowledge = extractKnowledge(convs);
const decisions = knowledge.filter(k => k.type === 'decision');
expect(decisions.length).toBeGreaterThan(0);
expect(decisions[0].content).toContain('decided');
expect(decisions[0].importance).toBe('important');
});
it('extracts preferences from user messages', () => {
const convs = [{
title: 'Preferences',
messages: [
{ role: 'user' as const, text: 'I prefer bullet-point summaries over long paragraphs' },
],
source: 'claude' as const,
}];
const knowledge = extractKnowledge(convs);
const prefs = knowledge.filter(k => k.type === 'preference');
expect(prefs.length).toBeGreaterThan(0);
expect(prefs[0].content).toContain('prefer');
expect(prefs[0].importance).toBe('important');
});
it('extracts facts about the user', () => {
const convs = [{
title: 'About me',
messages: [
{ role: 'user' as const, text: 'I work at Egzakta Advisory as a partner and consultant' },
],
source: 'chatgpt' as const,
}];
const knowledge = extractKnowledge(convs);
const facts = knowledge.filter(k => k.type === 'fact');
expect(facts.length).toBeGreaterThan(0);
expect(facts[0].content).toContain('Egzakta');
});
it('extracts conversation topics', () => {
const convs = [{
title: 'Building a SaaS Platform',
messages: [
{ role: 'user' as const, text: 'Help me plan the architecture' },
],
source: 'chatgpt' as const,
}];
const knowledge = extractKnowledge(convs);
const topics = knowledge.filter(k => k.type === 'topic');
expect(topics.length).toBeGreaterThan(0);
expect(topics[0].content).toContain('Building a SaaS Platform');
});
it('caps extraction at 100 items', () => {
const convs = Array.from({ length: 200 }, (_, i) => ({
title: `Conversation ${i}`,
messages: [
{ role: 'user' as const, text: `I decided to use approach ${i} for the implementation` },
],
source: 'chatgpt' as const,
}));
const knowledge = extractKnowledge(convs);
expect(knowledge.length).toBeLessThanOrEqual(100);
});
it('deduplicates similar content', () => {
const convs = [
{
title: 'Chat 1',
messages: [{ role: 'user' as const, text: 'I prefer TypeScript over JavaScript for type safety' }],
source: 'chatgpt' as const,
},
{
title: 'Chat 2',
messages: [{ role: 'user' as const, text: 'I prefer TypeScript over JavaScript for type safety' }],
source: 'chatgpt' as const,
},
];
const knowledge = extractKnowledge(convs);
const prefs = knowledge.filter(k => k.type === 'preference');
expect(prefs.length).toBe(1); // Deduped
});
it('skips very short and very long messages', () => {
const convs = [{
title: 'Test',
messages: [
{ role: 'user' as const, text: 'Hi' }, // Too short
{ role: 'user' as const, text: 'x'.repeat(600) }, // Too long
{ role: 'user' as const, text: 'I decided to use Python for data analysis tasks' }, // Just right
],
source: 'chatgpt' as const,
}];
const knowledge = extractKnowledge(convs);
const decisions = knowledge.filter(k => k.type === 'decision');
expect(decisions.length).toBe(1);
});
it('skips assistant messages', () => {
const convs = [{
title: 'Test',
messages: [
{ role: 'assistant' as const, text: 'I decided to use a different approach for this solution' },
],
source: 'chatgpt' as const,
}];
const knowledge = extractKnowledge(convs);
const decisions = knowledge.filter(k => k.type === 'decision');
expect(decisions.length).toBe(0);
});
});
describe('processImport (end-to-end)', () => {
it('processes a ChatGPT export end-to-end', () => {
const data = [
{
title: 'Project Planning',
create_time: 1709000000,
mapping: {
'n1': { message: { author: { role: 'user' }, content: { parts: ['I decided to use React with TypeScript for the frontend'] }, create_time: 1 } },
'n2': { message: { author: { role: 'assistant' }, content: { parts: ['Great choice!'] }, create_time: 2 } },
'n3': { message: { author: { role: 'user' }, content: { parts: ['I prefer concise responses without filler words'] }, create_time: 3 } },
},
},
];
const result = processImport(data, 'chatgpt');
expect(result.source).toBe('chatgpt');
expect(result.conversationsFound).toBe(1);
expect(result.conversationsParsed).toBe(1);
expect(result.knowledgeExtracted.length).toBeGreaterThan(0);
expect(result.errors).toHaveLength(0);
const decisions = result.knowledgeExtracted.filter(k => k.type === 'decision');
expect(decisions.length).toBeGreaterThan(0);
});
it('processes a Claude export end-to-end', () => {
const data = [
{
name: 'Architecture Discussion',
created_at: '2024-03-01T10:00:00Z',
chat_messages: [
{ sender: 'human', text: 'I work at Acme Corp as a senior engineer', created_at: '2024-03-01T10:00:01Z' },
{ sender: 'assistant', text: 'Nice to meet you!', created_at: '2024-03-01T10:00:05Z' },
],
},
];
const result = processImport(data, 'claude');
expect(result.source).toBe('claude');
expect(result.conversationsFound).toBe(1);
expect(result.knowledgeExtracted.length).toBeGreaterThan(0);
});
it('returns errors for invalid data', () => {
const result = processImport('not json', 'chatgpt');
expect(result.conversationsFound).toBe(0);
expect(result.knowledgeExtracted).toHaveLength(0);
});
it('reports when no conversations found', () => {
const result = processImport([], 'chatgpt');
expect(result.errors).toContain('No conversations found in export');
});
});

View File

@@ -0,0 +1,125 @@
import { describe, it, expect, afterEach } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { MindDB } from '@waggle/hive-mind-core';
import { FrameStore } from '@waggle/hive-mind-core';
import { needsMigration, migrateToMultiMind } from '../src/migration.js';
describe('Migration: default.mind → personal.mind', () => {
let tmpDir: string;
function makeTmpDir(): string {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-migration-'));
return dir;
}
afterEach(() => {
if (tmpDir && fs.existsSync(tmpDir)) {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});
describe('needsMigration', () => {
it('returns true when default.mind exists and personal.mind does not', () => {
tmpDir = makeTmpDir();
// Create a real MindDB so it's a valid SQLite file
const db = new MindDB(path.join(tmpDir, 'default.mind'));
db.close();
expect(needsMigration(tmpDir)).toBe(true);
});
it('returns false when personal.mind already exists', () => {
tmpDir = makeTmpDir();
// Both exist
const db1 = new MindDB(path.join(tmpDir, 'default.mind'));
db1.close();
const db2 = new MindDB(path.join(tmpDir, 'personal.mind'));
db2.close();
expect(needsMigration(tmpDir)).toBe(false);
});
it('returns false on fresh install (nothing exists)', () => {
tmpDir = makeTmpDir();
expect(needsMigration(tmpDir)).toBe(false);
});
});
describe('migrateToMultiMind', () => {
it('migrates default.mind to personal.mind with data preserved', () => {
tmpDir = makeTmpDir();
const defaultPath = path.join(tmpDir, 'default.mind');
// Create a MindDB with real data
const db = new MindDB(defaultPath);
// Create a session (foreign key requirement)
db.getDatabase().prepare(
"INSERT OR IGNORE INTO sessions (gop_id, status) VALUES (?, 'active')"
).run('test-gop');
const frames = new FrameStore(db);
frames.createIFrame('test-gop', 'migration test content', 'normal');
db.close();
const result = migrateToMultiMind(tmpDir);
expect(result.migrated).toBe(true);
expect(result.message).toBe('Migrated default.mind to personal.mind');
// Verify personal.mind has the data
const personalPath = path.join(tmpDir, 'personal.mind');
expect(fs.existsSync(personalPath)).toBe(true);
const personalDb = new MindDB(personalPath);
const personalFrames = new FrameStore(personalDb);
const state = personalFrames.reconstructState('test-gop');
expect(state.iframe).not.toBeNull();
expect(state.iframe!.content).toBe('migration test content');
personalDb.close();
});
it('keeps default.mind as backup (.bak) after migration', () => {
tmpDir = makeTmpDir();
const db = new MindDB(path.join(tmpDir, 'default.mind'));
db.close();
migrateToMultiMind(tmpDir);
expect(fs.existsSync(path.join(tmpDir, 'default.mind.bak'))).toBe(true);
expect(fs.existsSync(path.join(tmpDir, 'default.mind'))).toBe(false);
});
it('creates workspaces directory', () => {
tmpDir = makeTmpDir();
const db = new MindDB(path.join(tmpDir, 'default.mind'));
db.close();
migrateToMultiMind(tmpDir);
const wsDir = path.join(tmpDir, 'workspaces');
expect(fs.existsSync(wsDir)).toBe(true);
expect(fs.statSync(wsDir).isDirectory()).toBe(true);
});
it('does not migrate twice (idempotent)', () => {
tmpDir = makeTmpDir();
const db = new MindDB(path.join(tmpDir, 'default.mind'));
db.close();
const first = migrateToMultiMind(tmpDir);
expect(first.migrated).toBe(true);
const second = migrateToMultiMind(tmpDir);
expect(second.migrated).toBe(false);
expect(second.message).toBe('No migration needed');
});
it('returns no migration needed when nothing exists', () => {
tmpDir = makeTmpDir();
const result = migrateToMultiMind(tmpDir);
expect(result.migrated).toBe(false);
expect(result.message).toBe('No migration needed');
});
});
});

View File

@@ -0,0 +1,115 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { MindDB } from '@waggle/hive-mind-core';
import { SkillHashStore, computeSkillHash } from '../src/skill-hashes.js';
describe('SkillHashStore', () => {
let tmpDir: string;
let db: MindDB;
let store: SkillHashStore;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-skill-hash-'));
db = new MindDB(path.join(tmpDir, 'test.mind'));
store = new SkillHashStore(db);
});
afterEach(() => {
db.close();
fs.rmSync(tmpDir, { recursive: true, force: true });
});
it('setHash + getHash round-trips correctly', () => {
const hash = computeSkillHash('# My Skill\nDo something useful');
store.setHash('my-skill', hash);
const stored = store.getHash('my-skill');
expect(stored).toBeDefined();
expect(stored!.name).toBe('my-skill');
expect(stored!.hash).toBe(hash);
expect(stored!.verified_at).toBeTruthy();
});
it('computeSkillHash produces consistent SHA-256', () => {
const content = '# Draft Memo\nHelp the user draft professional memos.';
const hash1 = computeSkillHash(content);
const hash2 = computeSkillHash(content);
expect(hash1).toBe(hash2);
// SHA-256 hex is 64 characters
expect(hash1).toHaveLength(64);
// Should be hex string
expect(hash1).toMatch(/^[0-9a-f]{64}$/);
});
it('checkAll detects changed skill', () => {
const originalContent = '# Skill v1\nOriginal content';
store.verify('changed-skill', originalContent);
const result = store.checkAll([
{ name: 'changed-skill', content: '# Skill v2\nModified content' },
]);
expect(result.changed).toEqual(['changed-skill']);
expect(result.added).toEqual([]);
expect(result.removed).toEqual([]);
});
it('checkAll detects new (added) skill', () => {
const result = store.checkAll([
{ name: 'brand-new-skill', content: '# New Skill\nFresh content' },
]);
expect(result.added).toEqual(['brand-new-skill']);
expect(result.changed).toEqual([]);
expect(result.removed).toEqual([]);
});
it('checkAll detects removed skill', () => {
store.verify('old-skill', '# Old Skill\nGone now');
const result = store.checkAll([]);
expect(result.removed).toEqual(['old-skill']);
expect(result.changed).toEqual([]);
expect(result.added).toEqual([]);
});
it('checkAll returns empty when nothing changed', () => {
const content = '# Stable Skill\nNothing changed here';
store.verify('stable-skill', content);
const result = store.checkAll([
{ name: 'stable-skill', content },
]);
expect(result.changed).toEqual([]);
expect(result.added).toEqual([]);
expect(result.removed).toEqual([]);
});
it('verify updates hash to current content', () => {
const v1 = '# Skill v1';
const v2 = '# Skill v2';
store.verify('evolving-skill', v1);
const hashBefore = store.getHash('evolving-skill')!.hash;
expect(hashBefore).toBe(computeSkillHash(v1));
store.verify('evolving-skill', v2);
const hashAfter = store.getHash('evolving-skill')!.hash;
expect(hashAfter).toBe(computeSkillHash(v2));
expect(hashAfter).not.toBe(hashBefore);
});
it('removeHash cleans up', () => {
store.verify('doomed-skill', '# Doomed');
expect(store.getHash('doomed-skill')).toBeDefined();
store.removeHash('doomed-skill');
expect(store.getHash('doomed-skill')).toBeUndefined();
});
});

View File

@@ -0,0 +1,99 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import { MindDB } from '@waggle/hive-mind-core';
import { AwarenessLayer } from '@waggle/hive-mind-core';
describe('Structured Task Model', () => {
let db: MindDB;
let awareness: AwarenessLayer;
beforeEach(() => {
db = new MindDB(':memory:');
awareness = new AwarenessLayer(db);
});
afterEach(() => {
db.close();
});
it('stores task with metadata', () => {
const item = awareness.add('task', 'Fix the login bug', 0, undefined, {
status: 'pending',
context: 'auth module',
priority: 'high',
});
expect(item).toBeDefined();
expect(item.content).toBe('Fix the login bug');
const meta = awareness.parseMetadata(item);
expect(meta.status).toBe('pending');
expect(meta.context).toBe('auth module');
expect(meta.priority).toBe('high');
});
it('stores task without metadata (defaults to empty object)', () => {
const item = awareness.add('task', 'Simple task');
expect(item.metadata).toBe('{}');
const meta = awareness.parseMetadata(item);
expect(meta).toEqual({});
});
it('updates metadata on existing task', () => {
const item = awareness.add('task', 'Deploy v2.0', 0, undefined, { status: 'pending' });
awareness.updateMetadata(item.id, { status: 'in_progress', result: 'deploying...' });
const updated = awareness.get(item.id);
expect(updated).toBeDefined();
const meta = awareness.parseMetadata(updated!);
expect(meta.status).toBe('in_progress');
expect(meta.result).toBe('deploying...');
});
it('merges metadata without losing existing fields', () => {
const item = awareness.add('task', 'Multi-step task', 0, undefined, {
status: 'pending',
context: 'deployment',
});
awareness.updateMetadata(item.id, { status: 'in_progress' });
const updated = awareness.get(item.id);
const meta = awareness.parseMetadata(updated!);
expect(meta.status).toBe('in_progress');
expect(meta.context).toBe('deployment'); // preserved
});
it('throws when updating metadata for nonexistent item', () => {
expect(() => awareness.updateMetadata(999, { status: 'done' })).toThrow('Awareness item 999 not found');
});
it('retrieves tasks by status', () => {
awareness.add('task', 'Task A', 0, undefined, { status: 'pending' });
awareness.add('task', 'Task B', 0, undefined, { status: 'done' });
awareness.add('task', 'Task C', 0, undefined, { status: 'pending' });
const pending = awareness.getByStatus('pending');
expect(pending.length).toBe(2);
expect(pending.every(i => awareness.parseMetadata(i).status === 'pending')).toBe(true);
});
it('getByStatus returns empty array when no matches', () => {
awareness.add('task', 'Task A', 0, undefined, { status: 'pending' });
const inProgress = awareness.getByStatus('in_progress');
expect(inProgress).toEqual([]);
});
it('getByStatus ignores items without metadata status', () => {
awareness.add('task', 'No metadata task');
awareness.add('task', 'Has status', 0, undefined, { status: 'pending' });
const pending = awareness.getByStatus('pending');
expect(pending.length).toBe(1);
});
it('get() retrieves a single item by id', () => {
const item = awareness.add('task', 'Find me', 5);
const found = awareness.get(item.id);
expect(found).toBeDefined();
expect(found!.content).toBe('Find me');
expect(found!.priority).toBe(5);
});
it('get() returns undefined for nonexistent id', () => {
const found = awareness.get(999);
expect(found).toBeUndefined();
});
});

View File

@@ -0,0 +1,297 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { frameToEntity, entityToSyncedFrame, TeamSync, type TeamSyncConfig } from '../src/team-sync.js';
import type { MemoryFrame } from '@waggle/hive-mind-core';
describe('frameToEntity', () => {
it('converts a MemoryFrame to team entity format', () => {
const frame: MemoryFrame = {
id: 42,
frame_type: 'I',
gop_id: 'project-context',
t: 3,
base_frame_id: null,
content: 'The project uses React and TypeScript',
importance: 'important',
access_count: 5,
created_at: '2026-03-12T10:00:00.000Z',
last_accessed: '2026-03-12T12:00:00.000Z',
};
const entity = frameToEntity(frame, 'user-abc', 'Marko');
expect(entity.entityType).toBe('memory_frame');
expect(entity.name).toBe('project-context');
expect(entity.properties.frameType).toBe('I');
expect(entity.properties.t).toBe(3);
expect(entity.properties.baseFrameId).toBeNull();
expect(entity.properties.content).toBe('The project uses React and TypeScript');
expect(entity.properties.importance).toBe('important');
expect(entity.properties.authorId).toBe('user-abc');
expect(entity.properties.authorName).toBe('Marko');
expect(entity.properties.localId).toBe(42);
});
it('preserves P-frame base_frame_id', () => {
const frame: MemoryFrame = {
id: 43,
frame_type: 'P',
gop_id: 'project-context',
t: 4,
base_frame_id: 42,
content: 'Updated: now also using Tailwind',
importance: 'normal',
access_count: 0,
created_at: '2026-03-12T11:00:00.000Z',
last_accessed: '2026-03-12T11:00:00.000Z',
};
const entity = frameToEntity(frame, 'user-xyz', 'Ana');
expect(entity.properties.frameType).toBe('P');
expect(entity.properties.baseFrameId).toBe(42);
expect(entity.properties.authorName).toBe('Ana');
});
});
describe('entityToSyncedFrame', () => {
it('converts a team entity back to SyncedFrame', () => {
const entity = {
id: 'uuid-remote-1',
name: 'project-context',
properties: {
frameType: 'I',
t: 3,
baseFrameId: null,
content: 'The project uses React and TypeScript',
importance: 'important',
authorId: 'user-abc',
authorName: 'Marko',
localId: 42,
},
createdAt: '2026-03-12T10:00:00.000Z',
};
const frame = entityToSyncedFrame(entity);
expect(frame.remoteId).toBe('uuid-remote-1');
expect(frame.gopId).toBe('project-context');
expect(frame.t).toBe(3);
expect(frame.frameType).toBe('I');
expect(frame.content).toBe('The project uses React and TypeScript');
expect(frame.importance).toBe('important');
expect(frame.authorId).toBe('user-abc');
expect(frame.authorName).toBe('Marko');
expect(frame.createdAt).toBe('2026-03-12T10:00:00.000Z');
});
it('handles missing properties gracefully', () => {
const entity = {
id: 'uuid-remote-2',
name: 'some-gop',
properties: {},
createdAt: '2026-03-12T10:00:00.000Z',
};
const frame = entityToSyncedFrame(entity);
expect(frame.remoteId).toBe('uuid-remote-2');
expect(frame.gopId).toBe('some-gop');
expect(frame.t).toBe(0);
expect(frame.frameType).toBe('I');
expect(frame.content).toBe('');
expect(frame.importance).toBe('normal');
expect(frame.authorId).toBe('');
expect(frame.authorName).toBe('');
});
});
describe('TeamSync', () => {
const mockConfig: TeamSyncConfig = {
teamServerUrl: 'https://team.waggle.dev',
teamSlug: 'test-team',
authToken: 'test-jwt-token',
userId: 'user-abc',
displayName: 'Marko',
};
let fetchSpy: ReturnType<typeof vi.fn>;
beforeEach(() => {
fetchSpy = vi.fn();
vi.stubGlobal('fetch', fetchSpy);
});
afterEach(() => {
vi.restoreAllMocks();
});
describe('pushFrame', () => {
it('sends frame to team server entities endpoint', async () => {
fetchSpy.mockResolvedValue({
ok: true,
json: async () => ({ id: 'remote-uuid-1' }),
});
const sync = new TeamSync(mockConfig);
const frame: MemoryFrame = {
id: 1,
frame_type: 'I',
gop_id: 'test-gop',
t: 0,
base_frame_id: null,
content: 'Test content',
importance: 'normal',
access_count: 0,
created_at: '2026-03-12T10:00:00.000Z',
last_accessed: '2026-03-12T10:00:00.000Z',
};
const result = await sync.pushFrame(frame);
expect(result).toEqual({ remoteId: 'remote-uuid-1' });
expect(fetchSpy).toHaveBeenCalledOnce();
const [url, opts] = fetchSpy.mock.calls[0];
expect(url).toBe('https://team.waggle.dev/api/teams/test-team/entities');
expect(opts.method).toBe('POST');
expect(opts.headers['Authorization']).toBe('Bearer test-jwt-token');
const body = JSON.parse(opts.body);
expect(body.entityType).toBe('memory_frame');
expect(body.name).toBe('test-gop');
expect(body.properties.content).toBe('Test content');
expect(body.properties.authorId).toBe('user-abc');
expect(body.properties.authorName).toBe('Marko');
});
it('returns null on server error', async () => {
fetchSpy.mockResolvedValue({
ok: false,
status: 500,
statusText: 'Internal Server Error',
});
const sync = new TeamSync(mockConfig);
const frame: MemoryFrame = {
id: 1, frame_type: 'I', gop_id: 'test', t: 0, base_frame_id: null,
content: 'x', importance: 'normal', access_count: 0,
created_at: '2026-03-12T10:00:00.000Z', last_accessed: '2026-03-12T10:00:00.000Z',
};
const result = await sync.pushFrame(frame);
expect(result).toBeNull();
});
it('returns null on network error', async () => {
fetchSpy.mockRejectedValue(new Error('Network unreachable'));
const sync = new TeamSync(mockConfig);
const frame: MemoryFrame = {
id: 1, frame_type: 'I', gop_id: 'test', t: 0, base_frame_id: null,
content: 'x', importance: 'normal', access_count: 0,
created_at: '2026-03-12T10:00:00.000Z', last_accessed: '2026-03-12T10:00:00.000Z',
};
const result = await sync.pushFrame(frame);
expect(result).toBeNull();
});
});
describe('pullFrames', () => {
it('fetches frames from team server', async () => {
fetchSpy.mockResolvedValue({
ok: true,
json: async () => ([
{
id: 'uuid-1',
name: 'gop-a',
properties: { frameType: 'I', t: 0, content: 'First', importance: 'important', authorId: 'u1', authorName: 'Marko' },
createdAt: '2026-03-12T10:00:00.000Z',
},
{
id: 'uuid-2',
name: 'gop-b',
properties: { frameType: 'P', t: 1, content: 'Second', importance: 'normal', authorId: 'u2', authorName: 'Ana' },
createdAt: '2026-03-12T11:00:00.000Z',
},
]),
});
const sync = new TeamSync(mockConfig);
const frames = await sync.pullFrames();
expect(frames).toHaveLength(2);
expect(frames[0].gopId).toBe('gop-a');
expect(frames[0].authorName).toBe('Marko');
expect(frames[1].gopId).toBe('gop-b');
expect(frames[1].authorName).toBe('Ana');
const [url, opts] = fetchSpy.mock.calls[0];
expect(url).toContain('/api/teams/test-team/entities?type=memory_frame');
expect(opts.headers['Authorization']).toBe('Bearer test-jwt-token');
});
it('filters by since timestamp when provided', async () => {
fetchSpy.mockResolvedValue({
ok: true,
json: async () => ([
{
id: 'uuid-1', name: 'gop-a',
properties: { frameType: 'I', t: 0, content: 'Old', authorId: 'u1', authorName: 'M' },
createdAt: '2026-03-11T10:00:00.000Z',
},
{
id: 'uuid-2', name: 'gop-b',
properties: { frameType: 'I', t: 0, content: 'New', authorId: 'u2', authorName: 'A' },
createdAt: '2026-03-12T15:00:00.000Z',
},
]),
});
const sync = new TeamSync(mockConfig);
const frames = await sync.pullFrames('2026-03-12T00:00:00.000Z');
// Only the frame after the since timestamp
expect(frames).toHaveLength(1);
expect(frames[0].content).toBe('New');
});
it('returns empty array on server error', async () => {
fetchSpy.mockResolvedValue({ ok: false, status: 500, statusText: 'Error' });
const sync = new TeamSync(mockConfig);
const frames = await sync.pullFrames();
expect(frames).toEqual([]);
});
it('returns empty array on network error', async () => {
fetchSpy.mockRejectedValue(new Error('Offline'));
const sync = new TeamSync(mockConfig);
const frames = await sync.pullFrames();
expect(frames).toEqual([]);
});
});
describe('sync timestamp tracking', () => {
it('starts with null timestamp', () => {
const sync = new TeamSync(mockConfig);
expect(sync.getLastSyncTimestamp()).toBeNull();
});
it('updates timestamp after successful pull', async () => {
fetchSpy.mockResolvedValue({ ok: true, json: async () => ([]) });
const sync = new TeamSync(mockConfig);
await sync.pullFrames();
expect(sync.getLastSyncTimestamp()).toBeTruthy();
});
it('allows manual timestamp setting', () => {
const sync = new TeamSync(mockConfig);
sync.setLastSyncTimestamp('2026-03-12T10:00:00.000Z');
expect(sync.getLastSyncTimestamp()).toBe('2026-03-12T10:00:00.000Z');
});
});
});

View File

@@ -0,0 +1,112 @@
import { describe, it, expect, beforeEach } from 'vitest';
import { TelemetryCollector } from '../src/telemetry.js';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
describe('TelemetryCollector', () => {
let tmpDir: string;
beforeEach(() => {
tmpDir = path.join(os.tmpdir(), `waggle-telemetry-test-${Date.now()}`);
fs.mkdirSync(tmpDir, { recursive: true });
});
it('records events when enabled', () => {
const collector = new TelemetryCollector(tmpDir, true);
collector.recordToolUse('web_search');
collector.recordToolUse('web_search');
collector.recordToolUse('save_memory');
const report = collector.getReport();
expect(report.totalEvents).toBe(3);
const webSearch = report.events.find(e => e.name === 'web_search');
expect(webSearch?.count).toBe(2);
});
it('silently drops events when disabled', () => {
const collector = new TelemetryCollector(tmpDir, false);
collector.recordToolUse('web_search');
collector.recordCommand('/research');
collector.recordError('timeout');
const report = collector.getReport();
expect(report.totalEvents).toBe(0);
});
it('no PII in collected data — tool names only', () => {
const collector = new TelemetryCollector(tmpDir, true);
collector.recordToolUse('connector_github_create_issue');
collector.recordError('api_timeout');
collector.recordCapabilityGap('email sending');
const report = collector.getReport();
for (const event of report.events) {
// Only category, name, count, date — no message content or file paths
expect(Object.keys(event).sort()).toEqual(['category', 'count', 'date', 'name']);
}
});
it('daily aggregation: same-day events merge counts', () => {
const collector = new TelemetryCollector(tmpDir, true);
collector.recordToolUse('bash');
collector.recordToolUse('bash');
collector.recordToolUse('bash');
const report = collector.getReport();
const bashEvents = report.events.filter(e => e.name === 'bash');
expect(bashEvents).toHaveLength(1);
expect(bashEvents[0].count).toBe(3);
});
it('flush writes to telemetry.json', () => {
const collector = new TelemetryCollector(tmpDir, true);
collector.recordToolUse('search_memory');
collector.flush();
const filePath = path.join(tmpDir, 'telemetry.json');
expect(fs.existsSync(filePath)).toBe(true);
const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
expect(data).toHaveLength(1);
expect(data[0].name).toBe('search_memory');
collector.close();
fs.rmSync(tmpDir, { recursive: true, force: true });
});
it('report covers last N days', () => {
const collector = new TelemetryCollector(tmpDir, true);
collector.recordToolUse('today_tool');
const report = collector.getReport(7);
expect(report.events.length).toBeGreaterThan(0);
expect(report.dateRange.from).toBeTruthy();
});
it('setEnabled toggles collection', () => {
const collector = new TelemetryCollector(tmpDir, false);
expect(collector.isEnabled()).toBe(false);
collector.setEnabled(true);
expect(collector.isEnabled()).toBe(true);
collector.recordToolUse('test_tool');
expect(collector.getReport().totalEvents).toBe(1);
});
it('recordSession aggregates duration and interaction count', () => {
const collector = new TelemetryCollector(tmpDir, true);
collector.recordSession(30000, 15);
collector.recordSession(20000, 10);
const report = collector.getReport();
const duration = report.events.find(e => e.name === 'duration_total_ms');
const interactions = report.events.find(e => e.name === 'interaction_count');
const sessions = report.events.find(e => e.name === 'session_count');
expect(duration?.count).toBe(50000);
expect(interactions?.count).toBe(25);
expect(sessions?.count).toBe(2);
});
});

View File

@@ -0,0 +1,136 @@
import { describe, it, expect, afterEach } from 'vitest';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { VaultStore } from '../src/vault.js';
describe('Vault Concurrency & Atomic Writes (11B-8)', () => {
const tempDirs: string[] = [];
function makeTempDir(): string {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-vault-conc-'));
tempDirs.push(dir);
return dir;
}
afterEach(() => {
for (const dir of tempDirs) {
fs.rmSync(dir, { recursive: true, force: true });
}
tempDirs.length = 0;
});
it('5 concurrent setAsync calls — all 5 keys exist after', async () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
// Fire 5 concurrent writes
await Promise.all([
vault.setAsync('key-1', 'value-1'),
vault.setAsync('key-2', 'value-2'),
vault.setAsync('key-3', 'value-3'),
vault.setAsync('key-4', 'value-4'),
vault.setAsync('key-5', 'value-5'),
]);
// Verify all 5 keys exist and have correct values
for (let i = 1; i <= 5; i++) {
const entry = vault.get(`key-${i}`);
expect(entry).not.toBeNull();
expect(entry!.value).toBe(`value-${i}`);
}
// Verify via list that all 5 are present
const list = vault.list();
expect(list).toHaveLength(5);
});
it('concurrent setAsync and deleteAsync do not corrupt vault', async () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
// Set some initial keys
vault.set('keep-1', 'v1');
vault.set('remove-1', 'v2');
vault.set('keep-2', 'v3');
// Concurrently set new keys and delete existing ones
await Promise.all([
vault.setAsync('new-1', 'new-v1'),
vault.deleteAsync('remove-1'),
vault.setAsync('new-2', 'new-v2'),
]);
expect(vault.get('keep-1')?.value).toBe('v1');
expect(vault.get('keep-2')?.value).toBe('v3');
expect(vault.get('new-1')?.value).toBe('new-v1');
expect(vault.get('new-2')?.value).toBe('new-v2');
expect(vault.get('remove-1')).toBeNull();
});
it('atomic write — no vault.json.tmp left behind after write', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
vault.set('test', 'value');
const tmpPath = path.join(dir, 'vault.json.tmp');
expect(fs.existsSync(tmpPath)).toBe(false);
expect(fs.existsSync(path.join(dir, 'vault.json'))).toBe(true);
});
it('deleteAsync returns correct existed flag', async () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
vault.set('exists', 'val');
const existed = await vault.deleteAsync('exists');
expect(existed).toBe(true);
const notExisted = await vault.deleteAsync('never-was');
expect(notExisted).toBe(false);
});
it('setAsync overwrites existing key', async () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
vault.set('overwrite-me', 'old-value');
await vault.setAsync('overwrite-me', 'new-value');
const entry = vault.get('overwrite-me');
expect(entry).not.toBeNull();
expect(entry!.value).toBe('new-value');
});
it('10 concurrent setAsync calls on same key — last one wins', async () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
// Chain them — since writeLock serializes, the last .then() should win
const promises: Promise<void>[] = [];
for (let i = 0; i < 10; i++) {
promises.push(vault.setAsync('contested', `value-${i}`));
}
await Promise.all(promises);
const entry = vault.get('contested');
expect(entry).not.toBeNull();
// The value should be the last one written (value-9)
// because the write lock serializes them in order
expect(entry!.value).toBe('value-9');
});
it('sync set still works correctly (backward compatibility)', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
vault.set('sync-key', 'sync-value', { tag: 'test' });
const entry = vault.get('sync-key');
expect(entry).not.toBeNull();
expect(entry!.value).toBe('sync-value');
expect(entry!.metadata).toEqual({ tag: 'test' });
});
});

View File

@@ -0,0 +1,261 @@
/**
* Vault edge case tests — corrupted files, missing files,
* concurrent read/write safety, and large value storage.
*/
import { describe, it, expect, afterEach } from 'vitest';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { VaultStore } from '../src/vault.js';
describe('VaultStore edge cases', () => {
const tempDirs: string[] = [];
function makeTempDir(): string {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-vault-edge-'));
tempDirs.push(dir);
return dir;
}
afterEach(() => {
for (const dir of tempDirs) {
fs.rmSync(dir, { recursive: true, force: true });
}
tempDirs.length = 0;
});
// ── Corrupted vault.json ────────────────────────────────────────────
describe('corrupted vault.json', () => {
it('handles invalid JSON gracefully — returns null for get', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
// Store a valid secret first
vault.set('test-key', 'test-value');
expect(vault.get('test-key')!.value).toBe('test-value');
// Corrupt the vault.json file with invalid JSON
const vaultPath = path.join(dir, 'vault.json');
fs.writeFileSync(vaultPath, '{ this is not valid JSON !!!', 'utf-8');
// get should return null (not crash)
expect(vault.get('test-key')).toBeNull();
});
it('handles invalid JSON gracefully — list returns empty array', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
// Corrupt the vault.json file
const vaultPath = path.join(dir, 'vault.json');
fs.writeFileSync(vaultPath, '<xml>not json</xml>', 'utf-8');
// list should return empty (not crash)
expect(vault.list()).toEqual([]);
});
it('handles invalid JSON gracefully — has returns false', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
// Corrupt the vault.json file
const vaultPath = path.join(dir, 'vault.json');
fs.writeFileSync(vaultPath, '}}broken{{', 'utf-8');
// has should return false (not crash)
expect(vault.has('anything')).toBe(false);
});
it('can write new secrets after corruption — overwrites corrupted file', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
// Corrupt the vault.json file
const vaultPath = path.join(dir, 'vault.json');
fs.writeFileSync(vaultPath, 'CORRUPT!', 'utf-8');
// set should overwrite corrupted file with valid data
vault.set('recovery-key', 'recovered-value');
// Should be able to read back the new value
const entry = vault.get('recovery-key');
expect(entry).not.toBeNull();
expect(entry!.value).toBe('recovered-value');
// Verify the file is now valid JSON
const raw = fs.readFileSync(vaultPath, 'utf-8');
expect(() => JSON.parse(raw)).not.toThrow();
});
});
// ── Missing vault.json ──────────────────────────────────────────────
describe('missing vault.json', () => {
it('creates a new vault.json on next write when file is missing', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
const vaultPath = path.join(dir, 'vault.json');
// Ensure no vault.json exists initially
if (fs.existsSync(vaultPath)) {
fs.unlinkSync(vaultPath);
}
expect(fs.existsSync(vaultPath)).toBe(false);
// Write a secret — should create vault.json
vault.set('new-key', 'new-value');
expect(fs.existsSync(vaultPath)).toBe(true);
const entry = vault.get('new-key');
expect(entry).not.toBeNull();
expect(entry!.value).toBe('new-value');
});
it('get returns null when vault.json does not exist', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
const vaultPath = path.join(dir, 'vault.json');
// Ensure no vault.json exists
if (fs.existsSync(vaultPath)) {
fs.unlinkSync(vaultPath);
}
expect(vault.get('missing')).toBeNull();
});
it('list returns empty array when vault.json does not exist', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
const vaultPath = path.join(dir, 'vault.json');
// Ensure no vault.json exists
if (fs.existsSync(vaultPath)) {
fs.unlinkSync(vaultPath);
}
expect(vault.list()).toEqual([]);
});
});
// ── Concurrent reads during write ──────────────────────────────────
describe('concurrent reads during write', () => {
it('read during setAsync does not corrupt data', async () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
// Pre-populate with a known value
vault.set('existing', 'original-value');
// Start an async write, then immediately read
const writePromise = vault.setAsync('new-key', 'new-value');
const readResult = vault.get('existing');
await writePromise;
// The original key should still be readable (not corrupted)
expect(readResult).not.toBeNull();
expect(readResult!.value).toBe('original-value');
// The new key should also be present
const newEntry = vault.get('new-key');
expect(newEntry).not.toBeNull();
expect(newEntry!.value).toBe('new-value');
// The existing key should still be intact
const existingAfter = vault.get('existing');
expect(existingAfter).not.toBeNull();
expect(existingAfter!.value).toBe('original-value');
});
it('multiple rapid reads interleaved with writes produce consistent results', async () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
// Write initial data
vault.set('stable', 'stable-value');
// Fire off writes and reads in rapid succession
const writePromises = [];
for (let i = 0; i < 5; i++) {
writePromises.push(vault.setAsync(`rapid-${i}`, `value-${i}`));
}
// Interleave reads
const readResults: (string | null)[] = [];
for (let i = 0; i < 5; i++) {
const entry = vault.get('stable');
readResults.push(entry?.value ?? null);
}
await Promise.all(writePromises);
// All reads of the stable key should return the correct value
for (const val of readResults) {
expect(val).toBe('stable-value');
}
// All written keys should be present
for (let i = 0; i < 5; i++) {
const entry = vault.get(`rapid-${i}`);
expect(entry).not.toBeNull();
expect(entry!.value).toBe(`value-${i}`);
}
});
});
// ── Large value storage ────────────────────────────────────────────
describe('large value storage', () => {
it('stores and retrieves a 1MB string correctly', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
// Generate a 1MB string (1,048,576 characters)
const largeValue = 'A'.repeat(1024 * 1024);
expect(largeValue.length).toBe(1024 * 1024);
vault.set('large-secret', largeValue);
const entry = vault.get('large-secret');
expect(entry).not.toBeNull();
expect(entry!.value).toBe(largeValue);
expect(entry!.value.length).toBe(1024 * 1024);
});
it('large value is encrypted in vault.json — plaintext not present', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
// Use a distinctive pattern that would be easy to find if unencrypted
const largeValue = 'SECRET_MARKER_'.repeat(10000);
vault.set('large-encrypted', largeValue);
const rawContent = fs.readFileSync(path.join(dir, 'vault.json'), 'utf-8');
expect(rawContent).not.toContain('SECRET_MARKER_');
// Verify the encrypted field has the correct format
const parsed = JSON.parse(rawContent);
expect(parsed['large-encrypted'].encrypted).toMatch(/^[0-9a-f]+:[0-9a-f]+:[0-9a-f]+$/);
});
it('large value survives round-trip through different VaultStore instances', () => {
const dir = makeTempDir();
// Write with one instance
const vault1 = new VaultStore(dir);
const largeValue = 'B'.repeat(1024 * 1024);
vault1.set('large-roundtrip', largeValue);
// Read with a different instance (same key file)
const vault2 = new VaultStore(dir);
const entry = vault2.get('large-roundtrip');
expect(entry).not.toBeNull();
expect(entry!.value).toBe(largeValue);
expect(entry!.value.length).toBe(1024 * 1024);
});
});
});

View File

@@ -0,0 +1,391 @@
import { describe, it, expect, afterEach, vi } from 'vitest';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
// Hoisted mock for node:child_process so static imports in vault.ts are intercepted.
// Defaults to the real implementation; individual tests override via mockImplementation.
const mockExecFileSync = vi.hoisted(() => vi.fn());
vi.mock('node:child_process', async (importOriginal) => {
const actual = await importOriginal<typeof import('node:child_process')>();
return { ...actual, execFileSync: mockExecFileSync };
});
import { VaultStore, type VaultEntry } from '../src/vault.js';
describe('VaultStore', () => {
const tempDirs: string[] = [];
function makeTempDir(): string {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-vault-test-'));
tempDirs.push(dir);
return dir;
}
afterEach(() => {
for (const dir of tempDirs) {
fs.rmSync(dir, { recursive: true, force: true });
}
tempDirs.length = 0;
});
it('set and get — store a secret, retrieve it, value matches', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
vault.set('anthropic', 'sk-ant-secret-key-123');
const entry = vault.get('anthropic');
expect(entry).not.toBeNull();
expect(entry!.name).toBe('anthropic');
expect(entry!.value).toBe('sk-ant-secret-key-123');
expect(entry!.updatedAt).toBeTruthy();
});
it('set overwrites — set same name twice, get returns latest', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
vault.set('openai', 'sk-old-key');
vault.set('openai', 'sk-new-key');
const entry = vault.get('openai');
expect(entry).not.toBeNull();
expect(entry!.value).toBe('sk-new-key');
});
it('get nonexistent — returns null', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
expect(vault.get('doesnotexist')).toBeNull();
});
it('delete — removes secret, get returns null after', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
vault.set('anthropic', 'sk-ant-key');
expect(vault.delete('anthropic')).toBe(true);
expect(vault.get('anthropic')).toBeNull();
});
it('delete nonexistent — returns false', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
expect(vault.delete('nope')).toBe(false);
});
it('list — shows names + metadata without values', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
vault.set('anthropic', 'sk-ant-key', { models: ['claude-sonnet-4-6'] });
vault.set('openai', 'sk-openai-key', { models: ['gpt-4o'], baseUrl: 'https://api.openai.com' });
const entries = vault.list();
expect(entries).toHaveLength(2);
const names = entries.map(e => e.name);
expect(names).toContain('anthropic');
expect(names).toContain('openai');
// list must NOT contain secret values
for (const entry of entries) {
expect(entry).not.toHaveProperty('value');
expect(entry.updatedAt).toBeTruthy();
}
const anthropicEntry = entries.find(e => e.name === 'anthropic')!;
expect(anthropicEntry.metadata).toEqual({ models: ['claude-sonnet-4-6'] });
});
it('has — returns true for existing, false for missing', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
vault.set('anthropic', 'sk-ant-key');
expect(vault.has('anthropic')).toBe(true);
expect(vault.has('missing')).toBe(false);
});
it('encryption is real — vault.json does NOT contain plaintext value', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
const secret = 'sk-ant-super-secret-api-key-12345';
vault.set('anthropic', secret);
const rawContent = fs.readFileSync(path.join(dir, 'vault.json'), 'utf-8');
expect(rawContent).not.toContain(secret);
// The encrypted field should exist and contain hex data with colons (iv:tag:ciphertext)
const parsed = JSON.parse(rawContent);
expect(parsed.anthropic.encrypted).toMatch(/^[0-9a-f]+:[0-9a-f]+:[0-9a-f]+$/);
});
it('different VaultStore instances with same key can decrypt', () => {
const dir = makeTempDir();
const vault1 = new VaultStore(dir);
vault1.set('anthropic', 'sk-ant-shared-secret');
// Create a second instance pointing at the same directory (same key file)
const vault2 = new VaultStore(dir);
const entry = vault2.get('anthropic');
expect(entry).not.toBeNull();
expect(entry!.value).toBe('sk-ant-shared-secret');
});
it('migration from config — providers migrated to vault', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
const config = {
providers: {
anthropic: { apiKey: 'sk-ant-key-1', models: ['claude-sonnet-4-6'], baseUrl: undefined },
openai: { apiKey: 'sk-openai-key-1', models: ['gpt-4o'] },
},
};
const migrated = vault.migrateFromConfig(config);
expect(migrated).toBe(2);
const anthropic = vault.get('anthropic');
expect(anthropic).not.toBeNull();
expect(anthropic!.value).toBe('sk-ant-key-1');
expect(anthropic!.metadata?.models).toEqual(['claude-sonnet-4-6']);
const openai = vault.get('openai');
expect(openai).not.toBeNull();
expect(openai!.value).toBe('sk-openai-key-1');
});
it('migration skips existing — migrate twice, count stays same', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
const config = {
providers: {
anthropic: { apiKey: 'sk-ant-key-1', models: ['claude-sonnet-4-6'] },
},
};
const first = vault.migrateFromConfig(config);
expect(first).toBe(1);
const second = vault.migrateFromConfig(config);
expect(second).toBe(0);
});
it('key file is generated on first use and reused', () => {
const dir = makeTempDir();
new VaultStore(dir);
const keyPath = path.join(dir, '.vault-key');
expect(fs.existsSync(keyPath)).toBe(true);
// Key should be 64 hex chars (32 bytes)
const keyHex = fs.readFileSync(keyPath, 'utf-8').trim();
expect(keyHex).toMatch(/^[0-9a-f]{64}$/);
// Second instance should use the same key (not overwrite)
new VaultStore(dir);
const keyHex2 = fs.readFileSync(keyPath, 'utf-8').trim();
expect(keyHex2).toBe(keyHex);
});
it('setConnectorCredential — refresh token is NOT stored as plaintext in vault.json', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
const refreshToken = 'rt-super-secret-refresh-token-xyz';
vault.setConnectorCredential('github', {
type: 'oauth2',
value: 'gho_access_token_123',
refreshToken,
expiresAt: '2099-01-01T00:00:00Z',
scopes: ['repo', 'user'],
});
// Read raw vault.json and verify the refresh token is NOT in plaintext
const rawContent = fs.readFileSync(path.join(dir, 'vault.json'), 'utf-8');
expect(rawContent).not.toContain(refreshToken);
// The metadata for the main connector entry must NOT contain refreshToken
const parsed = JSON.parse(rawContent);
const mainEntry = parsed['connector:github'];
expect(mainEntry).toBeDefined();
expect(mainEntry.metadata).not.toHaveProperty('refreshToken');
// The refresh token should be stored as a separate encrypted entry
const refreshEntry = parsed['connector:github:refresh'];
expect(refreshEntry).toBeDefined();
expect(refreshEntry.encrypted).toMatch(/^[0-9a-f]+:[0-9a-f]+:[0-9a-f]+$/);
});
it('getConnectorCredential — returns decrypted refresh token correctly', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
const accessToken = 'gho_access_token_456';
const refreshToken = 'rt-secret-refresh-token-abc';
vault.setConnectorCredential('github', {
type: 'oauth2',
value: accessToken,
refreshToken,
expiresAt: '2099-01-01T00:00:00Z',
scopes: ['repo'],
});
const cred = vault.getConnectorCredential('github');
expect(cred).not.toBeNull();
expect(cred!.value).toBe(accessToken);
expect(cred!.refreshToken).toBe(refreshToken);
expect(cred!.type).toBe('oauth2');
expect(cred!.expiresAt).toBe('2099-01-01T00:00:00Z');
expect(cred!.scopes).toEqual(['repo']);
expect(cred!.isExpired).toBe(false);
});
it('setConnectorCredential — without refresh token does not create refresh entry', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
vault.setConnectorCredential('slack', {
type: 'bearer',
value: 'xoxb-token-123',
});
const rawContent = fs.readFileSync(path.join(dir, 'vault.json'), 'utf-8');
const parsed = JSON.parse(rawContent);
expect(parsed['connector:slack']).toBeDefined();
expect(parsed['connector:slack:refresh']).toBeUndefined();
const cred = vault.getConnectorCredential('slack');
expect(cred).not.toBeNull();
expect(cred!.value).toBe('xoxb-token-123');
expect(cred!.refreshToken).toBeUndefined();
});
it('setConnectorCredential — clears refresh token when re-set without one', () => {
const dir = makeTempDir();
const vault = new VaultStore(dir);
// First set with refresh token
vault.setConnectorCredential('github', {
type: 'oauth2',
value: 'gho_token_1',
refreshToken: 'rt-old-refresh',
});
expect(vault.getConnectorCredential('github')!.refreshToken).toBe('rt-old-refresh');
// Re-set without refresh token
vault.setConnectorCredential('github', {
type: 'oauth2',
value: 'gho_token_2',
});
const cred = vault.getConnectorCredential('github');
expect(cred!.value).toBe('gho_token_2');
expect(cred!.refreshToken).toBeUndefined();
expect(vault.has('connector:github:refresh')).toBe(false);
});
it('corrupted key file — truncated hex throws clear error', () => {
const dir = makeTempDir();
const keyPath = path.join(dir, '.vault-key');
// Write a truncated key (only 10 hex chars = 5 bytes instead of 32)
fs.writeFileSync(keyPath, 'abcdef0123', { mode: 0o600 });
expect(() => new VaultStore(dir)).toThrowError(
/Vault key file is corrupted — expected 32 bytes, got 5/
);
});
it('corrupted key file — non-hex content throws clear error', () => {
const dir = makeTempDir();
const keyPath = path.join(dir, '.vault-key');
// Non-hex content: Buffer.from('not-hex-at-all', 'hex') silently produces a short buffer
fs.writeFileSync(keyPath, 'not-hex-at-all-garbage-content', { mode: 0o600 });
expect(() => new VaultStore(dir)).toThrowError(
/Vault key file is corrupted — expected 32 bytes/
);
});
it('corrupted key file — empty file throws clear error', () => {
const dir = makeTempDir();
const keyPath = path.join(dir, '.vault-key');
fs.writeFileSync(keyPath, '', { mode: 0o600 });
expect(() => new VaultStore(dir)).toThrowError(
/Vault key file is corrupted — expected 32 bytes, got 0/
);
});
it('Windows key protection — icacls is attempted on win32', () => {
const dir = makeTempDir();
const keyPath = path.join(dir, '.vault-key');
// Ensure no key file exists so ensureKey() will generate one
if (fs.existsSync(keyPath)) fs.unlinkSync(keyPath);
// Mock process.platform to 'win32'
const originalPlatform = Object.getOwnPropertyDescriptor(process, 'platform')!;
Object.defineProperty(process, 'platform', { value: 'win32', configurable: true });
// Use the hoisted mock: first call = whoami, second call = icacls
mockExecFileSync.mockImplementation((cmd: string) => {
if (cmd === 'whoami') return 'DOMAIN\\testuser';
return undefined;
});
try {
new VaultStore(dir);
// Verify whoami was called first, then icacls with the resolved user
expect(mockExecFileSync).toHaveBeenCalledWith('whoami', expect.objectContaining({ encoding: 'utf-8' }));
expect(mockExecFileSync).toHaveBeenCalledWith(
'icacls',
expect.arrayContaining([keyPath, '/inheritance:r', '/grant:r']),
expect.objectContaining({ stdio: 'ignore' })
);
} finally {
Object.defineProperty(process, 'platform', originalPlatform);
mockExecFileSync.mockReset();
}
});
it('Windows key protection — icacls failure does not prevent vault creation', () => {
const dir = makeTempDir();
const keyPath = path.join(dir, '.vault-key');
if (fs.existsSync(keyPath)) fs.unlinkSync(keyPath);
const originalPlatform = Object.getOwnPropertyDescriptor(process, 'platform')!;
Object.defineProperty(process, 'platform', { value: 'win32', configurable: true });
// Mock: whoami succeeds but icacls throws
mockExecFileSync.mockImplementation((cmd: string) => {
if (cmd === 'whoami') return 'DOMAIN\\testuser';
throw new Error('icacls not found');
});
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
try {
const vault = new VaultStore(dir);
// Vault should still work despite icacls failure
vault.set('test', 'value');
expect(vault.get('test')!.value).toBe('value');
} finally {
Object.defineProperty(process, 'platform', originalPlatform);
mockExecFileSync.mockReset();
warnSpy.mockRestore();
}
});
});

View File

@@ -0,0 +1,21 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "ESNext",
"moduleResolution": "bundler",
"lib": ["ES2022"],
"outDir": "dist",
"rootDir": "src",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"declaration": true,
"declarationMap": true,
"sourceMap": true,
"composite": true
},
"include": ["src/**/*.ts"],
"exclude": ["node_modules", "dist", "tests"]
}

View File

@@ -0,0 +1,9 @@
import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
globals: true,
testTimeout: 30_000,
include: ['tests/**/*.test.ts'],
},
});