moving
Some checks failed
Installer Smoke / installer-smoke (push) Has been cancelled

This commit is contained in:
Oleg Maslov
2026-09-02 10:10:29 +02:00
commit 0c3e2ead3b
3841 changed files with 970576 additions and 0 deletions

View File

@@ -0,0 +1,5 @@
export * from './types.js';
export { InteractionStore } from './interaction-store.js';
export { ComplianceStatusChecker } from './status-checker.js';
export { ReportGenerator } from './report-generator.js';
export { ComplianceTemplateStore } from './template-store.js';

View File

@@ -0,0 +1,208 @@
/**
* InteractionStore — CRUD for ai_interactions table (EU AI Act Art. 12).
*
* Records every AI interaction with model, tokens, cost, tools, and human oversight.
* Operates on the .mind DB alongside FrameStore and InstallAuditStore.
*/
import type { MindDB } from '@waggle/hive-mind-core';
import type { AIInteraction, RecordInteractionInput, HumanAction, ModelInventoryEntry, OversightLogEntry } from './types.js';
export class InteractionStore {
private db: MindDB;
constructor(db: MindDB) {
this.db = db;
// Review Major #8: previously ensureTable() duplicated the DDL from schema.ts and
// drifted whenever the canonical schema changed. Now schema.ts + MindDB.runMigrations()
// own the table definition, including the input_text/output_text columns and the
// append-only triggers (Critical #1, #3).
}
/** Record an AI interaction event. */
record(input: RecordInteractionInput): AIInteraction {
const raw = this.db.getDatabase();
// Review Major #4: use lastInsertRowid from .run() instead of reading back
// ORDER BY id DESC LIMIT 1 — under concurrent writes (WaggleDance multi-agent),
// the LIMIT 1 row may belong to a different writer.
const result = raw.prepare(`
INSERT INTO ai_interactions (
workspace_id, session_id, model, provider,
input_tokens, output_tokens, cost_usd,
tools_called, human_action, risk_context, imported_from, persona,
input_text, output_text
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
input.workspaceId ?? null,
input.sessionId ?? null,
input.model,
input.provider,
input.inputTokens,
input.outputTokens,
input.costUsd,
JSON.stringify(input.toolsCalled ?? []),
input.humanAction ?? 'none',
input.riskContext ?? null,
input.importedFrom ?? null,
input.persona ?? null,
input.inputText ?? null,
input.outputText ?? null,
);
const row = raw.prepare('SELECT * FROM ai_interactions WHERE id = ?').get(result.lastInsertRowid) as Record<string, unknown>;
return this.rowToInteraction(row);
}
/** Get interactions for a workspace within a date range. */
getByWorkspace(workspaceId: string, from?: string, to?: string): AIInteraction[] {
if (from && !/^\d{4}-\d{2}-\d{2}/.test(from)) throw new Error('Invalid date format');
if (to && !/^\d{4}-\d{2}-\d{2}/.test(to)) throw new Error('Invalid date format');
const raw = this.db.getDatabase();
let sql = 'SELECT * FROM ai_interactions WHERE workspace_id = ?';
const params: unknown[] = [workspaceId];
if (from) { sql += ' AND timestamp >= ?'; params.push(from); }
if (to) { sql += ' AND timestamp <= ?'; params.push(to); }
sql += ' ORDER BY timestamp DESC LIMIT 1000';
return (raw.prepare(sql).all(...params) as Record<string, unknown>[]).map(r => this.rowToInteraction(r));
}
/** Get all interactions within a date range. */
getByDateRange(from: string, to: string, workspaceId?: string): AIInteraction[] {
if (from && !/^\d{4}-\d{2}-\d{2}/.test(from)) throw new Error('Invalid date format');
if (to && !/^\d{4}-\d{2}-\d{2}/.test(to)) throw new Error('Invalid date format');
const raw = this.db.getDatabase();
let sql = 'SELECT * FROM ai_interactions WHERE timestamp >= ? AND timestamp <= ?';
const params: unknown[] = [from, to];
if (workspaceId) { sql += ' AND workspace_id = ?'; params.push(workspaceId); }
sql += ' ORDER BY timestamp DESC LIMIT 1000';
return (raw.prepare(sql).all(...params) as Record<string, unknown>[]).map(r => this.rowToInteraction(r));
}
/** Get total interaction count. */
count(workspaceId?: string): number {
const raw = this.db.getDatabase();
if (workspaceId) {
const row = raw.prepare('SELECT COUNT(*) as cnt FROM ai_interactions WHERE workspace_id = ?').get(workspaceId) as { cnt: number };
return row.cnt;
}
const row = raw.prepare('SELECT COUNT(*) as cnt FROM ai_interactions').get() as { cnt: number };
return row.cnt;
}
/** Get the oldest log timestamp (Art. 19 retention check). */
getOldestTimestamp(): string | null {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT MIN(timestamp) as oldest FROM ai_interactions').get() as { oldest: string | null };
return row.oldest;
}
/**
* Returns the `first_run_at` timestamp from the MindDB's `meta` table — set on
* schema initialization and backfilled for pre-existing DBs. Used by the Art. 19
* retention checker to distinguish 'new system' from 'pruned logs'.
*/
getFirstRunAt(): string | null {
return this.db.getFirstRunAt();
}
/** Get model inventory (aggregated usage per model). */
getModelInventory(from?: string, to?: string, workspaceId?: string): ModelInventoryEntry[] {
const raw = this.db.getDatabase();
let sql = `
SELECT model, provider,
COUNT(*) as calls,
SUM(input_tokens) as input_tokens,
SUM(output_tokens) as output_tokens,
SUM(cost_usd) as cost_usd
FROM ai_interactions WHERE 1=1
`;
const params: unknown[] = [];
if (from) { sql += ' AND timestamp >= ?'; params.push(from); }
if (to) { sql += ' AND timestamp <= ?'; params.push(to); }
if (workspaceId) { sql += ' AND workspace_id = ?'; params.push(workspaceId); }
sql += ' GROUP BY model, provider ORDER BY calls DESC';
return (raw.prepare(sql).all(...params) as Record<string, unknown>[]).map(r => ({
model: r.model as string,
provider: r.provider as string,
calls: r.calls as number,
inputTokens: r.input_tokens as number,
outputTokens: r.output_tokens as number,
costUsd: r.cost_usd as number,
}));
}
/** Get human oversight actions (Art. 14). */
getOversightLog(from?: string, to?: string, workspaceId?: string): OversightLogEntry[] {
const raw = this.db.getDatabase();
let sql = "SELECT * FROM ai_interactions WHERE human_action != 'none' AND human_action IS NOT NULL";
const params: unknown[] = [];
if (from) { sql += ' AND timestamp >= ?'; params.push(from); }
if (to) { sql += ' AND timestamp <= ?'; params.push(to); }
if (workspaceId) { sql += ' AND workspace_id = ?'; params.push(workspaceId); }
sql += ' ORDER BY timestamp DESC';
return (raw.prepare(sql).all(...params) as Record<string, unknown>[]).map(r => ({
timestamp: r.timestamp as string,
action: r.human_action as HumanAction,
tool: (JSON.parse(r.tools_called as string) as string[])[0] ?? 'none',
detail: r.persona ? `Persona: ${r.persona}` : '',
}));
}
/** Get human oversight counts for compliance check. */
getOversightCounts(workspaceId?: string): { total: number; approved: number; denied: number; modified: number } {
const raw = this.db.getDatabase();
let sql = "SELECT human_action, COUNT(*) as cnt FROM ai_interactions WHERE human_action != 'none' AND human_action IS NOT NULL";
const params: unknown[] = [];
if (workspaceId) { sql += ' AND workspace_id = ?'; params.push(workspaceId); }
sql += ' GROUP BY human_action';
const rows = raw.prepare(sql).all(...params) as { human_action: string; cnt: number }[];
const counts = { total: 0, approved: 0, denied: 0, modified: 0 };
for (const row of rows) {
counts.total += row.cnt;
if (row.human_action === 'approved') counts.approved = row.cnt;
if (row.human_action === 'denied') counts.denied = row.cnt;
if (row.human_action === 'modified') counts.modified = row.cnt;
}
return counts;
}
/** Get recent interactions. */
getRecent(limit: number = 20): AIInteraction[] {
const raw = this.db.getDatabase();
return (raw.prepare('SELECT * FROM ai_interactions ORDER BY id DESC LIMIT ?').all(limit) as Record<string, unknown>[])
.map(r => this.rowToInteraction(r));
}
private rowToInteraction(row: Record<string, unknown>): AIInteraction {
return {
id: row.id as number,
timestamp: row.timestamp as string,
workspaceId: row.workspace_id as string | null,
sessionId: row.session_id as string | null,
model: row.model as string,
provider: row.provider as string,
inputTokens: row.input_tokens as number,
outputTokens: row.output_tokens as number,
costUsd: row.cost_usd as number,
toolsCalled: JSON.parse(row.tools_called as string) as string[],
humanAction: (row.human_action as HumanAction) ?? 'none',
riskContext: row.risk_context as string | null,
importedFrom: row.imported_from as string | null,
persona: row.persona as string | null,
inputText: (row.input_text as string | null | undefined) ?? null,
outputText: (row.output_text as string | null | undefined) ?? null,
};
}
}

View File

@@ -0,0 +1,104 @@
/**
* ReportGenerator — generates AI Act compliance audit reports.
*
* Produces JSON reports (PDF generation deferred to document tooling).
* Reports cover: compliance status, model inventory, oversight log,
* harvest provenance, and interaction counts.
*/
import type { InteractionStore } from './interaction-store.js';
import type { HarvestSourceStore } from '@waggle/hive-mind-core';
import { ComplianceStatusChecker } from './status-checker.js';
import type { AuditReport, AuditReportRequest, AIActRiskLevel } from './types.js';
const REPORT_VERSION = '1.0';
export interface ReportGeneratorDeps {
interactionStore: InteractionStore;
harvestStore: HarvestSourceStore;
getWorkspaceRisk?: (workspaceId: string) => AIActRiskLevel;
getWorkspaceName?: (workspaceId: string) => string;
/**
* Return the ISO timestamp of the last risk classification for this
* workspace, or null if the workspace has no persisted classification
* date (e.g., was created before the field existed).
*/
getWorkspaceRiskClassifiedAt?: (workspaceId: string) => string | null;
}
export class ReportGenerator {
private interactions: InteractionStore;
private harvest: HarvestSourceStore;
private getWorkspaceRisk: (id: string) => AIActRiskLevel;
private getWorkspaceName: (id: string) => string;
private getWorkspaceRiskClassifiedAt: (id: string) => string | null;
constructor(deps: ReportGeneratorDeps) {
this.interactions = deps.interactionStore;
this.harvest = deps.harvestStore;
this.getWorkspaceRisk = deps.getWorkspaceRisk ?? (() => 'minimal');
this.getWorkspaceName = deps.getWorkspaceName ?? ((id) => id);
this.getWorkspaceRiskClassifiedAt = deps.getWorkspaceRiskClassifiedAt ?? (() => null);
}
/** Generate a full audit report. */
generate(request: AuditReportRequest): AuditReport {
const checker = new ComplianceStatusChecker(this.interactions);
const complianceStatus = checker.check(request.workspaceId);
const report: AuditReport = {
report: {
version: REPORT_VERSION,
generatedAt: new Date().toISOString(),
period: { from: request.from, to: request.to },
generatedBy: 'Waggle OS',
},
workspace: request.workspaceId ? {
id: request.workspaceId,
name: this.getWorkspaceName(request.workspaceId),
riskLevel: this.getWorkspaceRisk(request.workspaceId),
riskClassifiedAt: this.getWorkspaceRiskClassifiedAt(request.workspaceId),
} : null,
complianceStatus,
modelInventory: [],
humanOversightLog: [],
harvestProvenance: [],
interactionCount: 0,
};
// Model inventory
if (request.include.models) {
report.modelInventory = this.interactions.getModelInventory(
request.from, request.to, request.workspaceId,
);
}
// Human oversight log
if (request.include.oversight) {
report.humanOversightLog = this.interactions.getOversightLog(
request.from, request.to, request.workspaceId,
);
}
// Harvest provenance
if (request.include.provenance) {
const sources = this.harvest.getAll();
report.harvestProvenance = sources.map(s => ({
source: s.displayName,
importedAt: s.lastSyncedAt ?? s.createdAt,
itemsImported: s.itemsImported,
framesCreated: s.framesCreated,
}));
}
// Interaction count
if (request.include.interactions) {
const interactions = this.interactions.getByDateRange(
request.from, request.to, request.workspaceId,
);
report.interactionCount = interactions.length;
}
return report;
}
}

View File

@@ -0,0 +1,175 @@
/**
* ComplianceStatusChecker — evaluates AI Act compliance per workspace.
*
* Checks:
* - Art. 12: Automatic event logging (ai_interactions count)
* - Art. 14: Human oversight (approval/denial actions recorded)
* - Art. 19: Log retention (oldest log >= 6 months ago)
* - Art. 26: Deployer monitoring (active monitors)
* - Art. 50: Model transparency (models disclosed)
*/
import type { InteractionStore } from './interaction-store.js';
import type { ComplianceStatus, ArticleStatus } from './types.js';
const SIX_MONTHS_MS = 180 * 24 * 60 * 60 * 1000;
export class ComplianceStatusChecker {
private store: InteractionStore;
constructor(store: InteractionStore) {
this.store = store;
}
/** Evaluate full compliance status for a workspace (or all workspaces if null). */
check(workspaceId?: string): ComplianceStatus {
const art12 = this.checkArt12(workspaceId);
const art14 = this.checkArt14(workspaceId);
const art19 = this.checkArt19();
const art26 = this.checkArt26();
const art50 = this.checkArt50(workspaceId);
const statuses = [art12.status, art14.status, art19.status, art26.status, art50.status];
const overall = statuses.includes('non-compliant') ? 'non-compliant'
: statuses.includes('warning') ? 'warning'
: 'compliant';
return {
overall,
art12Logging: art12,
art14Oversight: art14,
art19Retention: art19,
art26Monitoring: art26,
art50Transparency: art50,
};
}
/** Art. 12: Automatic recording of events. */
private checkArt12(workspaceId?: string): ArticleStatus & { totalInteractions: number } {
const total = this.store.count(workspaceId);
if (total === 0) {
// M11: if the DB has been active for >24h with zero interactions, escalate to non-compliant
const firstRun = this.store.getFirstRunAt();
const activeOver24h = firstRun
? (Date.now() - new Date(firstRun).getTime()) > 24 * 60 * 60 * 1000
: false;
return {
status: activeOver24h ? 'non-compliant' : 'warning',
detail: activeOver24h
? 'No interactions logged despite system being active for over 24 hours. Verify logging pipeline.'
: 'No interactions logged yet. Logging activates automatically on first AI interaction.',
totalInteractions: 0,
};
}
return {
status: 'compliant',
detail: `${total} interactions logged with full model, token, cost, and tool tracking.`,
totalInteractions: total,
};
}
/** Art. 14: Human oversight capability. */
private checkArt14(workspaceId?: string): ArticleStatus & { humanActions: number; approvalRate: number } {
const counts = this.store.getOversightCounts(workspaceId);
if (counts.total === 0) {
return {
status: 'compliant',
detail: 'Human oversight capabilities available (approval gates, tool deny lists). No oversight actions recorded yet.',
humanActions: 0,
approvalRate: 0,
};
}
const approvalRate = counts.total > 0
? Math.round((counts.approved / counts.total) * 100)
: 0;
return {
status: 'compliant',
detail: `${counts.total} human oversight actions: ${counts.approved} approved, ${counts.denied} denied, ${counts.modified} modified.`,
humanActions: counts.total,
approvalRate,
};
}
/** Art. 19: Log retention (minimum 6 months). */
private checkArt19(): ArticleStatus & { oldestLogDate: string | null; retentionDays: number } {
const oldest = this.store.getOldestTimestamp();
if (!oldest) {
return {
status: 'compliant',
detail: 'No logs to retain yet. Retention policy is permanent by default.',
oldestLogDate: null,
retentionDays: 0,
};
}
const oldestDate = new Date(oldest);
const now = new Date();
const retentionMs = now.getTime() - oldestDate.getTime();
const retentionDays = Math.floor(retentionMs / (24 * 60 * 60 * 1000));
// Review Critical #2: the previous expression was a tautology
// (`retentionMs >= SIX_MONTHS_MS || retentionDays < 180`) that covered every
// non-negative value of retentionDays. A deployment that pruned logs after 30
// days still reported compliant.
//
// Proper fix requires distinguishing 'system is young' from 'logs were pruned'.
// We track system age via MindDB's `meta.first_run_at` entry (set on schema init,
// backfilled for pre-existing DBs). If the system has been running for 180+ days
// but the oldest log is younger than that, something pruned the logs and we
// correctly report warning.
const firstRun = this.store.getFirstRunAt();
const systemAgeMs = firstRun ? now.getTime() - new Date(firstRun).getTime() : retentionMs;
const hasBeenRunning6Months = systemAgeMs >= SIX_MONTHS_MS;
const logsOlderThan6Months = retentionMs >= SIX_MONTHS_MS;
const meetsMinimum = !hasBeenRunning6Months || logsOlderThan6Months;
return {
status: meetsMinimum ? 'compliant' : 'warning',
detail: meetsMinimum
? hasBeenRunning6Months
? `Logs retained since ${oldest.split('T')[0]} (${retentionDays} days). Art. 19 minimum (180 days) met.`
: `Logs retained since ${oldest.split('T')[0]} (${retentionDays} days). System is still within its first 180 days — retention compliance will be enforceable after 2026-${(new Date(firstRun ?? now).getMonth() + 7).toString().padStart(2, '0')}.`
: `Oldest log: ${oldest.split('T')[0]} (${retentionDays} days) but system is ${Math.floor(systemAgeMs / (24 * 60 * 60 * 1000))} days old. Logs appear to have been pruned — EU AI Act Art. 19 requires 180-day minimum retention.`,
oldestLogDate: oldest,
retentionDays,
};
}
/** Art. 26: Deployer monitoring obligations. */
private checkArt26(): ArticleStatus & { activeMonitors: string[] } {
// Waggle always has these monitors active
const monitors = [
'cost_tracking', // CostTracker in packages/agent
'tool_logging', // Tool calls logged per interaction
'model_identification', // Model recorded per interaction
'persona_tracking', // Persona recorded per interaction
];
return {
status: 'compliant',
detail: `${monitors.length} active monitors: cost, tools, model ID, persona.`,
activeMonitors: monitors,
};
}
/** Art. 50: Transparency — models disclosed. */
private checkArt50(workspaceId?: string): ArticleStatus & { modelsDisclosed: boolean } {
const inventory = this.store.getModelInventory(undefined, undefined, workspaceId);
const modelsDisclosed = inventory.length > 0;
return {
status: 'compliant',
detail: modelsDisclosed
? `${inventory.length} model(s) in use, all identified in StatusBar and interaction logs.`
: 'Model identification active. Models will be disclosed on first interaction.',
modelsDisclosed,
};
}
}

View File

@@ -0,0 +1,242 @@
/**
* ComplianceTemplateStore — CRUD for user-editable compliance report templates.
*
* M-03: Templates let a user save a named "report shape" (which sections to include,
* a risk-class override, and optional org/footer text) and re-apply it to subsequent
* audit-report exports. Sections MERGE with the runtime selection (union semantics).
*
* Logo support is deferred to Bucket 2. See docs/plans/COMPLIANCE-AUDIT-2026-04-20.md.
*
* DDL is self-contained here (following HarvestRunStore pattern) rather than threaded
* through mind/schema.ts — compliance_templates is not part of the Art. 12 audit
* substrate and doesn't need append-only triggers or cross-layer migration tracking.
*/
import type { MindDB } from '@waggle/hive-mind-core';
import type {
AIActRiskLevel,
ComplianceTemplate,
ComplianceTemplateSections,
CreateComplianceTemplateInput,
UpdateComplianceTemplateInput,
} from './types.js';
const CREATE_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS compliance_templates (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
description TEXT,
sections_json TEXT NOT NULL DEFAULT '{}',
risk_classification TEXT CHECK (risk_classification IN ('minimal', 'limited', 'high-risk', 'unacceptable')),
org_name TEXT,
footer_text TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
)
`;
// TODO Bucket 2 (M-05 custom branding): add `logo_blob BLOB` + `logo_mime TEXT`
// columns. The `Edit templates` modal will grow a file-picker; renderComplianceReportPdf
// will render the blob into the PDF header when present.
const CREATE_INDEX_SQL = `
CREATE INDEX IF NOT EXISTS idx_compliance_templates_name ON compliance_templates (name)
`;
const DEFAULT_SECTIONS: ComplianceTemplateSections = {
interactions: true,
oversight: true,
models: true,
provenance: true,
riskAssessment: true,
fria: false,
};
/**
* Canonical name for the seeded KVARK template. Used for idempotency — the
* seed only fires when no row with this exact name exists.
*/
export const KVARK_TEMPLATE_NAME = 'KVARK Enterprise Audit';
export class ComplianceTemplateStore {
private db: MindDB;
constructor(db: MindDB) {
this.db = db;
this.ensureTable();
}
private ensureTable(): void {
const raw = this.db.getDatabase();
const existsRow = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='compliance_templates'",
).get();
if (existsRow) return;
raw.prepare(CREATE_TABLE_SQL).run();
raw.prepare(CREATE_INDEX_SQL).run();
}
create(input: CreateComplianceTemplateInput): ComplianceTemplate {
const name = input.name.trim();
if (!name) throw new Error('Template name is required');
const raw = this.db.getDatabase();
const result = raw.prepare(`
INSERT INTO compliance_templates (
name, description, sections_json, risk_classification, org_name, footer_text
) VALUES (?, ?, ?, ?, ?, ?)
`).run(
name,
input.description ?? null,
JSON.stringify(this.normalizeSections(input.sections)),
input.riskClassification ?? null,
input.orgName ?? null,
input.footerText ?? null,
);
return this.getById(Number(result.lastInsertRowid))!;
}
getById(id: number): ComplianceTemplate | null {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT * FROM compliance_templates WHERE id = ?').get(id) as
| Record<string, unknown>
| undefined;
return row ? this.rowToTemplate(row) : null;
}
list(): ComplianceTemplate[] {
const raw = this.db.getDatabase();
return (
raw
.prepare('SELECT * FROM compliance_templates ORDER BY updated_at DESC')
.all() as Record<string, unknown>[]
).map(r => this.rowToTemplate(r));
}
update(id: number, patch: UpdateComplianceTemplateInput): ComplianceTemplate | null {
const existing = this.getById(id);
if (!existing) return null;
const next: ComplianceTemplate = {
...existing,
name: patch.name !== undefined ? patch.name.trim() : existing.name,
description: patch.description !== undefined ? patch.description : existing.description,
sections: patch.sections !== undefined ? this.normalizeSections(patch.sections) : existing.sections,
riskClassification:
patch.riskClassification !== undefined ? patch.riskClassification : existing.riskClassification,
orgName: patch.orgName !== undefined ? patch.orgName : existing.orgName,
footerText: patch.footerText !== undefined ? patch.footerText : existing.footerText,
};
if (!next.name) throw new Error('Template name is required');
const raw = this.db.getDatabase();
raw.prepare(`
UPDATE compliance_templates SET
name = ?,
description = ?,
sections_json = ?,
risk_classification = ?,
org_name = ?,
footer_text = ?,
updated_at = datetime('now')
WHERE id = ?
`).run(
next.name,
next.description,
JSON.stringify(next.sections),
next.riskClassification,
next.orgName,
next.footerText,
id,
);
return this.getById(id);
}
/** Returns true if the row existed and was removed, false otherwise. */
delete(id: number): boolean {
const raw = this.db.getDatabase();
const info = raw.prepare('DELETE FROM compliance_templates WHERE id = ?').run(id);
return info.changes > 0;
}
/**
* Merge a template's section flags with runtime flags. Union semantics: a section
* is included if EITHER the template OR the runtime toggle turns it on. This keeps
* the template's role "additive" — it can never silently hide a section the user
* asked for in the current export.
*/
static mergeSections(
template: ComplianceTemplateSections,
runtime: ComplianceTemplateSections,
): ComplianceTemplateSections {
return {
interactions: template.interactions || runtime.interactions,
oversight: template.oversight || runtime.oversight,
models: template.models || runtime.models,
provenance: template.provenance || runtime.provenance,
riskAssessment: template.riskAssessment || runtime.riskAssessment,
fria: template.fria || runtime.fria,
};
}
private normalizeSections(s: Partial<ComplianceTemplateSections> | undefined): ComplianceTemplateSections {
return { ...DEFAULT_SECTIONS, ...(s ?? {}) };
}
/**
* Idempotent: seed the built-in "KVARK Enterprise Audit" template if no
* template with that exact name exists yet (M-06).
*
* The KVARK template is a sensible-default shape for sovereign enterprise
* deployments — every section on (including FRIA, since KVARK serves
* high-risk enterprise systems), risk pinned at `high-risk`, and org/footer
* text that signals the sovereign-deployment narrative.
*
* Custom KVARK-specific sections (per-department risk breakdown, IAM
* audit columns, data-residency attestation text) are deferred to
* Bucket 2 along with the logo field — the current template schema
* doesn't support custom sections yet.
*
* Returns the seeded template, or null if one already existed.
*/
seedKvarkTemplateIfMissing(): ComplianceTemplate | null {
const raw = this.db.getDatabase();
const existing = raw.prepare(
'SELECT id FROM compliance_templates WHERE name = ? LIMIT 1',
).get(KVARK_TEMPLATE_NAME) as { id: number } | undefined;
if (existing) return null;
return this.create({
name: KVARK_TEMPLATE_NAME,
description:
'Sovereign AI Act audit shape for enterprise on-prem deployments. ' +
'Includes all monitored articles (12/14/19/26/50) plus FRIA since KVARK ' +
'customers typically operate high-risk systems. Data never leaves the ' +
"customer perimeter; this template's org + footer text signal that " +
'posture directly on every exported report.',
sections: { ...DEFAULT_SECTIONS, fria: true },
riskClassification: 'high-risk',
orgName: 'KVARK Sovereign — Enterprise Deployment',
footerText:
'Confidential · EU AI Act attestation · KVARK sovereign deployment · Data remains within customer perimeter',
});
}
private rowToTemplate(row: Record<string, unknown>): ComplianceTemplate {
let sections: ComplianceTemplateSections;
try {
sections = this.normalizeSections(JSON.parse((row.sections_json as string) ?? '{}'));
} catch {
sections = { ...DEFAULT_SECTIONS };
}
return {
id: row.id as number,
name: row.name as string,
description: (row.description as string | null) ?? null,
sections,
riskClassification: (row.risk_classification as AIActRiskLevel | null) ?? null,
orgName: (row.org_name as string | null) ?? null,
footerText: (row.footer_text as string | null) ?? null,
createdAt: row.created_at as string,
updatedAt: row.updated_at as string,
};
}
}

View File

@@ -0,0 +1,201 @@
/**
* AI Act Compliance Types — interaction audit, risk classification, report format.
*/
// ── Risk Classification (Art. 26) ──
export type AIActRiskLevel = 'minimal' | 'limited' | 'high-risk' | 'unacceptable';
export type HumanAction = 'approved' | 'denied' | 'modified' | 'none';
// ── Interaction Audit (Art. 12) ──
export interface AIInteraction {
id: number;
timestamp: string;
workspaceId: string | null;
sessionId: string | null;
model: string;
provider: string;
inputTokens: number;
outputTokens: number;
costUsd: number;
toolsCalled: string[];
humanAction: HumanAction;
riskContext: string | null;
importedFrom: string | null;
persona: string | null;
// Review Critical #3 (compliance): EU AI Act Art. 12.1(a) requires recording the
// actual inputs and outputs of the system, not just token counts. Nullable because
// pre-existing DBs may have rows from before the columns were added.
inputText: string | null;
outputText: string | null;
}
export interface RecordInteractionInput {
workspaceId?: string;
sessionId?: string;
model: string;
provider: string;
inputTokens: number;
outputTokens: number;
costUsd: number;
toolsCalled?: string[];
humanAction?: HumanAction;
riskContext?: string;
importedFrom?: string;
persona?: string;
// Review Critical #3: optional because echo-mode interactions may not have meaningful
// content. Callers on the live agent path SHOULD pass these; record() does not reject
// absent values but the compliance status checker flags them as a gap.
inputText?: string;
outputText?: string;
}
// ── Compliance Status ──
export interface ArticleStatus {
status: 'compliant' | 'warning' | 'non-compliant';
detail: string;
}
export interface ComplianceStatus {
overall: 'compliant' | 'warning' | 'non-compliant';
art12Logging: ArticleStatus & { totalInteractions: number };
art14Oversight: ArticleStatus & { humanActions: number; approvalRate: number };
art19Retention: ArticleStatus & { oldestLogDate: string | null; retentionDays: number };
art26Monitoring: ArticleStatus & { activeMonitors: string[] };
art50Transparency: ArticleStatus & { modelsDisclosed: boolean };
}
// ── Audit Report ──
export interface ModelInventoryEntry {
model: string;
provider: string;
calls: number;
inputTokens: number;
outputTokens: number;
costUsd: number;
}
export interface OversightLogEntry {
timestamp: string;
action: HumanAction;
tool: string;
detail: string;
user?: string;
}
export interface HarvestProvenanceEntry {
source: string;
importedAt: string;
itemsImported: number;
framesCreated: number;
}
export interface AuditReportRequest {
workspaceId?: string;
from: string;
to: string;
format: 'json' | 'pdf' | 'both';
include: {
interactions: boolean;
oversight: boolean;
models: boolean;
provenance: boolean;
riskAssessment: boolean;
fria: boolean;
};
}
export interface AuditReport {
report: {
version: string;
generatedAt: string;
period: { from: string; to: string };
generatedBy: string;
};
workspace: {
id: string | null;
name: string;
riskLevel: AIActRiskLevel;
riskClassifiedAt: string | null;
} | null;
complianceStatus: ComplianceStatus;
modelInventory: ModelInventoryEntry[];
humanOversightLog: OversightLogEntry[];
harvestProvenance: HarvestProvenanceEntry[];
interactionCount: number;
}
// ── Compliance Templates (M-03) ──
//
// Templates let a user save a preferred audit-report shape (sections + risk class +
// org/footer overrides) and re-apply it to exports. Sections MERGE (union) with the
// user's runtime section toggles — a template only turns sections ON, never off.
// `riskClassification` overrides the workspace's risk level for that one report.
// Logo support is deferred to Bucket 2 (post-launch).
export interface ComplianceTemplateSections {
interactions: boolean;
oversight: boolean;
models: boolean;
provenance: boolean;
riskAssessment: boolean;
fria: boolean;
}
export interface ComplianceTemplate {
id: number;
name: string;
description: string | null;
sections: ComplianceTemplateSections;
riskClassification: AIActRiskLevel | null;
orgName: string | null;
footerText: string | null;
createdAt: string;
updatedAt: string;
}
export interface CreateComplianceTemplateInput {
name: string;
description?: string | null;
sections: ComplianceTemplateSections;
riskClassification?: AIActRiskLevel | null;
orgName?: string | null;
footerText?: string | null;
}
export interface UpdateComplianceTemplateInput {
name?: string;
description?: string | null;
sections?: ComplianceTemplateSections;
riskClassification?: AIActRiskLevel | null;
orgName?: string | null;
footerText?: string | null;
}
// ── Template auto-suggestion ──
export const TEMPLATE_RISK_MAP: Record<string, AIActRiskLevel> = {
'legal-review': 'high-risk',
'hr-management': 'high-risk',
'recruiting': 'high-risk',
'finance': 'high-risk',
'insurance': 'high-risk',
'credit-scoring': 'high-risk',
'sales-pipeline': 'limited',
'marketing-campaign': 'limited',
'research-project': 'minimal',
'code-review': 'minimal',
'product-launch': 'limited',
'agency-consulting': 'limited',
'blank': 'minimal',
'content-creation': 'minimal',
'customer-support': 'limited',
'data-analysis': 'limited',
'project-management': 'minimal',
'education': 'limited',
'healthcare': 'high-risk',
};

281
packages/core/src/config.ts Normal file
View File

@@ -0,0 +1,281 @@
import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';
import type { EmbeddingProviderConfig, EmbeddingProviderType } from '@waggle/hive-mind-core';
export interface ProviderEntry {
apiKey: string;
models: string[];
baseUrl?: string;
}
export interface TeamServerConfig {
url: string;
token?: string;
userId?: string;
displayName?: string;
}
export interface CliConfig {
allowlist?: string[];
}
interface ConfigData {
defaultModel: string;
providers: Record<string, ProviderEntry>;
mindPath?: string;
teamServer?: TeamServerConfig;
/** Governed CLI programs the agent may execute. */
cli?: CliConfig;
/** F8: Daily cost budget in dollars. null = no limit. */
dailyBudget?: number | null;
/** When true, exceeding dailyBudget blocks agent. When false, warns only. */
budgetHardCap?: boolean;
/** Model Pilot: fallback model when primary fails (429/500/timeout) */
fallbackModel?: string;
/** Model Pilot: budget-saver model when daily spend hits threshold */
budgetModel?: string;
/** Model Pilot: budget threshold as 0.0-1.0 fraction. Default 0.8 */
budgetThreshold?: number;
/** Agent Intelligence: max LLM iterations per conversation. Default 90. */
maxIterations?: number;
/** M2-7: Telemetry opt-in (default: false — privacy first) */
telemetryEnabled?: boolean;
/** M2-1: Embedding provider configuration */
embedding?: {
provider?: EmbeddingProviderType | 'auto';
ollamaUrl?: string;
ollamaModel?: string;
inprocessModel?: string;
};
/** Steal #6: on-demand relevance gating for MCP tools. */
mcpToolRetrieval?: {
enabled?: boolean;
threshold?: number;
topK?: number;
};
}
/** Resolved MCP tool-retrieval config (all fields present). */
export interface McpToolRetrievalSettings {
enabled: boolean;
threshold: number;
topK: number;
}
const DEFAULT_MODEL = 'claude-sonnet-4-6';
function getDefaultConfigDir(): string {
const home = process.env.HOME || process.env.USERPROFILE || os.homedir();
return path.join(home, '.waggle');
}
export class WaggleConfig {
private readonly configDir: string;
private readonly configPath: string;
private data: ConfigData;
constructor(configDir?: string) {
this.configDir = configDir ?? getDefaultConfigDir();
this.configPath = path.join(this.configDir, 'config.json');
// Ensure config directory exists
if (!fs.existsSync(this.configDir)) {
fs.mkdirSync(this.configDir, { recursive: true });
}
// Load existing config or use defaults
this.data = this.load();
}
private load(): ConfigData {
if (fs.existsSync(this.configPath)) {
const raw = fs.readFileSync(this.configPath, 'utf-8');
return JSON.parse(raw) as ConfigData;
}
return {
defaultModel: DEFAULT_MODEL,
providers: {},
};
}
save(): void {
fs.writeFileSync(this.configPath, JSON.stringify(this.data, null, 2), 'utf-8');
}
getDefaultModel(): string {
return this.data.defaultModel;
}
setDefaultModel(model: string): void {
this.data.defaultModel = model;
}
getProviders(): Record<string, ProviderEntry> {
return { ...(this.data.providers ?? {}) };
}
setProvider(name: string, entry: ProviderEntry): void {
if (!this.data.providers) this.data.providers = {};
this.data.providers[name] = entry;
}
removeProvider(name: string): void {
delete this.data.providers[name];
}
getMindPath(): string {
return this.data.mindPath ?? path.join(this.configDir, 'default.mind');
}
getConfigDir(): string {
return this.configDir;
}
// F8: Daily cost budget
getDailyBudget(): number | null {
return this.data.dailyBudget ?? null;
}
setDailyBudget(budget: number | null): void {
this.data.dailyBudget = budget;
}
getBudgetHardCap(): boolean {
return this.data.budgetHardCap ?? false;
}
setBudgetHardCap(enabled: boolean): void {
this.data.budgetHardCap = enabled;
}
// --- Model Pilot ---
getFallbackModel(): string | null {
return this.data.fallbackModel ?? null;
}
setFallbackModel(model: string): void {
this.data.fallbackModel = model;
}
clearFallbackModel(): void {
delete this.data.fallbackModel;
}
getBudgetModel(): string | null {
return this.data.budgetModel ?? null;
}
setBudgetModel(model: string): void {
this.data.budgetModel = model;
}
clearBudgetModel(): void {
delete this.data.budgetModel;
}
getBudgetThreshold(): number {
return this.data.budgetThreshold ?? 0.8;
}
setBudgetThreshold(threshold: number): void {
this.data.budgetThreshold = Math.max(0.5, Math.min(0.95, threshold));
}
// --- Agent Intelligence ---
getMaxIterations(): number {
return this.data.maxIterations ?? 90;
}
setMaxIterations(max: number): void {
this.data.maxIterations = Math.max(5, Math.min(500, max));
}
// --- Team Server (Phase 5) ---
getTeamServer(): TeamServerConfig | null {
return this.data.teamServer ?? null;
}
setTeamServer(config: TeamServerConfig): void {
this.data.teamServer = config;
}
clearTeamServer(): void {
delete this.data.teamServer;
}
isTeamConnected(): boolean {
return this.data.teamServer !== null && this.data.teamServer !== undefined && typeof this.data.teamServer.url === 'string' && this.data.teamServer.url.length > 0;
}
// --- Governed CLI access ---
getCliAllowlist(): string[] {
return [...(this.data.cli?.allowlist ?? [])];
}
setCliAllowlist(allowlist: string[]): void {
const seen = new Set<string>();
const next = allowlist.reduce<string[]>((result, entry) => {
const value = entry.trim();
const key = value.toLowerCase();
if (value && !seen.has(key)) {
seen.add(key);
result.push(value);
}
return result;
}, []);
this.data.cli = { ...(this.data.cli ?? {}), allowlist: next };
}
// --- Telemetry (M2-7) ---
getTelemetryEnabled(): boolean {
return this.data.telemetryEnabled ?? false;
}
setTelemetryEnabled(enabled: boolean): void {
this.data.telemetryEnabled = enabled;
this.save();
}
// --- Embedding Provider (M2-1) ---
getEmbeddingConfig(): EmbeddingProviderConfig {
const emb = this.data.embedding;
const config: EmbeddingProviderConfig = {
provider: (process.env.EMBEDDING_PROVIDER as EmbeddingProviderType | 'auto' | undefined) ?? emb?.provider ?? 'auto',
targetDimensions: 1024,
inprocess: {
model: process.env.EMBEDDING_MODEL ?? emb?.inprocessModel,
cacheDir: path.join(this.configDir, 'models'),
},
ollama: {
baseUrl: process.env.OLLAMA_HOST ?? emb?.ollamaUrl,
model: process.env.OLLAMA_EMBED_MODEL ?? emb?.ollamaModel,
},
// API keys injected separately from Vault — not stored in config.json
};
return config;
}
setEmbeddingProvider(provider: EmbeddingProviderType | 'auto'): void {
if (!this.data.embedding) this.data.embedding = {};
this.data.embedding.provider = provider;
}
// --- MCP tool retrieval (Steal #6) ---
/** Resolve MCP tool-retrieval settings, filling defaults (ON, threshold 20, top-k 10). */
getMcpToolRetrieval(): McpToolRetrievalSettings {
const cfg = this.data.mcpToolRetrieval;
return {
enabled: cfg?.enabled ?? true,
threshold: cfg?.threshold ?? 20,
topK: cfg?.topK ?? 10,
};
}
}

View File

@@ -0,0 +1,588 @@
/**
* Cron Store — SQLite persistence for solo cron schedules.
*
* Stores cron job definitions with expression validation via cron-parser,
* due-job queries, and run tracking. Follows the same pattern as
* InstallAuditStore — operates on the .mind DB with lazy table creation.
*/
import cronParser from 'cron-parser';
const { parseExpression } = cronParser;
import type { MindDB } from '@waggle/hive-mind-core';
// ── Types ──────────────────────────────────────────────────────────────
export type CronJobType = 'agent_task' | 'memory_consolidation' | 'workspace_health' | 'proactive' | 'prompt_optimization' | 'monthly_assessment' | 'connector_fetch' | 'loop';
export const VALID_JOB_TYPES: Set<string> = new Set([
'agent_task',
'memory_consolidation',
'workspace_health',
'proactive',
'prompt_optimization',
'monthly_assessment',
'connector_fetch',
// Loop v0: a stateful, memory-powered, report-only (L1) scheduled automation.
// Composes recall + maker (toolless LLM) + checker (LLMJudge) + memory write.
// job_type TEXT has no CHECK constraint, so this is additive — no migration.
'loop',
]);
export interface CronSchedule {
id: number;
name: string;
cron_expr: string;
job_type: CronJobType;
job_config: string;
workspace_id: string | null;
enabled: number; // SQLite integer boolean
last_run_at: string | null;
next_run_at: string | null;
created_at: string;
}
export interface CreateScheduleInput {
name: string;
cronExpr: string;
jobType: CronJobType;
jobConfig?: Record<string, unknown>;
workspaceId?: string;
enabled?: boolean;
}
export interface CronExecutionRow {
id: number;
schedule_id: number;
schedule_name: string;
executed_at: string;
duration_ms: number | null;
success: number; // SQLite integer boolean
result_summary: string | null;
error: string | null;
}
export interface CronRunLeaseRow {
id: number;
schedule_id: number;
schedule_name: string | null;
started_at: string;
pid: number | null;
}
export interface NotificationRow {
id: number;
title: string;
body: string;
category: string;
action_url: string | null;
read: number; // SQLite integer boolean
created_at: string;
}
// ── L2 assisted loops: durable "held action" approval queue ──────────────
// A held action is a self-contained proposed tool call drafted by a headless
// run (e.g. an assist-mode Loop) that needs a human's one-click approval before
// it executes. Durable so it survives a sidecar restart (the in-memory live
// approval Promise does not). Execute-on-approve, never mid-run suspend/resume.
export type PendingActionStatus = 'held' | 'approved' | 'denied' | 'executed' | 'failed' | 'expired';
export interface PendingActionRow {
id: string; // requestId (uuid)
workspace_id: string | null;
source: string; // e.g. 'loop:<scheduleId>'
tool_name: string;
args_json: string;
summary: string | null; // the maker's plain-language rationale
risk_level: string; // low | medium | high | critical
approval_class: string;
status: PendingActionStatus;
result_summary: string | null;
error: string | null;
created_at: string;
decided_at: string | null;
executed_at: string | null;
expires_at: string | null;
}
export interface SavePendingActionInput {
id: string;
workspaceId: string | null;
source: string;
toolName: string;
argsJson: string;
summary?: string;
riskLevel: string;
approvalClass: string;
expiresAt?: string;
}
// ── Table DDL ──────────────────────────────────────────────────────────
export const CRON_SCHEDULES_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS cron_schedules (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
cron_expr TEXT NOT NULL,
job_type TEXT NOT NULL,
job_config TEXT NOT NULL DEFAULT '{}',
workspace_id TEXT,
enabled INTEGER NOT NULL DEFAULT 1,
last_run_at TEXT,
next_run_at TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_cron_enabled_next ON cron_schedules (enabled, next_run_at);
`;
// W5.12: Cron execution history table
export const CRON_HISTORY_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS cron_execution_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
schedule_id INTEGER NOT NULL,
schedule_name TEXT NOT NULL,
executed_at TEXT NOT NULL DEFAULT (datetime('now')),
duration_ms INTEGER,
success INTEGER NOT NULL DEFAULT 1,
result_summary TEXT,
error TEXT,
FOREIGN KEY (schedule_id) REFERENCES cron_schedules(id)
);
CREATE INDEX IF NOT EXISTS idx_cron_history_schedule ON cron_execution_history (schedule_id, executed_at);
`;
export const CRON_RUN_LEASES_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS cron_run_leases (
id INTEGER PRIMARY KEY AUTOINCREMENT,
schedule_id INTEGER NOT NULL,
schedule_name TEXT,
started_at TEXT NOT NULL DEFAULT (datetime('now')),
pid INTEGER
);
`;
// W5.10: Notification persistence table
export const NOTIFICATIONS_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS notifications (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT NOT NULL,
body TEXT NOT NULL,
category TEXT NOT NULL DEFAULT 'system',
action_url TEXT,
read INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_notifications_created ON notifications (created_at);
`;
// L2: durable held-action approval queue
export const PENDING_ACTIONS_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS pending_actions (
id TEXT PRIMARY KEY,
workspace_id TEXT,
source TEXT NOT NULL,
tool_name TEXT NOT NULL,
args_json TEXT NOT NULL,
summary TEXT,
risk_level TEXT NOT NULL,
approval_class TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'held',
result_summary TEXT,
error TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
decided_at TEXT,
executed_at TEXT,
expires_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_pending_actions_status ON pending_actions (status, created_at);
`;
// ── Helpers ────────────────────────────────────────────────────────────
/** Parse a cron expression and return the next run time as ISO string. Throws on invalid expr. */
function computeNextRun(cronExpr: string): string {
const interval = parseExpression(cronExpr);
return interval.next().toISOString();
}
/**
* Validate a cron expression with the SAME parser create()/update() use.
* Returns null when parseable, else the parser's error message. Lets route
* layers (e.g. the automations /test preview) reject an expression the store
* would refuse to persist, without duplicating the parser dependency.
*/
export function cronExprError(cronExpr: string): string | null {
try {
parseExpression(cronExpr);
return null;
} catch (err) {
return err instanceof Error ? err.message : String(err);
}
}
// ── Store ──────────────────────────────────────────────────────────────
export class CronStore {
private db: MindDB;
constructor(db: MindDB) {
this.db = db;
this.ensureTable();
}
private ensureTable(): void {
const raw = this.db.getDatabase();
const exists = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='cron_schedules'",
).get();
if (!exists) {
raw.exec(CRON_SCHEDULES_TABLE_SQL);
}
// W5.12: Ensure cron execution history table
const histExists = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='cron_execution_history'",
).get();
if (!histExists) {
raw.exec(CRON_HISTORY_TABLE_SQL);
}
const leaseExists = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='cron_run_leases'",
).get();
if (!leaseExists) {
raw.exec(CRON_RUN_LEASES_TABLE_SQL);
}
// W5.10: Ensure notifications table
const notifExists = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='notifications'",
).get();
if (!notifExists) {
raw.exec(NOTIFICATIONS_TABLE_SQL);
}
// L2: Ensure pending_actions (held-action approval queue) table
const pendingExists = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='pending_actions'",
).get();
if (!pendingExists) {
raw.exec(PENDING_ACTIONS_TABLE_SQL);
}
}
/** Create a new cron schedule. Validates cron expression and job type. */
create(input: CreateScheduleInput): CronSchedule {
// Validate job type
if (!VALID_JOB_TYPES.has(input.jobType)) {
throw new Error(`Invalid job type: "${input.jobType}". Must be one of: ${[...VALID_JOB_TYPES].join(', ')}`);
}
// agent_task requires workspaceId
if (input.jobType === 'agent_task' && !input.workspaceId) {
throw new Error('agent_task jobs require a workspace ID');
}
// Validate cron expression (throws on invalid)
const nextRun = computeNextRun(input.cronExpr);
const raw = this.db.getDatabase();
const enabled = input.enabled === false ? 0 : 1;
const result = raw.prepare(`
INSERT INTO cron_schedules (name, cron_expr, job_type, job_config, workspace_id, enabled, next_run_at)
VALUES (?, ?, ?, ?, ?, ?, ?)
`).run(
input.name,
input.cronExpr,
input.jobType,
JSON.stringify(input.jobConfig ?? {}),
input.workspaceId ?? null,
enabled,
nextRun,
);
return raw.prepare(
'SELECT * FROM cron_schedules WHERE id = ?',
).get(result.lastInsertRowid) as CronSchedule;
}
/** List all schedules ordered by name. */
list(): CronSchedule[] {
return this.db.getDatabase().prepare(
'SELECT * FROM cron_schedules ORDER BY name ASC',
).all() as CronSchedule[];
}
/** Get a schedule by ID. Returns undefined if not found. */
getById(id: number): CronSchedule | undefined {
return this.db.getDatabase().prepare(
'SELECT * FROM cron_schedules WHERE id = ?',
).get(id) as CronSchedule | undefined;
}
/** Update a schedule. Recomputes next_run_at if cronExpr changes. Returns updated schedule. */
update(id: number, changes: Partial<Pick<CreateScheduleInput, 'name' | 'cronExpr' | 'jobConfig' | 'workspaceId' | 'enabled'>>): CronSchedule {
const setClauses: string[] = [];
const values: unknown[] = [];
if (changes.name !== undefined) {
setClauses.push('name = ?');
values.push(changes.name);
}
if (changes.cronExpr !== undefined) {
const nextRun = computeNextRun(changes.cronExpr);
setClauses.push('cron_expr = ?');
values.push(changes.cronExpr);
setClauses.push('next_run_at = ?');
values.push(nextRun);
}
if (changes.jobConfig !== undefined) {
setClauses.push('job_config = ?');
values.push(JSON.stringify(changes.jobConfig));
}
if (changes.workspaceId !== undefined) {
setClauses.push('workspace_id = ?');
values.push(changes.workspaceId);
}
if (changes.enabled !== undefined) {
setClauses.push('enabled = ?');
values.push(changes.enabled ? 1 : 0);
}
if (setClauses.length > 0) {
values.push(id);
this.db.getDatabase().prepare(
`UPDATE cron_schedules SET ${setClauses.join(', ')} WHERE id = ?`,
).run(...values);
}
return this.getById(id)!;
}
/** Delete a schedule by ID. */
delete(id: number): void {
this.db.getDatabase().prepare(
'DELETE FROM cron_schedules WHERE id = ?',
).run(id);
}
/** Get all enabled schedules whose next_run_at is in the past. */
getDue(): CronSchedule[] {
return this.db.getDatabase().prepare(
"SELECT * FROM cron_schedules WHERE enabled = 1 AND next_run_at <= datetime('now')",
).all() as CronSchedule[];
}
/** Mark a schedule as having just run. Updates last_run_at and recomputes next_run_at. */
markRun(id: number): void {
const schedule = this.getById(id);
if (!schedule) return;
const nextRun = computeNextRun(schedule.cron_expr);
this.db.getDatabase().prepare(
"UPDATE cron_schedules SET last_run_at = datetime('now'), next_run_at = ? WHERE id = ?",
).run(nextRun, id);
}
/** Clear all schedules (for testing). */
clear(): void {
this.db.getDatabase().prepare('DELETE FROM cron_schedules').run();
}
// ── W5.12: Cron Execution History ──────────────────────────────────
/** Record a cron job execution result. */
recordExecution(scheduleId: number, scheduleName: string, opts: {
executedAt?: string;
durationMs?: number;
success: boolean;
resultSummary?: string;
error?: string;
}): void {
this.db.getDatabase().prepare(
`INSERT INTO cron_execution_history (schedule_id, schedule_name, executed_at, duration_ms, success, result_summary, error)
VALUES (?, ?, COALESCE(?, datetime('now')), ?, ?, ?, ?)`,
).run(scheduleId, scheduleName, opts.executedAt ?? null, opts.durationMs ?? null, opts.success ? 1 : 0, opts.resultSummary ?? null, opts.error ?? null);
}
/** #17: count today's (UTC) executions for a schedule — ai_task daily cap. */
countExecutionsToday(scheduleId: number): number {
const row = this.db.getDatabase().prepare(
"SELECT COUNT(*) AS n FROM cron_execution_history WHERE schedule_id = ? AND executed_at >= date('now')",
).get(scheduleId) as { n: number };
return row.n;
}
/** Get execution history for a schedule (most recent first). */
getExecutionHistory(scheduleId: number, limit = 20): CronExecutionRow[] {
return this.db.getDatabase().prepare(
'SELECT * FROM cron_execution_history WHERE schedule_id = ? ORDER BY executed_at DESC LIMIT ?',
).all(scheduleId, limit) as CronExecutionRow[];
}
/** Get the most recent executions for boot-time failure-state recovery. */
getRecentExecutions(scheduleId: number, limit = 5): CronExecutionRow[] {
return this.db.getDatabase().prepare(
'SELECT * FROM cron_execution_history WHERE schedule_id = ? ORDER BY executed_at DESC, id DESC LIMIT ?',
).all(scheduleId, limit) as CronExecutionRow[];
}
/** Prune execution-history rows older than N days. recordExecution writes a
* row per tick (UX-Refactor Phase 3, Journey 16), so without retention the
* table grows unbounded (a per-minute job ≈ 525k rows/year). Mirrors
* optStore.pruneOlderThan(30). Returns the number of rows deleted. */
pruneExecutionHistory(olderThanDays: number): number {
const days = Math.max(1, Math.floor(olderThanDays));
const result = this.db.getDatabase().prepare(
"DELETE FROM cron_execution_history WHERE executed_at < datetime('now', ?)",
).run(`-${days} days`);
return result.changes;
}
// ── Interrupted-run leases ────────────────────────────────────────
/** Acquire a durable lease before a scheduled job begins execution. */
acquireRunLease(scheduleId: number, name: string, pid: number): number {
const result = this.db.getDatabase().prepare(
'INSERT INTO cron_run_leases (schedule_id, schedule_name, pid) VALUES (?, ?, ?)',
).run(scheduleId, name, pid);
return Number(result.lastInsertRowid);
}
/** Release a run lease after its scheduled job finishes. */
releaseRunLease(leaseId: number): void {
this.db.getDatabase().prepare(
'DELETE FROM cron_run_leases WHERE id = ?',
).run(leaseId);
}
/** List leases left behind by an interrupted process. */
listStaleRunLeases(): CronRunLeaseRow[] {
return this.db.getDatabase().prepare(
'SELECT * FROM cron_run_leases ORDER BY started_at ASC, id ASC',
).all() as CronRunLeaseRow[];
}
/** Clear all interrupted-run leases after boot recovery. */
clearRunLeases(): void {
this.db.getDatabase().prepare('DELETE FROM cron_run_leases').run();
}
// ── W5.10: Notification Persistence ────────────────────────────────
/** Save a notification. */
saveNotification(title: string, body: string, category = 'system', actionUrl?: string): number {
const result = this.db.getDatabase().prepare(
'INSERT INTO notifications (title, body, category, action_url) VALUES (?, ?, ?, ?)',
).run(title, body, category, actionUrl ?? null);
return Number(result.lastInsertRowid);
}
/** Get recent notifications (newest first). */
getNotifications(opts?: { since?: string; limit?: number; unreadOnly?: boolean }): NotificationRow[] {
const limit = opts?.limit ?? 50;
let sql = 'SELECT * FROM notifications';
const conditions: string[] = [];
const params: unknown[] = [];
if (opts?.since) { conditions.push('created_at > ?'); params.push(opts.since); }
if (opts?.unreadOnly) { conditions.push('read = 0'); }
if (conditions.length > 0) sql += ' WHERE ' + conditions.join(' AND ');
sql += ' ORDER BY created_at DESC LIMIT ?';
params.push(limit);
return this.db.getDatabase().prepare(sql).all(...params) as NotificationRow[];
}
/** Mark a notification as read. */
markNotificationRead(id: number): void {
this.db.getDatabase().prepare('UPDATE notifications SET read = 1 WHERE id = ?').run(id);
}
/** Mark all notifications as read. Returns the number of rows updated. */
markAllRead(): number {
const result = this.db.getDatabase().prepare('UPDATE notifications SET read = 1 WHERE read = 0').run();
return result.changes;
}
/** Count unread notifications. */
countUnread(): number {
const row = this.db.getDatabase().prepare('SELECT COUNT(*) as cnt FROM notifications WHERE read = 0').get() as { cnt: number };
return row.cnt;
}
// ── L2: Held-action approval queue ─────────────────────────────────
/** Enqueue a held action (status 'held'). Returns the persisted row. */
savePendingAction(input: SavePendingActionInput): PendingActionRow {
this.db.getDatabase().prepare(`
INSERT INTO pending_actions (id, workspace_id, source, tool_name, args_json, summary, risk_level, approval_class, status, expires_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'held', ?)
`).run(
input.id,
input.workspaceId,
input.source,
input.toolName,
input.argsJson,
input.summary ?? null,
input.riskLevel,
input.approvalClass,
input.expiresAt ?? null,
);
return this.getPendingAction(input.id)!;
}
/** List held actions (newest first) — defaults to the 'held' queue. */
listPendingActions(status: PendingActionStatus = 'held'): PendingActionRow[] {
return this.db.getDatabase().prepare(
'SELECT * FROM pending_actions WHERE status = ? ORDER BY created_at DESC',
).all(status) as PendingActionRow[];
}
/** Get one held action by id. */
getPendingAction(id: string): PendingActionRow | undefined {
return this.db.getDatabase().prepare(
'SELECT * FROM pending_actions WHERE id = ?',
).get(id) as PendingActionRow | undefined;
}
/**
* Atomically claim a held action — transition 'held' → 'approved' | 'denied'.
* This is the idempotency gate: exactly ONE caller wins (the `WHERE status =
* 'held'` makes it atomic in SQLite), so a double-approve / approve-after-deny
* is a no-op. Returns the updated row if THIS call won the claim, else
* undefined (already decided or missing).
*/
claimPendingAction(id: string, status: 'approved' | 'denied', decidedAt: string): PendingActionRow | undefined {
const result = this.db.getDatabase().prepare(
"UPDATE pending_actions SET status = ?, decided_at = ? WHERE id = ? AND status = 'held'",
).run(status, decidedAt, id);
if (result.changes === 0) return undefined;
return this.getPendingAction(id);
}
/**
* Record the terminal outcome of a claimed action ('approved' → 'executed' |
* 'failed'). Unconditional — the caller already won claimPendingAction(), so
* it owns the row and no further guard is needed.
*/
updatePendingActionResult(id: string, changes: { status: 'executed' | 'failed'; resultSummary?: string; error?: string; executedAt?: string }): void {
const sets: string[] = ['status = ?'];
const vals: unknown[] = [changes.status];
if (changes.resultSummary !== undefined) { sets.push('result_summary = ?'); vals.push(changes.resultSummary); }
if (changes.error !== undefined) { sets.push('error = ?'); vals.push(changes.error); }
if (changes.executedAt !== undefined) { sets.push('executed_at = ?'); vals.push(changes.executedAt); }
vals.push(id);
this.db.getDatabase().prepare(
`UPDATE pending_actions SET ${sets.join(', ')} WHERE id = ?`,
).run(...vals);
}
/** Flip held actions past their expires_at to 'expired'. Returns rows changed. */
expireStalePendingActions(): number {
const result = this.db.getDatabase().prepare(
"UPDATE pending_actions SET status = 'expired' WHERE status = 'held' AND expires_at IS NOT NULL AND expires_at < datetime('now')",
).run();
return result.changes;
}
/** Clear all pending actions (for testing). */
clearPendingActions(): void {
this.db.getDatabase().prepare('DELETE FROM pending_actions').run();
}
}

View File

@@ -0,0 +1,259 @@
/**
* FileIndexer — auto-index workspace files into their workspace mind (L-20).
*
* Design decisions (see docs/plans/FILE-TOOLS-AUDIT-2026-04-20.md):
* • Trigger: on-upload / on-overwrite (called from route handlers)
* • Target mind: per-workspace (the mind you pass in the constructor)
* • Granularity: 1 frame per file
* • Formats: .md, .markdown, .txt only for v1 (Bucket 2 adds PDF/DOCX)
* • Cleanup: re-index on overwrite, delete index on file remove
*
* A `file_index` table tracks the file_path → frame_id link so overwrites can
* swap the old frame cleanly and moves/deletes can locate the right frame
* without scanning every frame in the workspace.
*
* Large files are truncated to MAX_CONTENT_BYTES with a marker appended; the
* full file is still on disk, only the indexed frame is shortened. Keeps the
* mind DB size bounded even if a user drops a 5MB markdown dump.
*/
import { createHash } from 'node:crypto';
import path from 'node:path';
import type { MindDB } from '@waggle/hive-mind-core';
import { FrameStore } from '@waggle/hive-mind-core';
import { SessionStore } from '@waggle/hive-mind-core';
const FILE_INDEX_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS file_index (
file_path TEXT PRIMARY KEY,
frame_id INTEGER NOT NULL,
mime_type TEXT,
size_bytes INTEGER NOT NULL DEFAULT 0,
content_hash TEXT NOT NULL,
indexed_at TEXT NOT NULL DEFAULT (datetime('now'))
)
`;
const FILE_INDEX_IDX_SQL = `
CREATE INDEX IF NOT EXISTS idx_file_index_frame ON file_index (frame_id)
`;
const INDEX_SESSION_GOP_ID = 'session:file-index';
const INDEX_SESSION_PROJECT = 'file-indexer';
/** Maximum bytes of file content we store in a single frame. Beyond this, the
* content is truncated with a marker. Keeps mind size bounded on large dumps. */
export const MAX_CONTENT_BYTES = 64 * 1024;
/** Extensions we index for v1. PDF/DOCX/XLSX land in Bucket 2. */
const INDEXABLE_EXTENSIONS = new Set(['.md', '.markdown', '.txt']);
export interface FileIndexRow {
filePath: string;
frameId: number;
mimeType: string | null;
sizeBytes: number;
contentHash: string;
indexedAt: string;
}
export type FileIndexResult =
| { skipped: true; reason: 'unsupported_format' | 'unchanged' | 'empty' }
| { skipped: false; frameId: number; truncated: boolean };
export class FileIndexer {
private readonly db: MindDB;
private readonly frames: FrameStore;
private readonly sessions: SessionStore;
private cachedGopId: string | null = null;
constructor(db: MindDB) {
this.db = db;
this.frames = new FrameStore(db);
this.sessions = new SessionStore(db);
this.ensureTable();
}
private ensureTable(): void {
const raw = this.db.getDatabase();
const existsRow = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='file_index'",
).get();
if (existsRow) return;
raw.prepare(FILE_INDEX_TABLE_SQL).run();
raw.prepare(FILE_INDEX_IDX_SQL).run();
}
/** True iff the path's extension is one we index in v1. */
static shouldIndex(filePath: string): boolean {
const ext = path.extname(filePath).toLowerCase();
return INDEXABLE_EXTENSIONS.has(ext);
}
private ensureSession(): string {
if (this.cachedGopId) return this.cachedGopId;
const session = this.sessions.ensure(
INDEX_SESSION_GOP_ID,
INDEX_SESSION_PROJECT,
'Auto-indexed workspace files',
);
this.cachedGopId = session.gop_id;
return session.gop_id;
}
/**
* Index a file. Idempotent on unchanged content (same hash → skip). On
* overwrite with new content, deletes the old frame and inserts a fresh one.
*/
indexFile(filePath: string, content: Buffer, mime?: string): FileIndexResult {
if (!FileIndexer.shouldIndex(filePath)) {
return { skipped: true, reason: 'unsupported_format' };
}
if (content.length === 0) {
// Still remove any stale index so an emptied file doesn't leave an
// orphan frame pointing at old content.
this.removeFile(filePath);
return { skipped: true, reason: 'empty' };
}
const hash = createHash('sha256').update(content).digest('hex');
const raw = this.db.getDatabase();
const existingRow = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(filePath) as
| Record<string, unknown>
| undefined;
if (existingRow && existingRow.content_hash === hash) {
return { skipped: true, reason: 'unchanged' };
}
const normalized = this.normalizeText(content);
const truncated = content.length > MAX_CONTENT_BYTES;
const frameBody = this.buildFrameContent(filePath, normalized, truncated, mime);
const gopId = this.ensureSession();
// Atomicity contract (L-20 BLOCKER-1 fix): createIFrame → (conditional
// old-frame delete) → UPDATE/INSERT on file_index must commit as one unit.
// If any step throws mid-callback, better-sqlite3 rolls back the new frame
// and the index-row write, leaving the table in its pre-call state. Without
// this, a crash between the delete and the UPDATE would leave file_index
// pointing at a deleted frame_id (dangling) or an uncommitted new frame
// orphaned in the frames table.
const mutate = raw.transaction(() => {
const frame = this.frames.createIFrame(gopId, frameBody, 'normal', 'system');
if (existingRow) {
// Overwrite path: delete the old frame before we swap the row, unless
// the dedup coalesced two paths onto the same frame (leave that frame
// alone in that case — the other path still references it).
const oldFrameId = Number(existingRow.frame_id);
if (oldFrameId !== frame.id) {
const otherRef = raw.prepare('SELECT 1 FROM file_index WHERE frame_id = ? AND file_path != ? LIMIT 1').get(oldFrameId, filePath);
if (!otherRef) {
this.frames.delete(oldFrameId);
}
}
raw.prepare(`
UPDATE file_index SET
frame_id = ?,
mime_type = ?,
size_bytes = ?,
content_hash = ?,
indexed_at = datetime('now')
WHERE file_path = ?
`).run(frame.id, mime ?? null, content.length, hash, filePath);
} else {
raw.prepare(`
INSERT INTO file_index (file_path, frame_id, mime_type, size_bytes, content_hash)
VALUES (?, ?, ?, ?, ?)
`).run(filePath, frame.id, mime ?? null, content.length, hash);
}
return frame.id;
});
const frameId = mutate();
return { skipped: false, frameId, truncated };
}
/** Remove a file's frame + index row. Returns true if anything was removed. */
removeFile(filePath: string): boolean {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT frame_id FROM file_index WHERE file_path = ?').get(filePath) as
| { frame_id: number }
| undefined;
if (!row) return false;
const otherRef = raw.prepare('SELECT 1 FROM file_index WHERE frame_id = ? AND file_path != ? LIMIT 1').get(row.frame_id, filePath);
if (!otherRef) {
this.frames.delete(row.frame_id);
}
raw.prepare('DELETE FROM file_index WHERE file_path = ?').run(filePath);
return true;
}
/**
* Update the recorded path for a moved file. The frame content itself still
* carries the old path in its header — that's acceptable for v1 since the
* frame body is already the file's own content (not a description). A later
* reader sees the stale header only if they dig into raw frame content; the
* index table, which is what every other code path queries, is correct.
*/
moveFile(from: string, to: string): boolean {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(from) as
| Record<string, unknown>
| undefined;
if (!row) return false;
// If `to` already indexed, drop its entry (file was replaced by the move).
raw.prepare('DELETE FROM file_index WHERE file_path = ?').run(to);
raw.prepare('UPDATE file_index SET file_path = ?, indexed_at = datetime(\'now\') WHERE file_path = ?').run(to, from);
return true;
}
getRow(filePath: string): FileIndexRow | null {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(filePath) as
| Record<string, unknown>
| undefined;
if (!row) return null;
return {
filePath: row.file_path as string,
frameId: row.frame_id as number,
mimeType: (row.mime_type as string | null) ?? null,
sizeBytes: row.size_bytes as number,
contentHash: row.content_hash as string,
indexedAt: row.indexed_at as string,
};
}
listAll(): FileIndexRow[] {
const raw = this.db.getDatabase();
const rows = raw.prepare('SELECT * FROM file_index ORDER BY indexed_at DESC').all() as Record<string, unknown>[];
return rows.map(r => ({
filePath: r.file_path as string,
frameId: r.frame_id as number,
mimeType: (r.mime_type as string | null) ?? null,
sizeBytes: r.size_bytes as number,
contentHash: r.content_hash as string,
indexedAt: r.indexed_at as string,
}));
}
private normalizeText(content: Buffer): string {
// UTF-8 decode. Malformed bytes become U+FFFD; acceptable for index use.
const full = content.toString('utf-8');
if (Buffer.byteLength(full, 'utf-8') <= MAX_CONTENT_BYTES) return full;
// Truncate in character-safe chunks. Walk down until we're under the byte
// budget; avoids cutting a multi-byte character in half.
let candidate = full.slice(0, MAX_CONTENT_BYTES);
while (Buffer.byteLength(candidate, 'utf-8') > MAX_CONTENT_BYTES && candidate.length > 0) {
candidate = candidate.slice(0, candidate.length - 1);
}
return candidate;
}
private buildFrameContent(filePath: string, body: string, truncated: boolean, mime?: string): string {
const header = `[FILE: ${filePath}${mime ? ` · ${mime}` : ''}]`;
const suffix = truncated ? `\n\n[…truncated at ${MAX_CONTENT_BYTES} bytes for indexing]` : '';
return `${header}\n\n${body}${suffix}`;
}
}

View File

@@ -0,0 +1,639 @@
/**
* FileStore — workspace file storage abstraction.
*
* Provides a clean interface for reading/writing workspace files that abstracts
* over the underlying storage backend. Two implementations:
*
* - LocalFileStore: manages files in ~/.waggle/workspaces/{id}/files/ (virtual storage)
* - LinkedDirStore: reads/writes to workspace.directory (linked to external folder)
* - S3FileStore: MinIO/S3-backed storage for team/cloud deployments (lazy SDK import)
*
* All operations enforce path traversal protection — files cannot escape
* the workspace boundary.
*/
import fs from 'node:fs';
import path from 'node:path';
import { glob } from 'glob';
// Type-only import: erased at compile time, so it does not force @aws-sdk/client-s3
// (a devDependency) into the runtime bundle for local-only users.
import type { S3Client } from '@aws-sdk/client-s3';
// ── Interface ───────────────────────────────────────────────────────
export interface FileEntry {
name: string;
path: string; // relative to workspace root
size: number;
modified: string; // ISO timestamp
isDirectory: boolean;
}
export interface StorageInfo {
usedBytes: number;
fileCount: number;
storageType: 'virtual' | 'linked';
}
export interface FileStore {
// Read
readFile(relativePath: string): Promise<Buffer>;
listFiles(directory?: string): Promise<FileEntry[]>;
searchFiles(pattern: string): Promise<FileEntry[]>;
// Write
writeFile(relativePath: string, content: Buffer | string): Promise<void>;
deleteFile(relativePath: string): Promise<void>;
moveFile(from: string, to: string): Promise<void>;
// Meta
getStorageInfo(): Promise<StorageInfo>;
getRootPath(): string;
getStorageType(): 'virtual' | 'linked';
}
// ── Path safety ─────────────────────────────────────────────────────
/**
* Directory segments that, anywhere in a path, almost always hold secrets.
* Matched (normalized, case-insensitive) against each path segment.
*/
const SENSITIVE_DIR_SEGMENTS = new Set([
'.ssh', '.aws', '.gnupg', '.gpg', '.docker', '.kube', '.azure', '.terraform', '.terraform.d',
]);
/** Exact basenames (normalized) that are secret material. */
const SENSITIVE_BASENAMES = new Set([
'id_rsa', 'id_dsa', 'id_ecdsa', 'id_ed25519', 'authorized_keys', 'known_hosts',
'.netrc', '.pgpass', '.npmrc', '.pypirc', '.git-credentials',
'credentials', 'credentials.json', 'service-account.json',
'terraform.tfstate', 'terraform.tfstate.backup',
]);
/** Extensions that are (almost always) private-key material. */
const SENSITIVE_EXTENSIONS = new Set(['.pem']);
/** Backup/copy suffixes — strip and re-test the base (id_rsa.bak → id_rsa). */
const BACKUP_SUFFIX_RE = /\.(bak|old|backup|orig|copy|save|swp)$/i;
/** `.env` files are secrets — but the documented, checked-in templates are not. */
const ENV_TEMPLATE_ALLOW = new Set(['.env.example', '.env.sample', '.env.template', '.env.dist', '.env.defaults']);
/**
* Normalize one path segment to the name the OS will actually open: lower-case
* (case-insensitive FS), strip a Windows NTFS alternate-data-stream suffix
* (`id_rsa::$DATA` → `id_rsa`) and any trailing dots/spaces (`id_rsa.`, `.env `
* → the base) which Windows silently removes when opening.
*/
function normalizeSegment(seg: string): string {
return seg.toLowerCase().replace(/::.*$/, '').replace(/[. ]+$/, '');
}
function isSensitiveBase(base: string): boolean {
if (SENSITIVE_BASENAMES.has(base)) return true;
const dot = base.lastIndexOf('.');
if (dot > 0 && SENSITIVE_EXTENSIONS.has(base.slice(dot))) return true;
if (base === '.env' || base.startsWith('.env.')) return !ENV_TEMPLATE_ALLOW.has(base);
return false;
}
/**
* True when a path points at well-known secret material (SSH/GPG keys, cloud
* credentials, dotenv files, terraform state, …). Used to deny reads/writes
* inside LINKED external folders so an agent given a project directory cannot
* exfiltrate or clobber the user's secrets.
*
* It is a BLOCKLIST (defense-in-depth), not a sandbox: it raises the bar against
* obvious secrets but cannot enumerate every secret a home dir holds. Conservative
* on extensions (only *.pem) to avoid denying legitimate files. Path-separator
* agnostic; segments are normalized for case + Windows ADS/trailing-char tricks.
*/
export function isSensitiveFilePath(relativePath: string): boolean {
const segments = relativePath.replace(/\\/g, '/').split('/').map(normalizeSegment).filter(Boolean);
if (segments.length === 0) return false;
for (const seg of segments) {
if (SENSITIVE_DIR_SEGMENTS.has(seg)) return true;
}
const base = segments[segments.length - 1];
if (isSensitiveBase(base)) return true;
if (BACKUP_SUFFIX_RE.test(base) && isSensitiveBase(base.replace(BACKUP_SUFFIX_RE, ''))) return true;
return false;
}
interface ResolveSafeOptions {
/** Reject paths flagged by isSensitiveFilePath (linked external dirs only). */
denySensitive?: boolean;
}
/** realpathSync, falling back to the input if it can't be resolved (e.g. not created yet). */
function safeRealpath(p: string): string {
try { return fs.realpathSync(p); } catch { return p; }
}
/** Walk up to the deepest ancestor of `p` that exists on disk. */
function deepestExisting(p: string): string {
let cur = p;
while (cur !== path.dirname(cur) && !fs.existsSync(cur)) cur = path.dirname(cur);
return cur;
}
/**
* Resolve `relativePath` under `root` and assert it cannot escape the boundary.
*
* Two layers:
* 1. LEXICAL segment-boundary containment — the resolved path is the root or
* sits under `root + sep` (a bare `startsWith(root)` wrongly admits a sibling
* like `${root}-evil/secret`).
* 2. SYMLINK-aware containment — `path.resolve` is lexical but `fs.*` follows
* symlinks, so we realpath the deepest EXISTING ancestor of the target and
* re-check it is still under the realpath'd root. This blocks a benign-named
* symlink that points outside the boundary, while still permitting in-root
* symlinks (e.g. monorepo package links). The sensitive-file deny then runs
* on BOTH the lexical and the real in-root path, so an in-root symlink to a
* secret (`alias → ./.env`) cannot launder it past a benign basename.
*/
function resolveSafe(root: string, relativePath: string, opts: ResolveSafeOptions = {}): string {
const resolvedRoot = path.resolve(root);
const resolved = path.resolve(resolvedRoot, relativePath);
if (resolved !== resolvedRoot && !resolved.startsWith(resolvedRoot + path.sep)) {
throw new Error(`Path traversal denied: ${relativePath}`);
}
// Symlink-aware containment only applies once the root exists on disk — if it
// doesn't, nothing inside it exists to be a symlink, and walking the deepest
// existing ancestor above the (not-yet-created) root would compare against an
// unrelated real dir (e.g. an OS temp-dir symlink). Lexical containment holds.
const realRoot = safeRealpath(resolvedRoot);
let realTarget = realRoot;
if (fs.existsSync(resolvedRoot)) {
realTarget = safeRealpath(deepestExisting(resolved));
if (realTarget !== realRoot && !realTarget.startsWith(realRoot + path.sep)) {
throw new Error(`Path traversal denied (symlink): ${relativePath}`);
}
}
if (opts.denySensitive) {
const lexicalRel = path.relative(resolvedRoot, resolved);
const realRel = path.relative(realRoot, realTarget);
if (isSensitiveFilePath(lexicalRel) || isSensitiveFilePath(realRel)) {
throw new Error(`Access to sensitive file denied: ${relativePath}`);
}
}
return resolved;
}
/** Keep only glob matches that resolve back inside `root` (glob `../` patterns can escape cwd). */
function containGlobMatches(root: string, matches: string[]): string[] {
const resolvedRoot = path.resolve(root);
return matches.filter(m => {
const abs = path.resolve(resolvedRoot, m);
return abs === resolvedRoot || abs.startsWith(resolvedRoot + path.sep);
});
}
/**
* Convert a `*`/`?` glob to a RegExp that is SAFE against catastrophic
* backtracking (ReDoS). Every regex metacharacter is escaped first, so the only
* specials left are the two wildcards we re-introduce (`.*`, `.`) — neither can
* nest a quantifier, so a hostile pattern like `(a+)+x` becomes a harmless
* literal. Used by S3FileStore.searchFiles where the pattern is caller-controlled.
* Unanchored (substring match) to preserve the prior behavior.
*/
function globToRegExp(pattern: string): RegExp {
const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const body = escaped.replace(/\\\*/g, '.*').replace(/\\\?/g, '.');
return new RegExp(body, 'i');
}
/**
* Reject an S3 key relative-path that would escape its workspace prefix. S3 keys
* are literal, but MinIO (path-style, filesystem-backed) can normalize `..`, so a
* `..` segment could cross into a sibling workspace's prefix. Makes the file
* header's "all operations enforce path traversal protection" true for S3 too.
*/
function assertSafeRelativeKey(relativePath: string): void {
const segments = relativePath.replace(/\\/g, '/').split('/');
if (segments.some(s => s === '..')) {
throw new Error(`Path traversal denied: ${relativePath}`);
}
}
function ensureDir(dirPath: string): void {
if (!fs.existsSync(dirPath)) {
fs.mkdirSync(dirPath, { recursive: true });
}
}
// ── LocalFileStore (virtual workspace storage) ──────────────────────
export class LocalFileStore implements FileStore {
private readonly root: string;
constructor(dataDir: string, workspaceId: string) {
this.root = path.join(dataDir, 'workspaces', workspaceId, 'files');
}
getRootPath(): string { return this.root; }
getStorageType(): 'virtual' { return 'virtual'; }
async readFile(relativePath: string): Promise<Buffer> {
const fullPath = resolveSafe(this.root, relativePath);
return fs.readFileSync(fullPath);
}
async listFiles(directory?: string): Promise<FileEntry[]> {
const dir = directory ? resolveSafe(this.root, directory) : this.root;
if (!fs.existsSync(dir)) return [];
const entries = fs.readdirSync(dir, { withFileTypes: true });
return entries.map(entry => {
const fullPath = path.join(dir, entry.name);
const stat = fs.statSync(fullPath);
return {
name: entry.name,
path: path.relative(this.root, fullPath).replace(/\\/g, '/'),
size: stat.size,
modified: stat.mtime.toISOString(),
isDirectory: entry.isDirectory(),
};
});
}
async searchFiles(pattern: string): Promise<FileEntry[]> {
if (!fs.existsSync(this.root)) return [];
const matches = await glob(pattern, {
cwd: this.root,
nodir: true,
ignore: ['node_modules/**', '.git/**'],
});
// A glob pattern with `../` can escape cwd — keep only matches inside root.
return containGlobMatches(this.root, matches).slice(0, 200).map(match => {
const fullPath = path.join(this.root, match);
try {
const stat = fs.statSync(fullPath);
return {
name: path.basename(match),
path: match.replace(/\\/g, '/'),
size: stat.size,
modified: stat.mtime.toISOString(),
isDirectory: false,
};
} catch {
return { name: path.basename(match), path: match, size: 0, modified: '', isDirectory: false };
}
});
}
async writeFile(relativePath: string, content: Buffer | string): Promise<void> {
const fullPath = resolveSafe(this.root, relativePath);
// Lazy directory creation
ensureDir(path.dirname(fullPath));
fs.writeFileSync(fullPath, content);
}
async deleteFile(relativePath: string): Promise<void> {
const fullPath = resolveSafe(this.root, relativePath);
if (fs.existsSync(fullPath)) {
const stat = fs.statSync(fullPath);
if (stat.isDirectory()) {
fs.rmSync(fullPath, { recursive: true });
} else {
fs.unlinkSync(fullPath);
}
}
}
async moveFile(from: string, to: string): Promise<void> {
const fromPath = resolveSafe(this.root, from);
const toPath = resolveSafe(this.root, to);
ensureDir(path.dirname(toPath));
fs.renameSync(fromPath, toPath);
}
async getStorageInfo(): Promise<StorageInfo> {
if (!fs.existsSync(this.root)) {
return { usedBytes: 0, fileCount: 0, storageType: 'virtual' };
}
let usedBytes = 0;
let fileCount = 0;
const walk = (dir: string) => {
try {
const entries = fs.readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
walk(fullPath);
} else {
try {
usedBytes += fs.statSync(fullPath).size;
fileCount++;
} catch { /* skip unreadable */ }
}
}
} catch { /* skip unreadable dirs */ }
};
walk(this.root);
return { usedBytes, fileCount, storageType: 'virtual' };
}
}
// ── LinkedDirStore (linked to external directory) ───────────────────
export class LinkedDirStore implements FileStore {
private readonly root: string;
constructor(directory: string) {
this.root = directory;
}
getRootPath(): string { return this.root; }
getStorageType(): 'linked' { return 'linked'; }
// Linked stores point at a REAL external folder (a code project, even the home
// dir), so every path op denies well-known secret files — the agent cannot
// read or clobber ~/.ssh, .env, cloud credentials, etc. (LocalFileStore is a
// sandboxed virtual dir and needs no such deny.)
private static readonly DENY = { denySensitive: true } as const;
async readFile(relativePath: string): Promise<Buffer> {
const fullPath = resolveSafe(this.root, relativePath, LinkedDirStore.DENY);
return fs.readFileSync(fullPath);
}
async listFiles(directory?: string): Promise<FileEntry[]> {
const dir = directory ? resolveSafe(this.root, directory, LinkedDirStore.DENY) : this.root;
if (!fs.existsSync(dir)) return [];
const entries = fs.readdirSync(dir, { withFileTypes: true });
return entries
.filter(e => !e.name.startsWith('.') && e.name !== 'node_modules')
// Don't disclose the existence/size/mtime of non-dot secrets (credentials.json,
// id_rsa, known_hosts, …) — symmetric with searchFiles + the read deny.
.filter(e => !isSensitiveFilePath(e.name))
.map(entry => {
const fullPath = path.join(dir, entry.name);
try {
const stat = fs.statSync(fullPath);
return {
name: entry.name,
path: path.relative(this.root, fullPath).replace(/\\/g, '/'),
size: stat.size,
modified: stat.mtime.toISOString(),
isDirectory: entry.isDirectory(),
};
} catch {
return { name: entry.name, path: entry.name, size: 0, modified: '', isDirectory: false };
}
});
}
async searchFiles(pattern: string): Promise<FileEntry[]> {
const matches = await glob(pattern, {
cwd: this.root,
nodir: true,
ignore: ['node_modules/**', '.git/**', 'dist/**', 'build/**', '**/.ssh/**', '**/.aws/**', '**/.gnupg/**'],
});
// Contain `../`-escaping globs to the root, then drop any secret a creative
// pattern still matched — search never discloses the existence/path of secrets.
return containGlobMatches(this.root, matches).filter(m => !isSensitiveFilePath(m)).slice(0, 200).map(match => {
const fullPath = path.join(this.root, match);
try {
const stat = fs.statSync(fullPath);
return {
name: path.basename(match),
path: match.replace(/\\/g, '/'),
size: stat.size,
modified: stat.mtime.toISOString(),
isDirectory: false,
};
} catch {
return { name: path.basename(match), path: match, size: 0, modified: '', isDirectory: false };
}
});
}
async writeFile(relativePath: string, content: Buffer | string): Promise<void> {
const fullPath = resolveSafe(this.root, relativePath, LinkedDirStore.DENY);
ensureDir(path.dirname(fullPath));
fs.writeFileSync(fullPath, content);
}
async deleteFile(relativePath: string): Promise<void> {
const fullPath = resolveSafe(this.root, relativePath, LinkedDirStore.DENY);
if (fs.existsSync(fullPath)) {
fs.unlinkSync(fullPath);
}
}
async moveFile(from: string, to: string): Promise<void> {
const fromPath = resolveSafe(this.root, from, LinkedDirStore.DENY);
const toPath = resolveSafe(this.root, to, LinkedDirStore.DENY);
ensureDir(path.dirname(toPath));
fs.renameSync(fromPath, toPath);
}
async getStorageInfo(): Promise<StorageInfo> {
if (!fs.existsSync(this.root)) {
return { usedBytes: 0, fileCount: 0, storageType: 'linked' };
}
let usedBytes = 0;
let fileCount = 0;
const walk = (dir: string, depth = 0) => {
if (depth > 5) return; // Don't walk too deep in linked dirs
try {
const entries = fs.readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
if (entry.name === 'node_modules' || entry.name === '.git') continue;
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
walk(fullPath, depth + 1);
} else {
try {
usedBytes += fs.statSync(fullPath).size;
fileCount++;
} catch { /* skip */ }
}
}
} catch { /* skip */ }
};
walk(this.root);
return { usedBytes, fileCount, storageType: 'linked' };
}
}
// ── S3FileStore (MinIO/S3 for team workspace storage) ───────────────
export interface S3Config {
endpoint: string;
bucket: string;
accessKey: string;
secretKey: string;
prefix: string; // e.g., "workspaces/{id}/"
region?: string;
}
export class S3FileStore implements FileStore {
private config: S3Config;
// Lazily constructed S3 client; the SDK is imported on first use to avoid
// bundling it for local-only users.
private client: S3Client | undefined;
constructor(config: S3Config) {
this.config = config;
}
private async getClient(): Promise<S3Client> {
if (this.client) return this.client;
const { S3Client } = await import('@aws-sdk/client-s3');
this.client = new S3Client({
endpoint: `http://${this.config.endpoint}`,
region: this.config.region ?? 'us-east-1',
credentials: {
accessKeyId: this.config.accessKey,
secretAccessKey: this.config.secretKey,
},
forcePathStyle: true, // Required for MinIO
});
return this.client;
}
getRootPath(): string { return `s3://${this.config.bucket}/${this.config.prefix}`; }
getStorageType(): 'virtual' | 'linked' { return 'virtual'; }
async readFile(relativePath: string): Promise<Buffer> {
assertSafeRelativeKey(relativePath);
const client = await this.getClient();
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
const key = this.config.prefix + relativePath;
const response = await client.send(new GetObjectCommand({
Bucket: this.config.bucket,
Key: key,
}));
const body = response.Body;
if (!body) throw new Error(`S3 object has no body: ${key}`);
const chunks: Uint8Array[] = [];
// In Node.js the S3 streaming body is an async-iterable readable stream.
for await (const chunk of body as AsyncIterable<Uint8Array>) chunks.push(chunk);
return Buffer.concat(chunks);
}
async writeFile(relativePath: string, content: Buffer | string): Promise<void> {
assertSafeRelativeKey(relativePath);
const client = await this.getClient();
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
const key = this.config.prefix + relativePath;
await client.send(new PutObjectCommand({
Bucket: this.config.bucket,
Key: key,
Body: typeof content === 'string' ? Buffer.from(content) : content,
}));
}
async deleteFile(relativePath: string): Promise<void> {
assertSafeRelativeKey(relativePath);
const client = await this.getClient();
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3');
const key = this.config.prefix + relativePath;
await client.send(new DeleteObjectCommand({
Bucket: this.config.bucket,
Key: key,
}));
}
async listFiles(directory?: string): Promise<FileEntry[]> {
if (directory) assertSafeRelativeKey(directory);
const client = await this.getClient();
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
const pfx = this.config.prefix + (directory ? directory + '/' : '');
const response = await client.send(new ListObjectsV2Command({
Bucket: this.config.bucket,
Prefix: pfx,
Delimiter: '/',
}));
const entries: FileEntry[] = [];
for (const obj of response.Contents ?? []) {
if (!obj.Key) continue;
const relPath = obj.Key.slice(this.config.prefix.length);
entries.push({
name: relPath.split('/').pop() ?? relPath,
path: relPath,
size: obj.Size ?? 0,
modified: obj.LastModified?.toISOString() ?? '',
isDirectory: false,
});
}
for (const cpfx of response.CommonPrefixes ?? []) {
if (!cpfx.Prefix) continue;
const relPath = cpfx.Prefix.slice(this.config.prefix.length).replace(/\/$/, '');
entries.push({
name: relPath.split('/').pop() ?? relPath,
path: relPath,
size: 0,
modified: '',
isDirectory: true,
});
}
return entries;
}
async searchFiles(pattern: string): Promise<FileEntry[]> {
const all = await this.listFiles();
const regex = globToRegExp(pattern); // ReDoS-safe glob→regex (pattern is caller-controlled)
return all.filter(f => regex.test(f.path) || regex.test(f.name));
}
async moveFile(from: string, to: string): Promise<void> {
assertSafeRelativeKey(from);
assertSafeRelativeKey(to);
const client = await this.getClient();
const { CopyObjectCommand, DeleteObjectCommand } = await import('@aws-sdk/client-s3');
const fromKey = this.config.prefix + from;
const toKey = this.config.prefix + to;
await client.send(new CopyObjectCommand({
Bucket: this.config.bucket,
CopySource: `${this.config.bucket}/${fromKey}`,
Key: toKey,
}));
await client.send(new DeleteObjectCommand({
Bucket: this.config.bucket,
Key: fromKey,
}));
}
async getStorageInfo(): Promise<StorageInfo> {
const client = await this.getClient();
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
const response = await client.send(new ListObjectsV2Command({
Bucket: this.config.bucket,
Prefix: this.config.prefix,
}));
let usedBytes = 0;
let fileCount = 0;
for (const obj of response.Contents ?? []) {
usedBytes += obj.Size ?? 0;
fileCount++;
}
return { usedBytes, fileCount, storageType: 'virtual' };
}
}
// ── Factory ─────────────────────────────────────────────────────────
/**
* Create the appropriate FileStore for a workspace.
* If s3Config is provided, use S3FileStore (team/cloud deployment).
* If the workspace has a linked directory, use LinkedDirStore.
* Otherwise, use LocalFileStore (virtual managed storage).
*/
export function createFileStore(
dataDir: string,
workspaceId: string,
linkedDirectory?: string,
s3Config?: S3Config,
): FileStore {
if (s3Config) return new S3FileStore(s3Config);
if (linkedDirectory && fs.existsSync(linkedDirectory)) {
return new LinkedDirStore(linkedDirectory);
}
return new LocalFileStore(dataDir, workspaceId);
}

159
packages/core/src/index.ts Normal file
View File

@@ -0,0 +1,159 @@
// @waggle/core — Waggle-specific orchestration layer.
//
// As of CC Sesija B 2026-04-30 (PM Q3 Plan A ratification), the substrate
// (mind/ + harvest/ + logger + injection-scanner) lives in @waggle/hive-mind-core
// and is distributed as Apache 2.0 OSS via subtree-split. This barrel re-exports
// substrate symbols for backward compatibility — existing consumers (packages/agent,
// apps/web, packages/server, etc.) keep their `import { ... } from '@waggle/core'`
// imports unchanged.
//
// Waggle-specific (non-extracted) modules below the substrate re-exports stay
// in this package: config, multi-mind, workspace orchestration, vault, telemetry,
// install-audit, cron-store, skill-hashes, team-sync, file-store, file-indexer,
// memory-import, optimization-log, compliance/.
// ── Substrate re-exports from @waggle/hive-mind-core (Apache 2.0 OSS) ──
export {
// Logger + injection scanner
createCoreLogger, type CoreLogger,
scanForInjection, type ScanResult,
// mind/ — memory substrate
MindDB, EmbeddingDimMismatchError,
type EmbeddingFingerprint, type FingerprintCheck,
IdentityLayer, type Identity,
AwarenessLayer, type AwarenessItem, type AwarenessCategory,
FrameStore, stripHmPrefix, hashFrameContent,
RawArchive, hashRaw, readArchiveUids, withArchiveUid, type RawArchiveRow, type ArchiveInput,
MindErasure, type EraseResult,
SuppressionStore, type SuppressedSubject,
type MemoryFrame, type FrameType, type Importance, type FrameSource,
SessionStore, type Session,
HybridSearch, type SearchResult,
// D1 (oss-drift triage, 2026-06-11) — chunk-level retrieval (flag-gated, default OFF)
chunkRetrievalEnabled, rechunkAllFrames, type RechunkResult,
chunkText, type ChunkOptions, type FrameChunk,
KnowledgeGraph, type Entity, type Relation, type ValidationSchema,
SCHEMA_SQL, VEC_TABLE_SQL, CHUNKS_VEC_TABLE_SQL, SCHEMA_VERSION,
vecTableSqlForDim, chunksVecTableSqlForDim,
computeRelevance,
computeTemporalScore,
computePopularityScore,
computeContextualScore,
computeImportanceScore,
SCORING_PROFILES,
type ScoringProfile,
type ScoringWeights,
type Embedder,
createLiteLLMEmbedder, type LiteLLMEmbedderConfig,
createInProcessEmbedder, normalizeDimensions, type InProcessEmbedderConfig,
createOllamaEmbedder, type OllamaEmbedderConfig,
createApiEmbedder, type ApiEmbedderConfig,
createEmbeddingProvider, EmbeddingQuotaExceededError, getMinimumTierForProvider,
type EmbeddingProviderConfig, type EmbeddingProviderStatus, type EmbeddingProviderType,
type EmbeddingProviderInstance, type EmbeddingQuotaStatus,
normalizeEntityName, findDuplicates, isNoiseName, isLikelyAcronym,
Ontology, validateEntity, type EntitySchema, type ValidationResult,
ImprovementSignalStore,
type ImprovementSignal, type ActionableSignal, type SignalCategory, type ActionableThresholds,
ExecutionTraceStore, EXECUTION_TRACES_TABLE_SQL,
type ExecutionTrace, type ParsedExecutionTrace, type TraceOutcome,
type TracePayload, type TraceToolCall, type TraceReasoningStep,
type StartTraceInput, type FinalizeTraceInput, type TraceQueryFilter,
EvolutionRunStore, EVOLUTION_RUNS_TABLE_SQL,
type EvolutionRun, type EvolutionRunStatus, type EvolutionRunTarget,
type CreateEvolutionRunInput, type EvolutionRunFilter,
reconcileIndexes, reconcileFtsIndex, reconcileVecIndex, cleanOrphanVectors, cleanOrphanFts,
type ReconcileResult,
ConceptTracker, CONCEPT_MASTERY_TABLE_SQL,
type ConceptEntry, type ConceptUpdate,
// recall-context — temporal rendering helpers (W4.1 production port)
TEMPORAL_GUIDANCE, toDatePrefix, renderDatedSnippet, referenceDate, renderReferenceDateLine,
parseDateWindow, type DateWindow,
resolveRelativeDate, type ResolvedDate,
createInProcessReranker, type Reranker, type InProcessRerankerConfig,
// harvest/ — universal memory ingestion pipeline
HarvestSourceStore,
HarvestRunStore, type HarvestRun, type HarvestRunStatus,
ChatGPTAdapter,
ClaudeAdapter,
ClaudeCodeAdapter,
GeminiAdapter,
UniversalAdapter,
MarkdownAdapter,
PlaintextAdapter,
UrlAdapter,
PdfAdapter,
HarvestPipeline, type LLMCallFn, type PipelineOptions,
extractMemoryLanes, writeMemoryLaneFrames,
MIND_FACT_PREFIX, MIND_EVENT_PREFIX, MIND_PROFILE_PREFIX,
type MemoryLaneExtraction, type ExtractedEvent, type ExtractedFact, type ExtractedProfile,
type WriteLaneFramesResult,
// D2 — LLM-based KG entity extraction (oss-drift triage, 2026-06-11)
extractKgEntities, writeKgEntities, KG_ENTITY_TYPES,
type KgEntity, type KgEntityType, type KgEntityExtraction, type WriteKgEntitiesResult,
// W4.6 — raw-turn storage + RAWDETAIL recall lane
writeRawTurnFrames, rawTurnHeader, parseRawTurnHeader, rawTurnConvKey,
MIND_RAWTURN_PREFIX, MAX_TURNS_PER_ITEM, RAWDETAIL_KILL_SWITCH,
type WriteRawTurnsResult, type ParsedRawTurnHeader,
fetchRawDetailLane, rawTurnBody, RAW_DETAIL_K,
type RawTurnHit, type RawDetailLaneOptions,
dedup, harvestSetHash,
HARVEST_FRAME_CONTENT_CAP,
type ImportSourceType, type ImportItemType, type UniversalImportItem, type DistilledKnowledge,
type HarvestPipelineResult, type HarvestSource, type SourceAdapter, type FilesystemAdapter,
type ClassifiedItem, type ExtractedContent, type KnowledgeProvenance,
// Plan A AMENDMENT 2026-04-30 — multi-workspace orchestration
MultiMind, type MultiMindSearchResult, type MindSource, type SearchScope,
MultiMindCache, type MultiMindCacheConfig,
WorkspaceManager, type WorkspaceConfig, type CreateWorkspaceOptions,
// Supersession (P) + bridge (B) frame producer (mono-parity 2026-07-05)
detectSupersessionChains, detectEntityGroups, applyConsolidation,
collectObservations, getCurrentValues,
type ConsolidationLlm, type Observation, type SupersessionChain,
type EntityGroup, type ConsolidationResult, type CollectObservationsOptions,
} from '@waggle/hive-mind-core';
// ── Waggle-specific orchestration (stays in @waggle/core) ──
export { WaggleConfig, type ProviderEntry, type TeamServerConfig } from './config.js';
export { needsMigration, migrateToMultiMind } from './migration.js';
export { TeamSync, frameToEntity, entityToSyncedFrame, type TeamSyncConfig, type SyncedFrame } from './team-sync.js';
export {
InstallAuditStore, INSTALL_AUDIT_TABLE_SQL,
type InstallAuditEntry, type RecordAuditInput,
type AuditAction, type AuditRiskLevel, type AuditTrustSource,
type AuditApprovalClass, type AuditInitiator, type AuditCapabilityType,
} from './install-audit.js';
export {
CronStore, CRON_SCHEDULES_TABLE_SQL, CRON_RUN_LEASES_TABLE_SQL, VALID_JOB_TYPES, cronExprError,
type CronSchedule, type CreateScheduleInput, type CronJobType,
type CronExecutionRow, type CronRunLeaseRow,
type PendingActionRow, type PendingActionStatus, type SavePendingActionInput,
} from './cron-store.js';
export { VaultStore, type VaultEntry } from './vault.js';
export { TelemetryStore, TelemetryCollector, TELEMETRY_EVENTS, type TelemetryEvent, type TelemetrySummary } from './telemetry.js';
export {
SkillHashStore, computeSkillHash, SKILL_HASHES_TABLE_SQL,
type SkillHash,
} from './skill-hashes.js';
export { processImport, parseChatGPTExport, parseClaudeExport, extractKnowledge } from './memory-import.js';
export { createFileStore, LocalFileStore, LinkedDirStore, S3FileStore, isSensitiveFilePath, type FileStore, type FileEntry, type StorageInfo, type S3Config } from './file-store.js';
export { FileIndexer, MAX_CONTENT_BYTES, type FileIndexRow, type FileIndexResult } from './file-indexer.js';
export type { ImportSource, ImportResult, ExtractedKnowledge, ParsedConversation, ConversationMessage } from './memory-import.js';
export {
OptimizationLogStore, OPTIMIZATION_LOG_TABLE_SQL,
type OptimizationLogEntry, type CreateOptimizationLogInput,
} from './optimization-log.js';
// ── Compliance (AI Act) — stays in @waggle/core (NOT extracted per .github/sync.md) ──
export { InteractionStore } from './compliance/interaction-store.js';
export { ComplianceStatusChecker } from './compliance/status-checker.js';
export { ReportGenerator, type ReportGeneratorDeps } from './compliance/report-generator.js';
export { ComplianceTemplateStore, KVARK_TEMPLATE_NAME } from './compliance/template-store.js';
export { TEMPLATE_RISK_MAP } from './compliance/types.js';
export type {
AIActRiskLevel, HumanAction, AIInteraction, RecordInteractionInput,
ComplianceStatus, ArticleStatus, AuditReport, AuditReportRequest,
ModelInventoryEntry, OversightLogEntry, HarvestProvenanceEntry,
ComplianceTemplate, ComplianceTemplateSections,
CreateComplianceTemplateInput, UpdateComplianceTemplateInput,
} from './compliance/types.js';

View File

@@ -0,0 +1,221 @@
/**
* Install Audit Store — persistent audit trail for capability install events.
*
* Records every install-relevant action (proposed, approved, installed, rejected,
* failed) so there is a verifiable history of what was installed, when, why,
* and by whom.
*
* Follows the same pattern as ImprovementSignalStore — operates on the .mind DB.
*/
import type { MindDB } from '@waggle/hive-mind-core';
import {
sqlInList, RISK_LEVELS, APPROVAL_CLASSES, AUDIT_ACTIONS,
AUDIT_CAPABILITY_TYPES, AUDIT_INITIATORS, TRUST_SOURCES,
} from '@waggle/shared';
// ── Types ──────────────────────────────────────────────────────────────
// P7/D15 A2b: the audit vocabulary is now canonical in @waggle/shared. The
// Audit*-prefixed names are kept as aliases (re-exported) so every downstream
// `import { AuditAction, ... } from '@waggle/core'` keeps working unchanged.
// All six sets were already byte-identical to the shared ones (incl. P5/D4's
// 'uninstalled'), so this is a pure structural re-point — no value change.
export type {
AuditAction, AuditCapabilityType, AuditInitiator,
RiskLevel as AuditRiskLevel, ApprovalClass as AuditApprovalClass, TrustSource as AuditTrustSource,
} from '@waggle/shared';
import type {
AuditAction, AuditCapabilityType, AuditInitiator,
RiskLevel as AuditRiskLevel, ApprovalClass as AuditApprovalClass, TrustSource as AuditTrustSource,
} from '@waggle/shared';
export interface InstallAuditEntry {
id: number;
timestamp: string;
capability_name: string;
capability_type: AuditCapabilityType;
source: string;
version: string | null;
risk_level: AuditRiskLevel;
trust_source: AuditTrustSource;
approval_class: AuditApprovalClass;
action: AuditAction;
initiator: AuditInitiator;
detail: string;
}
export interface RecordAuditInput {
capabilityName: string;
capabilityType: AuditCapabilityType;
source: string;
version?: string | null;
riskLevel: AuditRiskLevel;
trustSource: AuditTrustSource;
approvalClass: AuditApprovalClass;
action: AuditAction;
initiator: AuditInitiator;
detail?: string;
}
// ── Table DDL ──────────────────────────────────────────────────────────
// P7/D15 A3: the CHECK lists are generated from the canonical @waggle/shared
// arrays via sqlInList, so the SQLite constraint and the TS union can no longer
// drift (divergence #14 — the old comment admitted "drift silently crashes
// record()"). The mirror DDL in hive-mind-core/src/mind/schema.ts stays a
// standalone literal (it's the OSS substrate, §7.5) but is locked to these same
// canonical lists by the parity test in install-audit-check-parity.test.ts.
// P7/D15 #15: trust_source now also carries a CHECK (was unconstrained at the DB
// while the TS type claimed a closed set). Every historical value came from the
// typed AuditTrustSource (the pre-security-gate 6-set ⊂ the current 7-set), so
// the rebuild migration's row copy can never violate it.
export const INSTALL_AUDIT_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS install_audit (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp TEXT NOT NULL DEFAULT (datetime('now')),
capability_name TEXT NOT NULL,
capability_type TEXT NOT NULL CHECK (capability_type IN (${sqlInList(AUDIT_CAPABILITY_TYPES)})),
source TEXT NOT NULL,
version TEXT,
risk_level TEXT NOT NULL CHECK (risk_level IN (${sqlInList(RISK_LEVELS)})),
trust_source TEXT NOT NULL CHECK (trust_source IN (${sqlInList(TRUST_SOURCES)})),
approval_class TEXT NOT NULL CHECK (approval_class IN (${sqlInList(APPROVAL_CLASSES)})),
action TEXT NOT NULL CHECK (action IN (${sqlInList(AUDIT_ACTIONS)})),
initiator TEXT NOT NULL CHECK (initiator IN (${sqlInList(AUDIT_INITIATORS)})),
detail TEXT NOT NULL DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_audit_capability ON install_audit (capability_name, action);
CREATE INDEX IF NOT EXISTS idx_audit_timestamp ON install_audit (timestamp DESC);
`;
// ── Store ──────────────────────────────────────────────────────────────
export class InstallAuditStore {
private db: MindDB;
constructor(db: MindDB) {
this.db = db;
this.ensureTable();
}
private ensureTable(): void {
const raw = this.db.getDatabase();
const existing = raw.prepare(
"SELECT sql FROM sqlite_master WHERE type='table' AND name='install_audit'",
).get() as { sql: string } | undefined;
if (!existing) {
raw.exec(INSTALL_AUDIT_TABLE_SQL);
return;
}
// P5/D4 migration: pre-'uninstalled' installs carry a narrower action CHECK
// baked into the table DDL. SQLite can't ALTER a CHECK, so rebuild the table
// when the stored DDL lacks the new value. Idempotent — a no-op once migrated.
// #15: also rebuild when the stored DDL has no trust_source CHECK (the column
// was previously unconstrained). "CHECK (trust_source IN" is a safe sentinel —
// it appears nowhere else in this DDL.
const needsActionWiden = !existing.sql.includes("'uninstalled'");
const needsTrustSourceCheck = !existing.sql.includes('CHECK (trust_source IN');
if (needsActionWiden || needsTrustSourceCheck) {
this.rebuildForWidenedActionCheck(raw);
}
}
/**
* Rebuild install_audit with the widened `action` CHECK, preserving all rows.
* Classic SQLite 12-step table redefinition, wrapped in a transaction so a
* crash mid-rebuild leaves the original table intact.
*/
private rebuildForWidenedActionCheck(raw: ReturnType<MindDB['getDatabase']>): void {
const migrate = raw.transaction(() => {
raw.exec('ALTER TABLE install_audit RENAME TO install_audit_legacy');
// SQLite carries indexes along with RENAME (still named idx_audit_*), so
// INSTALL_AUDIT_TABLE_SQL's CREATE INDEX IF NOT EXISTS would no-op and the
// DROP below would take the indexes with the legacy table. Drop them first
// (mirrors hive-mind-core db.ts) so they get recreated on the new table.
raw.exec('DROP INDEX IF EXISTS idx_audit_capability');
raw.exec('DROP INDEX IF EXISTS idx_audit_timestamp');
raw.exec(INSTALL_AUDIT_TABLE_SQL);
raw.exec(`
INSERT INTO install_audit (
id, timestamp, capability_name, capability_type, source, version,
risk_level, trust_source, approval_class, action, initiator, detail
)
SELECT id, timestamp, capability_name, capability_type, source, version,
risk_level, trust_source, approval_class, action, initiator, detail
FROM install_audit_legacy
`);
raw.exec('DROP TABLE install_audit_legacy');
});
migrate();
}
/** Record an install audit event. */
record(input: RecordAuditInput): InstallAuditEntry {
const raw = this.db.getDatabase();
raw.prepare(`
INSERT INTO install_audit (
capability_name, capability_type, source, version,
risk_level, trust_source, approval_class, action, initiator, detail
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
input.capabilityName,
input.capabilityType,
input.source,
input.version ?? null,
input.riskLevel,
input.trustSource,
input.approvalClass,
input.action,
input.initiator,
input.detail ?? '',
);
// Return the inserted row
return raw.prepare(
'SELECT * FROM install_audit ORDER BY id DESC LIMIT 1',
).get() as InstallAuditEntry;
}
/** Get audit history for a specific capability. */
getByCapability(name: string): InstallAuditEntry[] {
return this.db.getDatabase().prepare(
'SELECT * FROM install_audit WHERE capability_name = ? ORDER BY id DESC',
).all(name) as InstallAuditEntry[];
}
/** Get audit history filtered by action type. */
getByAction(action: AuditAction): InstallAuditEntry[] {
return this.db.getDatabase().prepare(
'SELECT * FROM install_audit WHERE action = ? ORDER BY id DESC',
).all(action) as InstallAuditEntry[];
}
/** Get recent audit entries (most recent first). */
getRecent(limit: number = 20): InstallAuditEntry[] {
return this.db.getDatabase().prepare(
'SELECT * FROM install_audit ORDER BY id DESC LIMIT ?',
).all(limit) as InstallAuditEntry[];
}
/** Get recent entries for one capability type (most recent first) — backs
* the shared Extend-layer audit read (GET /api/extend/audit?type=, C18). */
getRecentByType(type: AuditCapabilityType, limit: number = 20): InstallAuditEntry[] {
return this.db.getDatabase().prepare(
'SELECT * FROM install_audit WHERE capability_type = ? ORDER BY id DESC LIMIT ?',
).all(type, limit) as InstallAuditEntry[];
}
/** Get all entries (for testing). */
getAll(): InstallAuditEntry[] {
return this.db.getDatabase().prepare(
'SELECT * FROM install_audit ORDER BY id ASC',
).all() as InstallAuditEntry[];
}
/** Clear all entries (for testing). */
clear(): void {
this.db.getDatabase().prepare('DELETE FROM install_audit').run();
}
}

View File

@@ -0,0 +1,292 @@
/**
* Memory Import — parse ChatGPT and Claude exports, extract knowledge.
*
* Flow: raw JSON → parse conversations → extract knowledge items → ImportResult
*/
export type ImportSource = 'chatgpt' | 'claude';
export interface ConversationMessage {
role: 'user' | 'assistant';
text: string;
timestamp?: string;
}
export interface ParsedConversation {
title: string;
messages: ConversationMessage[];
createdAt?: string;
source: ImportSource;
}
export interface ExtractedKnowledge {
type: 'decision' | 'fact' | 'preference' | 'topic';
content: string;
source: ImportSource;
conversationTitle: string;
importance: 'important' | 'normal';
}
export interface ImportResult {
source: ImportSource;
conversationsFound: number;
conversationsParsed: number;
knowledgeExtracted: ExtractedKnowledge[];
errors: string[];
}
// ── Raw export shapes (untrusted external JSON; all fields optional) ──
/** A ChatGPT export node carries an optional message in its `mapping` entry. */
interface ChatGPTMessage {
author?: { role?: string };
content?: { parts?: unknown[] };
create_time?: number;
}
interface ChatGPTNode {
message?: ChatGPTMessage;
}
interface ChatGPTConversation {
title?: string;
mapping?: Record<string, ChatGPTNode>;
create_time?: number;
}
interface ClaudeMessage {
sender?: string;
role?: string;
text?: string;
content?: string;
created_at?: string;
timestamp?: string;
}
interface ClaudeConversation {
name?: string;
title?: string;
chat_messages?: ClaudeMessage[];
messages?: ClaudeMessage[];
created_at?: string;
create_time?: string;
}
/** Pull the `conversations` array out of an export that may be the array itself. */
function extractConversations(json: unknown): unknown[] {
if (Array.isArray(json)) return json;
if (json && typeof json === 'object' && 'conversations' in json) {
const c = (json as { conversations?: unknown }).conversations;
if (Array.isArray(c)) return c;
}
return [];
}
// ── Parsers ───────────────────────────────────────
export function parseChatGPTExport(json: unknown): ParsedConversation[] {
const conversations = extractConversations(json);
return conversations.map((raw) => {
const conv = raw as ChatGPTConversation;
const title = conv.title || 'Untitled';
const messages: ConversationMessage[] = [];
// ChatGPT uses a mapping object with node IDs
if (conv.mapping && typeof conv.mapping === 'object') {
const nodes = Object.values(conv.mapping);
// Sort by create_time for chronological order
const sorted = nodes
.filter((n) => (n?.message?.content?.parts?.length ?? 0) > 0)
.sort((a, b) => (a.message?.create_time ?? 0) - (b.message?.create_time ?? 0));
for (const node of sorted) {
const msg = node.message;
if (!msg || !msg.author?.role) continue;
const role = msg.author.role === 'user' ? 'user' : 'assistant';
if (msg.author.role === 'system') continue; // Skip system messages
const text = (msg.content?.parts ?? [])
.filter((p): p is string => typeof p === 'string')
.join('\n')
.trim();
if (!text) continue;
messages.push({
role,
text,
timestamp: msg.create_time ? new Date(msg.create_time * 1000).toISOString() : undefined,
});
}
}
return {
title,
messages,
createdAt: conv.create_time ? new Date(conv.create_time * 1000).toISOString() : undefined,
source: 'chatgpt' as ImportSource,
};
}).filter(c => c.messages.length > 0);
}
export function parseClaudeExport(json: unknown): ParsedConversation[] {
const conversations = extractConversations(json);
return conversations.map((raw) => {
const conv = raw as ClaudeConversation;
const title = conv.name || conv.title || 'Untitled';
const messages: ConversationMessage[] = [];
const chatMessages = conv.chat_messages ?? conv.messages ?? [];
for (const msg of chatMessages) {
const role = (msg.sender === 'human' || msg.role === 'user') ? 'user' : 'assistant';
const text = (msg.text ?? msg.content ?? '').trim();
if (!text) continue;
messages.push({
role,
text,
timestamp: msg.created_at ?? msg.timestamp,
});
}
return {
title,
messages,
createdAt: conv.created_at ?? conv.create_time,
source: 'claude' as ImportSource,
};
}).filter(c => c.messages.length > 0);
}
// ── Knowledge Extraction ──────────────────────────
const DECISION_PATTERNS = [
/\b(?:decided|chose|going with|we'll use|confirmed|agreed|settled on|picked|selected)\b/i,
/\b(?:decision|choice):\s/i,
/\blet's go with\b/i,
];
const PREFERENCE_PATTERNS = [
/\b(?:I prefer|I like|I always|I never|I don't like|don't use|use .+ instead)\b/i,
/\b(?:my preference|I'd rather|please always|please never|from now on)\b/i,
/\b(?:my style is|I want you to|format it as)\b/i,
];
const FACT_PATTERNS = [
/\b(?:my name is|I work at|I'm a|I am a|my company|my team|my role|our product|our stack)\b/i,
/\b(?:we use|our tech|we're building|the project is|the codebase)\b/i,
];
export function extractKnowledge(conversations: ParsedConversation[]): ExtractedKnowledge[] {
const knowledge: ExtractedKnowledge[] = [];
const seen = new Set<string>(); // Deduplicate
for (const conv of conversations) {
// Extract conversation topic
if (conv.title && conv.title !== 'Untitled' && conv.title.length > 5) {
const topicKey = `topic:${conv.title.toLowerCase().trim()}`;
if (!seen.has(topicKey)) {
seen.add(topicKey);
knowledge.push({
type: 'topic',
content: `Conversation topic: ${conv.title}`,
source: conv.source,
conversationTitle: conv.title,
importance: 'normal',
});
}
}
// Scan user messages for decisions, preferences, facts
for (const msg of conv.messages) {
if (msg.role !== 'user') continue;
if (msg.text.length < 10 || msg.text.length > 500) continue; // Skip too short/long
const sentences = msg.text.split(/[.!?\n]+/).map(s => s.trim()).filter(s => s.length > 10);
for (const sentence of sentences) {
const contentKey = sentence.toLowerCase().slice(0, 80);
if (seen.has(contentKey)) continue;
// Check for decisions
if (DECISION_PATTERNS.some(p => p.test(sentence))) {
seen.add(contentKey);
knowledge.push({
type: 'decision',
content: `Decision: ${sentence}`,
source: conv.source,
conversationTitle: conv.title,
importance: 'important',
});
continue;
}
// Check for preferences
if (PREFERENCE_PATTERNS.some(p => p.test(sentence))) {
seen.add(contentKey);
knowledge.push({
type: 'preference',
content: `Preference: ${sentence}`,
source: conv.source,
conversationTitle: conv.title,
importance: 'important',
});
continue;
}
// Check for facts
if (FACT_PATTERNS.some(p => p.test(sentence))) {
seen.add(contentKey);
knowledge.push({
type: 'fact',
content: `${sentence}`,
source: conv.source,
conversationTitle: conv.title,
importance: 'normal',
});
continue;
}
}
}
}
// Cap at 100 items to avoid polluting memory
return knowledge.slice(0, 100);
}
// ── Main Import Function ──────────────────────────
export function processImport(jsonData: unknown, source: ImportSource): ImportResult {
const errors: string[] = [];
let conversations: ParsedConversation[];
try {
conversations = source === 'chatgpt'
? parseChatGPTExport(jsonData)
: parseClaudeExport(jsonData);
} catch (err: unknown) {
const message = err instanceof Error ? err.message : String(err);
return {
source,
conversationsFound: 0,
conversationsParsed: 0,
knowledgeExtracted: [],
errors: [`Parse error: ${message}`],
};
}
if (conversations.length === 0) {
errors.push('No conversations found in export');
}
const knowledge = extractKnowledge(conversations);
return {
source,
conversationsFound: conversations.length,
conversationsParsed: conversations.filter(c => c.messages.length > 0).length,
knowledgeExtracted: knowledge,
errors,
};
}

View File

@@ -0,0 +1,45 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
/**
* Check if a migration from default.mind to personal.mind is needed.
* Returns true if default.mind exists AND personal.mind does NOT exist.
*/
export function needsMigration(waggleDir: string): boolean {
const defaultMind = path.join(waggleDir, 'default.mind');
const personalMind = path.join(waggleDir, 'personal.mind');
return fs.existsSync(defaultMind) && !fs.existsSync(personalMind);
}
/**
* Migrate from the old single-mind layout (default.mind) to the
* M4 multi-mind layout (personal.mind + workspaces/).
*
* Steps when migration is needed:
* 1. Copy default.mind -> personal.mind
* 2. Rename default.mind -> default.mind.bak
* 3. Create workspaces/ directory
*
* This is a pure file operation -- no data transformation is performed.
*/
export function migrateToMultiMind(waggleDir: string): { migrated: boolean; message: string } {
if (!needsMigration(waggleDir)) {
return { migrated: false, message: 'No migration needed' };
}
const defaultMind = path.join(waggleDir, 'default.mind');
const personalMind = path.join(waggleDir, 'personal.mind');
const backupMind = path.join(waggleDir, 'default.mind.bak');
const workspacesDir = path.join(waggleDir, 'workspaces');
// 1. Copy default.mind -> personal.mind
fs.copyFileSync(defaultMind, personalMind);
// 2. Rename default.mind -> default.mind.bak
fs.renameSync(defaultMind, backupMind);
// 3. Create workspaces/ directory (idempotent)
fs.mkdirSync(workspacesDir, { recursive: true });
return { migrated: true, message: 'Migrated default.mind to personal.mind' };
}

View File

@@ -0,0 +1,154 @@
/**
* Optimization Log Store — SQLite persistence for GEPA prompt optimization data.
*
* Captures agent interaction metadata (system prompts, tools used, turn counts,
* correction signals) for background analysis by the PromptOptimizer.
*
* Follows the same lazy-table pattern as CronStore and InstallAuditStore —
* operates on the .mind DB with auto-table creation.
*/
import type { MindDB } from '@waggle/hive-mind-core';
// ── Types ──────────────────────────────────────────────────────────────
export interface OptimizationLogEntry {
id: number;
session_id: string;
workspace_id: string;
system_prompt: string;
tools_used: string; // JSON array
turn_count: number;
was_correction: number; // SQLite integer boolean
input_tokens: number;
output_tokens: number;
timestamp: string;
}
export interface CreateOptimizationLogInput {
sessionId: string;
workspaceId: string;
systemPrompt: string;
toolsUsed: string[];
turnCount: number;
wasCorrection: boolean;
inputTokens?: number;
outputTokens?: number;
}
// ── Table DDL ──────────────────────────────────────────────────────────
export const OPTIMIZATION_LOG_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS optimization_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
session_id TEXT NOT NULL,
workspace_id TEXT NOT NULL,
system_prompt TEXT NOT NULL,
tools_used TEXT NOT NULL DEFAULT '[]',
turn_count INTEGER NOT NULL DEFAULT 0,
was_correction INTEGER NOT NULL DEFAULT 0,
input_tokens INTEGER NOT NULL DEFAULT 0,
output_tokens INTEGER NOT NULL DEFAULT 0,
timestamp TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_optlog_workspace ON optimization_log (workspace_id, timestamp DESC);
CREATE INDEX IF NOT EXISTS idx_optlog_timestamp ON optimization_log (timestamp DESC);
`;
// ── Store ──────────────────────────────────────────────────────────────
export class OptimizationLogStore {
private db: MindDB;
constructor(db: MindDB) {
this.db = db;
this.ensureTable();
}
private ensureTable(): void {
const raw = this.db.getDatabase();
const exists = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='optimization_log'",
).get();
if (!exists) {
raw.exec(OPTIMIZATION_LOG_TABLE_SQL);
}
}
/** Insert a new optimization log entry capturing an agent interaction. */
insert(input: CreateOptimizationLogInput): OptimizationLogEntry {
const raw = this.db.getDatabase();
const result = raw.prepare(`
INSERT INTO optimization_log (session_id, workspace_id, system_prompt, tools_used, turn_count, was_correction, input_tokens, output_tokens)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`).run(
input.sessionId,
input.workspaceId,
input.systemPrompt,
JSON.stringify(input.toolsUsed),
input.turnCount,
input.wasCorrection ? 1 : 0,
input.inputTokens ?? 0,
input.outputTokens ?? 0,
);
return raw.prepare(
'SELECT * FROM optimization_log WHERE id = ?',
).get(result.lastInsertRowid) as OptimizationLogEntry;
}
/** Get recent optimization log entries, ordered by most recent first. */
getRecent(limit: number = 50): OptimizationLogEntry[] {
return this.db.getDatabase().prepare(
'SELECT * FROM optimization_log ORDER BY timestamp DESC, id DESC LIMIT ?',
).all(limit) as OptimizationLogEntry[];
}
/** Get log entries for a specific workspace. */
getByWorkspace(workspaceId: string, limit: number = 50): OptimizationLogEntry[] {
return this.db.getDatabase().prepare(
'SELECT * FROM optimization_log WHERE workspace_id = ? ORDER BY timestamp DESC, id DESC LIMIT ?',
).all(workspaceId, limit) as OptimizationLogEntry[];
}
/** Get aggregate stats: total entries, correction rate, avg turn count, total tokens. */
getStats(): { total: number; correctionRate: number; avgTurnCount: number; totalInputTokens: number; totalOutputTokens: number } {
const raw = this.db.getDatabase();
const row = raw.prepare(`
SELECT
COUNT(*) as total,
COALESCE(AVG(was_correction * 1.0), 0) as correction_rate,
COALESCE(AVG(turn_count), 0) as avg_turn_count,
COALESCE(SUM(input_tokens), 0) as total_input_tokens,
COALESCE(SUM(output_tokens), 0) as total_output_tokens
FROM optimization_log
`).get() as {
total: number;
correction_rate: number;
avg_turn_count: number;
total_input_tokens: number;
total_output_tokens: number;
};
return {
total: row.total,
correctionRate: row.correction_rate,
avgTurnCount: row.avg_turn_count,
totalInputTokens: row.total_input_tokens,
totalOutputTokens: row.total_output_tokens,
};
}
/** Delete entries older than the given number of days. */
pruneOlderThan(days: number): number {
const result = this.db.getDatabase().prepare(
`DELETE FROM optimization_log WHERE timestamp < datetime('now', '-' || ? || ' days')`,
).run(days);
return result.changes;
}
/** Clear all entries (for testing). */
clear(): void {
this.db.getDatabase().prepare('DELETE FROM optimization_log').run();
}
}

View File

@@ -0,0 +1,122 @@
/**
* Skill Hash Store — SHA-256 content hashing for skill update detection.
*
* Stores hashes of skill file content in the .mind DB. On server startup,
* compares current file hashes to stored hashes to detect changes.
* Follows the same ensureTable pattern as CronStore / InstallAuditStore.
*/
import * as crypto from 'node:crypto';
import type { MindDB } from '@waggle/hive-mind-core';
// ── Types ──────────────────────────────────────────────────────────────
export interface SkillHash {
name: string;
hash: string;
verified_at: string;
}
// ── Table DDL ──────────────────────────────────────────────────────────
export const SKILL_HASHES_TABLE_SQL = `
CREATE TABLE IF NOT EXISTS skill_hashes (
name TEXT PRIMARY KEY,
hash TEXT NOT NULL,
verified_at TEXT NOT NULL DEFAULT (datetime('now'))
);
`;
// ── Helpers ────────────────────────────────────────────────────────────
/** Compute SHA-256 hex digest of skill content. */
export function computeSkillHash(content: string): string {
return crypto.createHash('sha256').update(content).digest('hex');
}
// ── Store ──────────────────────────────────────────────────────────────
export class SkillHashStore {
private db: MindDB;
constructor(db: MindDB) {
this.db = db;
this.ensureTable();
}
private ensureTable(): void {
const raw = this.db.getDatabase();
const exists = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='skill_hashes'",
).get();
if (!exists) {
raw.exec(SKILL_HASHES_TABLE_SQL);
}
}
/** Store or update hash for a skill. */
setHash(name: string, hash: string): void {
this.db.getDatabase().prepare(`
INSERT INTO skill_hashes (name, hash) VALUES (?, ?)
ON CONFLICT (name) DO UPDATE SET hash = ?, verified_at = datetime('now')
`).run(name, hash, hash);
}
/** Get stored hash for a skill. */
getHash(name: string): SkillHash | undefined {
return this.db.getDatabase().prepare(
'SELECT * FROM skill_hashes WHERE name = ?',
).get(name) as SkillHash | undefined;
}
/** Remove hash (when skill is deleted). */
removeHash(name: string): void {
this.db.getDatabase().prepare('DELETE FROM skill_hashes WHERE name = ?').run(name);
}
/**
* Check all skills against stored hashes.
* Returns lists of changed, added, and removed skill names.
*/
checkAll(currentSkills: Array<{ name: string; content: string }>): {
changed: string[];
added: string[];
removed: string[];
} {
const storedHashes = this.db.getDatabase().prepare(
'SELECT name, hash FROM skill_hashes',
).all() as SkillHash[];
const storedMap = new Map(storedHashes.map(h => [h.name, h.hash]));
const currentNames = new Set(currentSkills.map(s => s.name));
const changed: string[] = [];
const added: string[] = [];
for (const skill of currentSkills) {
const currentHash = computeSkillHash(skill.content);
const storedHash = storedMap.get(skill.name);
if (!storedHash) {
added.push(skill.name);
} else if (storedHash !== currentHash) {
changed.push(skill.name);
}
}
const removed = storedHashes
.filter(h => !currentNames.has(h.name))
.map(h => h.name);
return { changed, added, removed };
}
/** Mark a skill as verified (update hash to current content). */
verify(name: string, content: string): void {
this.setHash(name, computeSkillHash(content));
}
/** Clear all hashes (for testing). */
clear(): void {
this.db.getDatabase().prepare('DELETE FROM skill_hashes').run();
}
}

View File

@@ -0,0 +1,186 @@
/**
* TeamSync — Syncs memory frames between local .mind and team server.
*
* Uses the team server's entities API (entityType='memory_frame') for storage.
* Push-on-write: new frames are pushed to team server after local write.
* Pull-on-activate: frames are pulled from team server when workspace is activated.
* Attribution: each frame carries the author's userId and displayName.
*
* Sync protocol:
* - Frames are identified by a composite key: gopId + t (group-of-pictures ID + temporal index)
* - Pull uses ?type=memory_frame&since=<timestamp> to get only new frames
* - Push sends each frame as a teamEntity with entityType='memory_frame'
* - Last-write-wins at frame level (frames are append-mostly, conflicts are rare)
*/
import type { MemoryFrame, FrameType, Importance } from '@waggle/hive-mind-core';
import { createCoreLogger } from '@waggle/hive-mind-core';
const log = createCoreLogger('team-sync');
export interface TeamSyncConfig {
teamServerUrl: string;
teamSlug: string;
authToken: string;
userId: string;
displayName: string;
}
export interface SyncedFrame {
/** Server-side entity ID (UUID). */
remoteId: string;
gopId: string;
t: number;
frameType: FrameType;
content: string;
importance: Importance;
authorId: string;
authorName: string;
createdAt: string;
}
/**
* Convert a local MemoryFrame to the team server entity format.
*/
export function frameToEntity(frame: MemoryFrame, authorId: string, authorName: string) {
return {
entityType: 'memory_frame',
name: frame.gop_id,
properties: {
frameType: frame.frame_type,
t: frame.t,
baseFrameId: frame.base_frame_id,
content: frame.content,
importance: frame.importance,
authorId,
authorName,
localId: frame.id,
},
};
}
/**
* Convert a team server entity back to a SyncedFrame.
*/
export function entityToSyncedFrame(entity: {
id: string;
name: string;
properties: Record<string, unknown>;
createdAt: string;
}): SyncedFrame {
const props = entity.properties;
return {
remoteId: entity.id,
gopId: entity.name,
t: (props.t as number) ?? 0,
frameType: (props.frameType as FrameType) ?? 'I',
content: (props.content as string) ?? '',
importance: (props.importance as Importance) ?? 'normal',
authorId: (props.authorId as string) ?? '',
authorName: (props.authorName as string) ?? '',
createdAt: entity.createdAt,
};
}
/**
* TeamSync handles push/pull of memory frames to/from the team server.
* This is a stateless client — call pushFrame() after writing locally,
* call pullFrames() on workspace activation.
*/
export class TeamSync {
private config: TeamSyncConfig;
private lastSyncTimestamp: string | null = null;
constructor(config: TeamSyncConfig) {
this.config = config;
}
/**
* Push a single frame to the team server.
* Call this after writing a frame to the local workspace .mind.
*/
async pushFrame(frame: MemoryFrame): Promise<{ remoteId: string } | null> {
const entity = frameToEntity(frame, this.config.userId, this.config.displayName);
try {
const response = await fetch(
`${this.config.teamServerUrl}/api/teams/${this.config.teamSlug}/entities`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${this.config.authToken}`,
},
body: JSON.stringify(entity),
}
);
if (!response.ok) {
log.error(`Push failed: ${response.status} ${response.statusText}`);
return null;
}
const created = await response.json() as { id: string };
return { remoteId: created.id };
} catch (err) {
log.error('Push error:', err);
return null;
}
}
/**
* Pull all memory frames from the team server.
* Returns frames sorted by creation time (newest first).
* Optionally pass `since` to get only frames created after that timestamp.
*/
async pullFrames(since?: string): Promise<SyncedFrame[]> {
try {
const url = `${this.config.teamServerUrl}/api/teams/${this.config.teamSlug}/entities?type=memory_frame`;
// Note: `since` filtering would require server-side support. For now, pull all and filter client-side.
const response = await fetch(url, {
headers: {
'Authorization': `Bearer ${this.config.authToken}`,
},
});
if (!response.ok) {
log.error(`Pull failed: ${response.status} ${response.statusText}`);
return [];
}
const entities = await response.json() as Array<{
id: string;
name: string;
properties: Record<string, unknown>;
createdAt: string;
}>;
const frames = entities.map(entityToSyncedFrame);
// Client-side filter by timestamp if requested
if (since) {
return frames.filter(f => f.createdAt > since);
}
this.lastSyncTimestamp = new Date().toISOString();
return frames;
} catch (err) {
log.error('Pull error:', err);
return [];
}
}
/**
* Get the last sync timestamp (for incremental sync).
*/
getLastSyncTimestamp(): string | null {
return this.lastSyncTimestamp;
}
/**
* Set the last sync timestamp (e.g. loaded from persistent storage).
*/
setLastSyncTimestamp(ts: string): void {
this.lastSyncTimestamp = ts;
}
}

View File

@@ -0,0 +1,324 @@
/**
* TelemetryStore — local, privacy-first event tracking.
*
* All data stays in ~/.waggle/telemetry.db (SQLite).
* Default: OFF. User opts in via Settings toggle.
* No cloud reporting in M2 — data is queryable via local API only.
*
* NEVER tracked: message content, memory content, file paths,
* API keys, personal info, IP addresses, device IDs.
*/
import Database from 'better-sqlite3';
import type { Database as DatabaseType } from 'better-sqlite3';
import path from 'node:path';
import fs from 'node:fs';
/* ── Schema ── */
const TELEMETRY_SCHEMA = `
CREATE TABLE IF NOT EXISTS telemetry_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
event TEXT NOT NULL,
properties TEXT NOT NULL DEFAULT '{}',
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_telemetry_event ON telemetry_events (event, created_at);
CREATE INDEX IF NOT EXISTS idx_telemetry_date ON telemetry_events (created_at);
`;
/* ── Event constants ── */
export const TELEMETRY_EVENTS = {
// Onboarding funnel
ONBOARDING_START: 'onboarding_start',
ONBOARDING_STEP: 'onboarding_step',
ONBOARDING_COMPLETE: 'onboarding_complete',
ONBOARDING_SKIP: 'onboarding_skip',
// Session engagement
SESSION_START: 'session_start',
SESSION_END: 'session_end',
// Infrastructure
EMBEDDING_PROVIDER: 'embedding_provider',
LLM_PROVIDER: 'llm_provider',
// Feature usage
TEMPLATE_SELECTED: 'template_selected',
WORKSPACE_CREATED: 'workspace_created',
FIRST_AGENT_RESPONSE: 'first_agent_response',
SLASH_COMMAND_USED: 'slash_command_used',
// App lifecycle
APP_START: 'app_start',
APP_ERROR: 'app_error',
} as const;
/* ── Types ── */
export interface TelemetryEvent {
id: number;
event: string;
properties: Record<string, unknown>;
created_at: string;
}
export interface TelemetrySummary {
enabled: boolean;
totalEvents: number;
firstEvent: string | null;
lastEvent: string | null;
onboardingCompleted: boolean;
totalSessions: number;
embeddingProvider: string | null;
templatesUsed: string[];
eventBreakdown: Record<string, number>;
}
/* ── Store ── */
export class TelemetryStore {
private db: DatabaseType;
private enabled: boolean;
constructor(dataDir: string, enabled = false) {
this.enabled = enabled;
const dbPath = path.join(dataDir, 'telemetry.db');
this.db = new Database(dbPath);
this.db.pragma('journal_mode = WAL');
this.db.exec(TELEMETRY_SCHEMA);
}
/** Track an event. No-op if telemetry is disabled. */
track(event: string, properties?: Record<string, unknown>): void {
if (!this.enabled) return;
this.db.prepare(
'INSERT INTO telemetry_events (event, properties) VALUES (?, ?)'
).run(event, JSON.stringify(properties ?? {}));
}
/** Enable/disable telemetry at runtime. */
setEnabled(enabled: boolean): void {
this.enabled = enabled;
}
/** Whether telemetry is currently enabled. */
isEnabled(): boolean {
return this.enabled;
}
/** Get aggregated summary. */
getSummary(): TelemetrySummary {
const total = (this.db.prepare(
'SELECT COUNT(*) as cnt FROM telemetry_events'
).get() as { cnt: number }).cnt;
const first = this.db.prepare(
'SELECT MIN(created_at) as d FROM telemetry_events'
).get() as { d: string | null };
const last = this.db.prepare(
'SELECT MAX(created_at) as d FROM telemetry_events'
).get() as { d: string | null };
const onboarded = (this.db.prepare(
"SELECT COUNT(*) as cnt FROM telemetry_events WHERE event = 'onboarding_complete'"
).get() as { cnt: number }).cnt > 0;
const sessions = (this.db.prepare(
"SELECT COUNT(*) as cnt FROM telemetry_events WHERE event = 'session_start'"
).get() as { cnt: number }).cnt;
const embRow = this.db.prepare(
"SELECT properties FROM telemetry_events WHERE event = 'embedding_provider' ORDER BY created_at DESC LIMIT 1"
).get() as { properties: string } | undefined;
const embProvider = embRow
? (JSON.parse(embRow.properties) as Record<string, unknown>).provider as string ?? null
: null;
const tplRows = this.db.prepare(
"SELECT DISTINCT json_extract(properties, '$.templateId') as tid FROM telemetry_events WHERE event IN ('template_selected', 'workspace_created') AND json_extract(properties, '$.templateId') IS NOT NULL"
).all() as Array<{ tid: string }>;
const breakdownRows = this.db.prepare(
'SELECT event, COUNT(*) as cnt FROM telemetry_events GROUP BY event ORDER BY cnt DESC'
).all() as Array<{ event: string; cnt: number }>;
const breakdown: Record<string, number> = {};
for (const row of breakdownRows) {
breakdown[row.event] = row.cnt;
}
return {
enabled: this.enabled,
totalEvents: total,
firstEvent: first.d,
lastEvent: last.d,
onboardingCompleted: onboarded,
totalSessions: sessions,
embeddingProvider: embProvider,
templatesUsed: tplRows.map(r => r.tid),
eventBreakdown: breakdown,
};
}
/** Get raw events with optional filters. */
getEvents(options?: { event?: string; since?: string; until?: string; limit?: number }): TelemetryEvent[] {
const conditions: string[] = [];
const params: unknown[] = [];
if (options?.event) {
conditions.push('event = ?');
params.push(options.event);
}
if (options?.since) {
conditions.push('created_at >= ?');
params.push(options.since);
}
if (options?.until) {
conditions.push('created_at <= ?');
params.push(options.until);
}
const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
const limit = options?.limit ?? 100;
const rows = this.db.prepare(
`SELECT id, event, properties, created_at FROM telemetry_events ${where} ORDER BY created_at DESC LIMIT ?`
).all(...params, limit) as Array<{ id: number; event: string; properties: string; created_at: string }>;
return rows.map(r => ({
id: r.id,
event: r.event,
properties: JSON.parse(r.properties) as Record<string, unknown>,
created_at: r.created_at,
}));
}
/** Delete all telemetry data (user right-to-delete). */
clear(): { deleted: number } {
const result = this.db.prepare('DELETE FROM telemetry_events').run();
return { deleted: result.changes };
}
/** Close the database connection. */
close(): void {
this.db.close();
}
}
/* ── Collector ── */
const COLLECTOR_SCHEMA = `
CREATE TABLE IF NOT EXISTS collector_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
category TEXT NOT NULL,
name TEXT NOT NULL,
count INTEGER NOT NULL DEFAULT 1,
date TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_collector_date ON collector_events (date);
CREATE UNIQUE INDEX IF NOT EXISTS idx_collector_name_date ON collector_events (name, date);
`;
/**
* TelemetryCollector — higher-level daily-aggregated event tracker.
*
* Records tool usage, commands, errors, and capability gaps by day.
* Privacy-safe: only category + name + count, no content or PII.
*/
export class TelemetryCollector {
private db: DatabaseType;
private enabled: boolean;
private dataDir: string;
constructor(dataDir: string, enabled = false) {
this.dataDir = dataDir;
this.enabled = enabled;
const dbPath = path.join(dataDir, 'telemetry.db');
this.db = new Database(dbPath);
this.db.pragma('journal_mode = WAL');
this.db.exec(COLLECTOR_SCHEMA);
}
recordToolUse(toolName: string): void {
if (!this.enabled) return;
this.upsertEvent('tool', toolName);
}
recordCommand(command: string): void {
if (!this.enabled) return;
this.upsertEvent('command', command);
}
recordError(errorName: string): void {
if (!this.enabled) return;
this.upsertEvent('error', errorName);
}
recordCapabilityGap(capability: string): void {
if (!this.enabled) return;
this.upsertEvent('capability_gap', capability);
}
recordSession(durationMs: number, interactionCount: number): void {
if (!this.enabled) return;
const today = new Date().toISOString().split('T')[0];
this.db.prepare(
`INSERT INTO collector_events (category, name, count, date) VALUES ('session', 'duration_total_ms', ?, ?)
ON CONFLICT(name, date) DO UPDATE SET count = count + excluded.count`
).run(durationMs, today);
this.db.prepare(
`INSERT INTO collector_events (category, name, count, date) VALUES ('session', 'interaction_count', ?, ?)
ON CONFLICT(name, date) DO UPDATE SET count = count + excluded.count`
).run(interactionCount, today);
this.db.prepare(
`INSERT INTO collector_events (category, name, count, date) VALUES ('session', 'session_count', 1, ?)
ON CONFLICT(name, date) DO UPDATE SET count = count + 1`
).run(today);
}
private upsertEvent(category: string, name: string): void {
const today = new Date().toISOString().split('T')[0];
this.db.prepare(
`INSERT INTO collector_events (category, name, count, date) VALUES (?, ?, 1, ?)
ON CONFLICT(name, date) DO UPDATE SET count = count + 1`
).run(category, name, today);
}
getReport(days = 7): {
totalEvents: number;
events: Array<{ name: string; count: number; category: string; date: string }>;
dateRange: { from: string | null };
} {
const since = new Date();
since.setDate(since.getDate() - days);
const sinceStr = since.toISOString().split('T')[0];
const rows = this.db.prepare(
`SELECT name, SUM(count) as total_count, category, date
FROM collector_events WHERE date >= ?
GROUP BY name, category, date ORDER BY date DESC, name`
).all(sinceStr) as Array<{ name: string; total_count: number; category: string; date: string }>;
const totalEvents = rows.reduce((sum, r) => sum + r.total_count, 0);
const events = rows.map(r => ({ name: r.name, count: r.total_count, category: r.category, date: r.date }));
const firstRow = this.db.prepare(
'SELECT MIN(date) as first_date FROM collector_events'
).get() as { first_date: string | null };
return { totalEvents, events, dateRange: { from: firstRow.first_date } };
}
/** Write events to telemetry.json for external consumption. */
flush(): void {
const rows = this.db.prepare(
'SELECT name, count, category, date FROM collector_events ORDER BY date DESC, name'
).all() as Array<{ name: string; count: number; category: string; date: string }>;
fs.writeFileSync(
path.join(this.dataDir, 'telemetry.json'),
JSON.stringify(rows, null, 2),
);
}
setEnabled(enabled: boolean): void { this.enabled = enabled; }
isEnabled(): boolean { return this.enabled; }
close(): void { this.db.close(); }
}

299
packages/core/src/vault.ts Normal file
View File

@@ -0,0 +1,299 @@
/**
* VaultStore — encrypted local secrets store.
*
* Encrypts secrets using AES-256-GCM (Node.js built-in crypto).
* Stores encrypted data in vault.json with a machine-local key file.
* Each entry is independently encrypted so individual secrets can be
* updated without re-encrypting everything.
*/
import * as crypto from 'node:crypto';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { execFileSync } from 'node:child_process';
import { createCoreLogger } from '@waggle/hive-mind-core';
const log = createCoreLogger('vault');
const ALGORITHM = 'aes-256-gcm';
const KEY_LENGTH = 32;
const IV_LENGTH = 16;
export interface VaultEntry {
name: string;
value: string;
metadata?: Record<string, unknown>;
updatedAt: string;
}
interface VaultRecord {
encrypted: string;
metadata?: Record<string, unknown>;
updatedAt: string;
}
export class VaultStore {
private dataDir: string;
private vaultPath: string;
private keyPath: string;
private encryptionKey: Buffer;
constructor(dataDir: string) {
this.dataDir = dataDir;
this.vaultPath = path.join(dataDir, 'vault.json');
this.keyPath = path.join(dataDir, '.vault-key');
// Ensure data directory exists
if (!fs.existsSync(this.dataDir)) {
fs.mkdirSync(this.dataDir, { recursive: true });
}
this.encryptionKey = this.ensureKey();
}
/** Ensure the encryption key exists. Generate if missing. */
private ensureKey(): Buffer {
if (fs.existsSync(this.keyPath)) {
const key = Buffer.from(fs.readFileSync(this.keyPath, 'utf-8').trim(), 'hex');
if (key.length !== KEY_LENGTH) {
throw new Error(
`Vault key file is corrupted — expected ${KEY_LENGTH} bytes, got ${key.length}. Delete ${this.keyPath} to regenerate.`
);
}
return key;
}
const key = crypto.randomBytes(KEY_LENGTH);
fs.writeFileSync(this.keyPath, key.toString('hex'), { mode: 0o600 });
// Review Critical #1: On Windows, restrict key file access to current user only.
// Previously used `require('node:child_process')` inline which fails silently under
// ESM (`type: module` in the sidecar) — the try/catch swallowed the ReferenceError
// and every Windows install left the vault key with no ACL restriction.
// Now imported statically at the top of the file; the try/catch only covers actual
// icacls failures (e.g. icacls.exe not on PATH in a minimal Windows image).
if (process.platform === 'win32') {
try {
// Resolve the current user via whoami (safer than process.env.USERNAME
// which can be absent or spoofed in containerized/scripted setups)
const currentUser = execFileSync('whoami', { encoding: 'utf-8' }).trim();
execFileSync('icacls', [
this.keyPath,
'/inheritance:r',
'/grant:r',
`${currentUser}:F`,
], { stdio: 'ignore' });
} catch (err) {
log.warn('Could not restrict key file permissions via icacls — vault key may be readable by other users', err);
}
}
return key;
}
/** Encrypt a plaintext string. Returns iv:authTag:ciphertext (all hex). */
private encrypt(plaintext: string): string {
const iv = crypto.randomBytes(IV_LENGTH);
const cipher = crypto.createCipheriv(ALGORITHM, this.encryptionKey, iv);
const encrypted = Buffer.concat([cipher.update(plaintext, 'utf-8'), cipher.final()]);
const authTag = cipher.getAuthTag();
return `${iv.toString('hex')}:${authTag.toString('hex')}:${encrypted.toString('hex')}`;
}
/** Decrypt an encoded string (iv:authTag:ciphertext, all hex). */
private decrypt(encoded: string): string {
const parts = encoded.split(':');
if (parts.length !== 3) {
throw new Error('Vault entry format invalid — expected iv:authTag:ciphertext');
}
const [ivHex, authTagHex, ciphertextHex] = parts;
const iv = Buffer.from(ivHex, 'hex');
const authTag = Buffer.from(authTagHex, 'hex');
const ciphertext = Buffer.from(ciphertextHex, 'hex');
const decipher = crypto.createDecipheriv(ALGORITHM, this.encryptionKey, iv);
decipher.setAuthTag(authTag);
return decipher.update(ciphertext) + decipher.final('utf-8');
}
/** Read the vault file (encrypted entries). */
private readVault(): Record<string, VaultRecord> {
if (!fs.existsSync(this.vaultPath)) return {};
try {
// Review Critical #3: use a null-prototype object so a malicious `__proto__`
// key in vault.json cannot pollute Object.prototype when accessed. Matches
// the `name` validation at the route layer.
const parsed = JSON.parse(fs.readFileSync(this.vaultPath, 'utf-8'));
return Object.assign(Object.create(null), parsed);
} catch (err) {
// M5: corrupt vault.json — back up the corrupt file so data isn't silently lost
log.error('Failed to parse vault.json — backing up corrupt file', err);
const bakPath = this.vaultPath + '.bak';
try { fs.copyFileSync(this.vaultPath, bakPath); } catch { /* best effort */ }
return {};
}
}
/** Write the vault file atomically (write to .tmp, then replace). */
private writeVault(vault: Record<string, VaultRecord>): void {
const tmpPath = this.vaultPath + '.tmp';
const data = JSON.stringify(vault, null, 2);
fs.writeFileSync(tmpPath, data, { mode: 0o600 });
try {
try {
fs.renameSync(tmpPath, this.vaultPath);
} catch {
// M6: On Windows, rename fails with EPERM when target exists.
// Delete target first, then rename — preserves atomicity better than direct write.
try { fs.unlinkSync(this.vaultPath); } catch { /* target may not exist */ }
try {
fs.renameSync(tmpPath, this.vaultPath);
} catch (renameErr) {
// Last resort: direct write (non-atomic) — log the degradation
log.error('Atomic vault write failed, falling back to direct write', renameErr);
fs.writeFileSync(this.vaultPath, data, { mode: 0o600 });
}
}
} finally {
// M12: clean up .tmp if it still exists after any failure path
try { if (fs.existsSync(tmpPath)) fs.unlinkSync(tmpPath); } catch { /* best effort */ }
}
}
/** Set a secret (encrypts value, stores metadata alongside). Uses atomic write. */
set(name: string, value: string, metadata?: Record<string, unknown>): void {
const vault = this.readVault();
vault[name] = {
encrypted: this.encrypt(value),
metadata,
updatedAt: new Date().toISOString(),
};
this.writeVault(vault);
}
/**
* Async set — delegates to the synchronous set().
* Review Critical #2: the previous implementation used a promise chain
* (.writeLock.then(...)) which deferred the actual write to a microtask.
* A sync set() between the chain call and microtask execution would be
* overwritten when the deferred write fired. Since all vault I/O is
* synchronous (fs.readFileSync/writeFileSync), the lock is unnecessary —
* each callback runs to completion before the next microtask. Delegating
* to the sync method eliminates the interleaving window entirely.
*/
async setAsync(name: string, value: string, metadata?: Record<string, unknown>): Promise<void> {
this.set(name, value, metadata);
}
/** Get a decrypted secret by name. Returns null if not found or decryption fails. */
get(name: string): VaultEntry | null {
const vault = this.readVault();
const entry = vault[name];
if (!entry) return null;
try {
return {
name,
value: this.decrypt(entry.encrypted),
metadata: entry.metadata,
updatedAt: entry.updatedAt,
};
} catch {
return null; // Decryption failed (key mismatch, corrupted)
}
}
/** Delete a secret. Returns true if it existed. Uses atomic write. */
delete(name: string): boolean {
const vault = this.readVault();
if (!vault[name]) return false;
delete vault[name];
this.writeVault(vault);
return true;
}
/**
* Async delete — delegates to the synchronous delete().
* Review Critical #2: same rationale as setAsync — sync I/O means no
* interleaving risk. The previous promise-chain approach deferred the
* actual delete, creating a window where a sync caller's write could
* be overwritten.
*/
async deleteAsync(name: string): Promise<boolean> {
return this.delete(name);
}
/** List secret names (without values). */
list(): Array<{ name: string; metadata?: Record<string, unknown>; updatedAt: string }> {
const vault = this.readVault();
return Object.entries(vault).map(([name, entry]) => ({
name,
metadata: entry.metadata,
updatedAt: entry.updatedAt,
}));
}
/** Check if a secret exists. */
has(name: string): boolean {
const vault = this.readVault();
return name in vault;
}
/** Set a connector credential with typed metadata */
setConnectorCredential(connectorId: string, credential: {
type: 'api_key' | 'oauth2' | 'bearer' | 'basic';
value: string;
refreshToken?: string;
expiresAt?: string;
scopes?: string[];
}): void {
this.set(`connector:${connectorId}`, credential.value, {
credentialType: credential.type,
expiresAt: credential.expiresAt,
scopes: credential.scopes,
});
// Store refresh token as a separate encrypted entry (never in plaintext metadata)
if (credential.refreshToken) {
this.set(`connector:${connectorId}:refresh`, credential.refreshToken);
} else {
// Clear any previously stored refresh token if not provided
this.delete(`connector:${connectorId}:refresh`);
}
}
/** Get a connector credential with typed metadata */
getConnectorCredential(connectorId: string): {
value: string;
type: string;
refreshToken?: string;
expiresAt?: string;
scopes?: string[];
isExpired: boolean;
} | null {
const entry = this.get(`connector:${connectorId}`);
if (!entry) return null;
const expiresAt = entry.metadata?.expiresAt as string | undefined;
// Retrieve refresh token from its own encrypted entry
const refreshEntry = this.get(`connector:${connectorId}:refresh`);
return {
value: entry.value,
type: (entry.metadata?.credentialType as string) ?? 'api_key',
// Review Major #7: removed dead plaintext metadata fallback that contradicted
// the "never in plaintext metadata" security contract from setConnectorCredential.
refreshToken: refreshEntry?.value,
expiresAt,
scopes: entry.metadata?.scopes as string[] | undefined,
isExpired: expiresAt ? new Date(expiresAt) < new Date() : false,
};
}
/** Migrate plaintext providers from config.json to vault. Returns count migrated. */
migrateFromConfig(config: { providers?: Record<string, { apiKey: string; models?: string[]; baseUrl?: string }> }): number {
if (!config.providers) return 0;
let migrated = 0;
for (const [name, provider] of Object.entries(config.providers)) {
if (provider.apiKey && !this.has(name)) {
this.set(name, provider.apiKey, {
models: provider.models,
baseUrl: provider.baseUrl,
});
migrated++;
}
}
return migrated;
}
}