This commit is contained in:
5
packages/core/src/compliance/index.ts
Normal file
5
packages/core/src/compliance/index.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
export * from './types.js';
|
||||
export { InteractionStore } from './interaction-store.js';
|
||||
export { ComplianceStatusChecker } from './status-checker.js';
|
||||
export { ReportGenerator } from './report-generator.js';
|
||||
export { ComplianceTemplateStore } from './template-store.js';
|
||||
208
packages/core/src/compliance/interaction-store.ts
Normal file
208
packages/core/src/compliance/interaction-store.ts
Normal file
@@ -0,0 +1,208 @@
|
||||
/**
|
||||
* InteractionStore — CRUD for ai_interactions table (EU AI Act Art. 12).
|
||||
*
|
||||
* Records every AI interaction with model, tokens, cost, tools, and human oversight.
|
||||
* Operates on the .mind DB alongside FrameStore and InstallAuditStore.
|
||||
*/
|
||||
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
import type { AIInteraction, RecordInteractionInput, HumanAction, ModelInventoryEntry, OversightLogEntry } from './types.js';
|
||||
|
||||
export class InteractionStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
// Review Major #8: previously ensureTable() duplicated the DDL from schema.ts and
|
||||
// drifted whenever the canonical schema changed. Now schema.ts + MindDB.runMigrations()
|
||||
// own the table definition, including the input_text/output_text columns and the
|
||||
// append-only triggers (Critical #1, #3).
|
||||
}
|
||||
|
||||
/** Record an AI interaction event. */
|
||||
record(input: RecordInteractionInput): AIInteraction {
|
||||
const raw = this.db.getDatabase();
|
||||
|
||||
// Review Major #4: use lastInsertRowid from .run() instead of reading back
|
||||
// ORDER BY id DESC LIMIT 1 — under concurrent writes (WaggleDance multi-agent),
|
||||
// the LIMIT 1 row may belong to a different writer.
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO ai_interactions (
|
||||
workspace_id, session_id, model, provider,
|
||||
input_tokens, output_tokens, cost_usd,
|
||||
tools_called, human_action, risk_context, imported_from, persona,
|
||||
input_text, output_text
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
input.workspaceId ?? null,
|
||||
input.sessionId ?? null,
|
||||
input.model,
|
||||
input.provider,
|
||||
input.inputTokens,
|
||||
input.outputTokens,
|
||||
input.costUsd,
|
||||
JSON.stringify(input.toolsCalled ?? []),
|
||||
input.humanAction ?? 'none',
|
||||
input.riskContext ?? null,
|
||||
input.importedFrom ?? null,
|
||||
input.persona ?? null,
|
||||
input.inputText ?? null,
|
||||
input.outputText ?? null,
|
||||
);
|
||||
|
||||
const row = raw.prepare('SELECT * FROM ai_interactions WHERE id = ?').get(result.lastInsertRowid) as Record<string, unknown>;
|
||||
return this.rowToInteraction(row);
|
||||
}
|
||||
|
||||
/** Get interactions for a workspace within a date range. */
|
||||
getByWorkspace(workspaceId: string, from?: string, to?: string): AIInteraction[] {
|
||||
if (from && !/^\d{4}-\d{2}-\d{2}/.test(from)) throw new Error('Invalid date format');
|
||||
if (to && !/^\d{4}-\d{2}-\d{2}/.test(to)) throw new Error('Invalid date format');
|
||||
|
||||
const raw = this.db.getDatabase();
|
||||
let sql = 'SELECT * FROM ai_interactions WHERE workspace_id = ?';
|
||||
const params: unknown[] = [workspaceId];
|
||||
|
||||
if (from) { sql += ' AND timestamp >= ?'; params.push(from); }
|
||||
if (to) { sql += ' AND timestamp <= ?'; params.push(to); }
|
||||
sql += ' ORDER BY timestamp DESC LIMIT 1000';
|
||||
|
||||
return (raw.prepare(sql).all(...params) as Record<string, unknown>[]).map(r => this.rowToInteraction(r));
|
||||
}
|
||||
|
||||
/** Get all interactions within a date range. */
|
||||
getByDateRange(from: string, to: string, workspaceId?: string): AIInteraction[] {
|
||||
if (from && !/^\d{4}-\d{2}-\d{2}/.test(from)) throw new Error('Invalid date format');
|
||||
if (to && !/^\d{4}-\d{2}-\d{2}/.test(to)) throw new Error('Invalid date format');
|
||||
|
||||
const raw = this.db.getDatabase();
|
||||
let sql = 'SELECT * FROM ai_interactions WHERE timestamp >= ? AND timestamp <= ?';
|
||||
const params: unknown[] = [from, to];
|
||||
|
||||
if (workspaceId) { sql += ' AND workspace_id = ?'; params.push(workspaceId); }
|
||||
sql += ' ORDER BY timestamp DESC LIMIT 1000';
|
||||
|
||||
return (raw.prepare(sql).all(...params) as Record<string, unknown>[]).map(r => this.rowToInteraction(r));
|
||||
}
|
||||
|
||||
/** Get total interaction count. */
|
||||
count(workspaceId?: string): number {
|
||||
const raw = this.db.getDatabase();
|
||||
if (workspaceId) {
|
||||
const row = raw.prepare('SELECT COUNT(*) as cnt FROM ai_interactions WHERE workspace_id = ?').get(workspaceId) as { cnt: number };
|
||||
return row.cnt;
|
||||
}
|
||||
const row = raw.prepare('SELECT COUNT(*) as cnt FROM ai_interactions').get() as { cnt: number };
|
||||
return row.cnt;
|
||||
}
|
||||
|
||||
/** Get the oldest log timestamp (Art. 19 retention check). */
|
||||
getOldestTimestamp(): string | null {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT MIN(timestamp) as oldest FROM ai_interactions').get() as { oldest: string | null };
|
||||
return row.oldest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the `first_run_at` timestamp from the MindDB's `meta` table — set on
|
||||
* schema initialization and backfilled for pre-existing DBs. Used by the Art. 19
|
||||
* retention checker to distinguish 'new system' from 'pruned logs'.
|
||||
*/
|
||||
getFirstRunAt(): string | null {
|
||||
return this.db.getFirstRunAt();
|
||||
}
|
||||
|
||||
/** Get model inventory (aggregated usage per model). */
|
||||
getModelInventory(from?: string, to?: string, workspaceId?: string): ModelInventoryEntry[] {
|
||||
const raw = this.db.getDatabase();
|
||||
let sql = `
|
||||
SELECT model, provider,
|
||||
COUNT(*) as calls,
|
||||
SUM(input_tokens) as input_tokens,
|
||||
SUM(output_tokens) as output_tokens,
|
||||
SUM(cost_usd) as cost_usd
|
||||
FROM ai_interactions WHERE 1=1
|
||||
`;
|
||||
const params: unknown[] = [];
|
||||
|
||||
if (from) { sql += ' AND timestamp >= ?'; params.push(from); }
|
||||
if (to) { sql += ' AND timestamp <= ?'; params.push(to); }
|
||||
if (workspaceId) { sql += ' AND workspace_id = ?'; params.push(workspaceId); }
|
||||
sql += ' GROUP BY model, provider ORDER BY calls DESC';
|
||||
|
||||
return (raw.prepare(sql).all(...params) as Record<string, unknown>[]).map(r => ({
|
||||
model: r.model as string,
|
||||
provider: r.provider as string,
|
||||
calls: r.calls as number,
|
||||
inputTokens: r.input_tokens as number,
|
||||
outputTokens: r.output_tokens as number,
|
||||
costUsd: r.cost_usd as number,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Get human oversight actions (Art. 14). */
|
||||
getOversightLog(from?: string, to?: string, workspaceId?: string): OversightLogEntry[] {
|
||||
const raw = this.db.getDatabase();
|
||||
let sql = "SELECT * FROM ai_interactions WHERE human_action != 'none' AND human_action IS NOT NULL";
|
||||
const params: unknown[] = [];
|
||||
|
||||
if (from) { sql += ' AND timestamp >= ?'; params.push(from); }
|
||||
if (to) { sql += ' AND timestamp <= ?'; params.push(to); }
|
||||
if (workspaceId) { sql += ' AND workspace_id = ?'; params.push(workspaceId); }
|
||||
sql += ' ORDER BY timestamp DESC';
|
||||
|
||||
return (raw.prepare(sql).all(...params) as Record<string, unknown>[]).map(r => ({
|
||||
timestamp: r.timestamp as string,
|
||||
action: r.human_action as HumanAction,
|
||||
tool: (JSON.parse(r.tools_called as string) as string[])[0] ?? 'none',
|
||||
detail: r.persona ? `Persona: ${r.persona}` : '',
|
||||
}));
|
||||
}
|
||||
|
||||
/** Get human oversight counts for compliance check. */
|
||||
getOversightCounts(workspaceId?: string): { total: number; approved: number; denied: number; modified: number } {
|
||||
const raw = this.db.getDatabase();
|
||||
let sql = "SELECT human_action, COUNT(*) as cnt FROM ai_interactions WHERE human_action != 'none' AND human_action IS NOT NULL";
|
||||
const params: unknown[] = [];
|
||||
if (workspaceId) { sql += ' AND workspace_id = ?'; params.push(workspaceId); }
|
||||
sql += ' GROUP BY human_action';
|
||||
|
||||
const rows = raw.prepare(sql).all(...params) as { human_action: string; cnt: number }[];
|
||||
const counts = { total: 0, approved: 0, denied: 0, modified: 0 };
|
||||
for (const row of rows) {
|
||||
counts.total += row.cnt;
|
||||
if (row.human_action === 'approved') counts.approved = row.cnt;
|
||||
if (row.human_action === 'denied') counts.denied = row.cnt;
|
||||
if (row.human_action === 'modified') counts.modified = row.cnt;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
/** Get recent interactions. */
|
||||
getRecent(limit: number = 20): AIInteraction[] {
|
||||
const raw = this.db.getDatabase();
|
||||
return (raw.prepare('SELECT * FROM ai_interactions ORDER BY id DESC LIMIT ?').all(limit) as Record<string, unknown>[])
|
||||
.map(r => this.rowToInteraction(r));
|
||||
}
|
||||
|
||||
private rowToInteraction(row: Record<string, unknown>): AIInteraction {
|
||||
return {
|
||||
id: row.id as number,
|
||||
timestamp: row.timestamp as string,
|
||||
workspaceId: row.workspace_id as string | null,
|
||||
sessionId: row.session_id as string | null,
|
||||
model: row.model as string,
|
||||
provider: row.provider as string,
|
||||
inputTokens: row.input_tokens as number,
|
||||
outputTokens: row.output_tokens as number,
|
||||
costUsd: row.cost_usd as number,
|
||||
toolsCalled: JSON.parse(row.tools_called as string) as string[],
|
||||
humanAction: (row.human_action as HumanAction) ?? 'none',
|
||||
riskContext: row.risk_context as string | null,
|
||||
importedFrom: row.imported_from as string | null,
|
||||
persona: row.persona as string | null,
|
||||
inputText: (row.input_text as string | null | undefined) ?? null,
|
||||
outputText: (row.output_text as string | null | undefined) ?? null,
|
||||
};
|
||||
}
|
||||
}
|
||||
104
packages/core/src/compliance/report-generator.ts
Normal file
104
packages/core/src/compliance/report-generator.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
/**
|
||||
* ReportGenerator — generates AI Act compliance audit reports.
|
||||
*
|
||||
* Produces JSON reports (PDF generation deferred to document tooling).
|
||||
* Reports cover: compliance status, model inventory, oversight log,
|
||||
* harvest provenance, and interaction counts.
|
||||
*/
|
||||
|
||||
import type { InteractionStore } from './interaction-store.js';
|
||||
import type { HarvestSourceStore } from '@waggle/hive-mind-core';
|
||||
import { ComplianceStatusChecker } from './status-checker.js';
|
||||
import type { AuditReport, AuditReportRequest, AIActRiskLevel } from './types.js';
|
||||
|
||||
const REPORT_VERSION = '1.0';
|
||||
|
||||
export interface ReportGeneratorDeps {
|
||||
interactionStore: InteractionStore;
|
||||
harvestStore: HarvestSourceStore;
|
||||
getWorkspaceRisk?: (workspaceId: string) => AIActRiskLevel;
|
||||
getWorkspaceName?: (workspaceId: string) => string;
|
||||
/**
|
||||
* Return the ISO timestamp of the last risk classification for this
|
||||
* workspace, or null if the workspace has no persisted classification
|
||||
* date (e.g., was created before the field existed).
|
||||
*/
|
||||
getWorkspaceRiskClassifiedAt?: (workspaceId: string) => string | null;
|
||||
}
|
||||
|
||||
export class ReportGenerator {
|
||||
private interactions: InteractionStore;
|
||||
private harvest: HarvestSourceStore;
|
||||
private getWorkspaceRisk: (id: string) => AIActRiskLevel;
|
||||
private getWorkspaceName: (id: string) => string;
|
||||
private getWorkspaceRiskClassifiedAt: (id: string) => string | null;
|
||||
|
||||
constructor(deps: ReportGeneratorDeps) {
|
||||
this.interactions = deps.interactionStore;
|
||||
this.harvest = deps.harvestStore;
|
||||
this.getWorkspaceRisk = deps.getWorkspaceRisk ?? (() => 'minimal');
|
||||
this.getWorkspaceName = deps.getWorkspaceName ?? ((id) => id);
|
||||
this.getWorkspaceRiskClassifiedAt = deps.getWorkspaceRiskClassifiedAt ?? (() => null);
|
||||
}
|
||||
|
||||
/** Generate a full audit report. */
|
||||
generate(request: AuditReportRequest): AuditReport {
|
||||
const checker = new ComplianceStatusChecker(this.interactions);
|
||||
const complianceStatus = checker.check(request.workspaceId);
|
||||
|
||||
const report: AuditReport = {
|
||||
report: {
|
||||
version: REPORT_VERSION,
|
||||
generatedAt: new Date().toISOString(),
|
||||
period: { from: request.from, to: request.to },
|
||||
generatedBy: 'Waggle OS',
|
||||
},
|
||||
workspace: request.workspaceId ? {
|
||||
id: request.workspaceId,
|
||||
name: this.getWorkspaceName(request.workspaceId),
|
||||
riskLevel: this.getWorkspaceRisk(request.workspaceId),
|
||||
riskClassifiedAt: this.getWorkspaceRiskClassifiedAt(request.workspaceId),
|
||||
} : null,
|
||||
complianceStatus,
|
||||
modelInventory: [],
|
||||
humanOversightLog: [],
|
||||
harvestProvenance: [],
|
||||
interactionCount: 0,
|
||||
};
|
||||
|
||||
// Model inventory
|
||||
if (request.include.models) {
|
||||
report.modelInventory = this.interactions.getModelInventory(
|
||||
request.from, request.to, request.workspaceId,
|
||||
);
|
||||
}
|
||||
|
||||
// Human oversight log
|
||||
if (request.include.oversight) {
|
||||
report.humanOversightLog = this.interactions.getOversightLog(
|
||||
request.from, request.to, request.workspaceId,
|
||||
);
|
||||
}
|
||||
|
||||
// Harvest provenance
|
||||
if (request.include.provenance) {
|
||||
const sources = this.harvest.getAll();
|
||||
report.harvestProvenance = sources.map(s => ({
|
||||
source: s.displayName,
|
||||
importedAt: s.lastSyncedAt ?? s.createdAt,
|
||||
itemsImported: s.itemsImported,
|
||||
framesCreated: s.framesCreated,
|
||||
}));
|
||||
}
|
||||
|
||||
// Interaction count
|
||||
if (request.include.interactions) {
|
||||
const interactions = this.interactions.getByDateRange(
|
||||
request.from, request.to, request.workspaceId,
|
||||
);
|
||||
report.interactionCount = interactions.length;
|
||||
}
|
||||
|
||||
return report;
|
||||
}
|
||||
}
|
||||
175
packages/core/src/compliance/status-checker.ts
Normal file
175
packages/core/src/compliance/status-checker.ts
Normal file
@@ -0,0 +1,175 @@
|
||||
/**
|
||||
* ComplianceStatusChecker — evaluates AI Act compliance per workspace.
|
||||
*
|
||||
* Checks:
|
||||
* - Art. 12: Automatic event logging (ai_interactions count)
|
||||
* - Art. 14: Human oversight (approval/denial actions recorded)
|
||||
* - Art. 19: Log retention (oldest log >= 6 months ago)
|
||||
* - Art. 26: Deployer monitoring (active monitors)
|
||||
* - Art. 50: Model transparency (models disclosed)
|
||||
*/
|
||||
|
||||
import type { InteractionStore } from './interaction-store.js';
|
||||
import type { ComplianceStatus, ArticleStatus } from './types.js';
|
||||
|
||||
const SIX_MONTHS_MS = 180 * 24 * 60 * 60 * 1000;
|
||||
|
||||
export class ComplianceStatusChecker {
|
||||
private store: InteractionStore;
|
||||
|
||||
constructor(store: InteractionStore) {
|
||||
this.store = store;
|
||||
}
|
||||
|
||||
/** Evaluate full compliance status for a workspace (or all workspaces if null). */
|
||||
check(workspaceId?: string): ComplianceStatus {
|
||||
const art12 = this.checkArt12(workspaceId);
|
||||
const art14 = this.checkArt14(workspaceId);
|
||||
const art19 = this.checkArt19();
|
||||
const art26 = this.checkArt26();
|
||||
const art50 = this.checkArt50(workspaceId);
|
||||
|
||||
const statuses = [art12.status, art14.status, art19.status, art26.status, art50.status];
|
||||
const overall = statuses.includes('non-compliant') ? 'non-compliant'
|
||||
: statuses.includes('warning') ? 'warning'
|
||||
: 'compliant';
|
||||
|
||||
return {
|
||||
overall,
|
||||
art12Logging: art12,
|
||||
art14Oversight: art14,
|
||||
art19Retention: art19,
|
||||
art26Monitoring: art26,
|
||||
art50Transparency: art50,
|
||||
};
|
||||
}
|
||||
|
||||
/** Art. 12: Automatic recording of events. */
|
||||
private checkArt12(workspaceId?: string): ArticleStatus & { totalInteractions: number } {
|
||||
const total = this.store.count(workspaceId);
|
||||
|
||||
if (total === 0) {
|
||||
// M11: if the DB has been active for >24h with zero interactions, escalate to non-compliant
|
||||
const firstRun = this.store.getFirstRunAt();
|
||||
const activeOver24h = firstRun
|
||||
? (Date.now() - new Date(firstRun).getTime()) > 24 * 60 * 60 * 1000
|
||||
: false;
|
||||
|
||||
return {
|
||||
status: activeOver24h ? 'non-compliant' : 'warning',
|
||||
detail: activeOver24h
|
||||
? 'No interactions logged despite system being active for over 24 hours. Verify logging pipeline.'
|
||||
: 'No interactions logged yet. Logging activates automatically on first AI interaction.',
|
||||
totalInteractions: 0,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: 'compliant',
|
||||
detail: `${total} interactions logged with full model, token, cost, and tool tracking.`,
|
||||
totalInteractions: total,
|
||||
};
|
||||
}
|
||||
|
||||
/** Art. 14: Human oversight capability. */
|
||||
private checkArt14(workspaceId?: string): ArticleStatus & { humanActions: number; approvalRate: number } {
|
||||
const counts = this.store.getOversightCounts(workspaceId);
|
||||
|
||||
if (counts.total === 0) {
|
||||
return {
|
||||
status: 'compliant',
|
||||
detail: 'Human oversight capabilities available (approval gates, tool deny lists). No oversight actions recorded yet.',
|
||||
humanActions: 0,
|
||||
approvalRate: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const approvalRate = counts.total > 0
|
||||
? Math.round((counts.approved / counts.total) * 100)
|
||||
: 0;
|
||||
|
||||
return {
|
||||
status: 'compliant',
|
||||
detail: `${counts.total} human oversight actions: ${counts.approved} approved, ${counts.denied} denied, ${counts.modified} modified.`,
|
||||
humanActions: counts.total,
|
||||
approvalRate,
|
||||
};
|
||||
}
|
||||
|
||||
/** Art. 19: Log retention (minimum 6 months). */
|
||||
private checkArt19(): ArticleStatus & { oldestLogDate: string | null; retentionDays: number } {
|
||||
const oldest = this.store.getOldestTimestamp();
|
||||
|
||||
if (!oldest) {
|
||||
return {
|
||||
status: 'compliant',
|
||||
detail: 'No logs to retain yet. Retention policy is permanent by default.',
|
||||
oldestLogDate: null,
|
||||
retentionDays: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const oldestDate = new Date(oldest);
|
||||
const now = new Date();
|
||||
const retentionMs = now.getTime() - oldestDate.getTime();
|
||||
const retentionDays = Math.floor(retentionMs / (24 * 60 * 60 * 1000));
|
||||
|
||||
// Review Critical #2: the previous expression was a tautology
|
||||
// (`retentionMs >= SIX_MONTHS_MS || retentionDays < 180`) that covered every
|
||||
// non-negative value of retentionDays. A deployment that pruned logs after 30
|
||||
// days still reported compliant.
|
||||
//
|
||||
// Proper fix requires distinguishing 'system is young' from 'logs were pruned'.
|
||||
// We track system age via MindDB's `meta.first_run_at` entry (set on schema init,
|
||||
// backfilled for pre-existing DBs). If the system has been running for 180+ days
|
||||
// but the oldest log is younger than that, something pruned the logs and we
|
||||
// correctly report warning.
|
||||
const firstRun = this.store.getFirstRunAt();
|
||||
const systemAgeMs = firstRun ? now.getTime() - new Date(firstRun).getTime() : retentionMs;
|
||||
const hasBeenRunning6Months = systemAgeMs >= SIX_MONTHS_MS;
|
||||
const logsOlderThan6Months = retentionMs >= SIX_MONTHS_MS;
|
||||
const meetsMinimum = !hasBeenRunning6Months || logsOlderThan6Months;
|
||||
|
||||
return {
|
||||
status: meetsMinimum ? 'compliant' : 'warning',
|
||||
detail: meetsMinimum
|
||||
? hasBeenRunning6Months
|
||||
? `Logs retained since ${oldest.split('T')[0]} (${retentionDays} days). Art. 19 minimum (180 days) met.`
|
||||
: `Logs retained since ${oldest.split('T')[0]} (${retentionDays} days). System is still within its first 180 days — retention compliance will be enforceable after 2026-${(new Date(firstRun ?? now).getMonth() + 7).toString().padStart(2, '0')}.`
|
||||
: `Oldest log: ${oldest.split('T')[0]} (${retentionDays} days) but system is ${Math.floor(systemAgeMs / (24 * 60 * 60 * 1000))} days old. Logs appear to have been pruned — EU AI Act Art. 19 requires 180-day minimum retention.`,
|
||||
oldestLogDate: oldest,
|
||||
retentionDays,
|
||||
};
|
||||
}
|
||||
|
||||
/** Art. 26: Deployer monitoring obligations. */
|
||||
private checkArt26(): ArticleStatus & { activeMonitors: string[] } {
|
||||
// Waggle always has these monitors active
|
||||
const monitors = [
|
||||
'cost_tracking', // CostTracker in packages/agent
|
||||
'tool_logging', // Tool calls logged per interaction
|
||||
'model_identification', // Model recorded per interaction
|
||||
'persona_tracking', // Persona recorded per interaction
|
||||
];
|
||||
|
||||
return {
|
||||
status: 'compliant',
|
||||
detail: `${monitors.length} active monitors: cost, tools, model ID, persona.`,
|
||||
activeMonitors: monitors,
|
||||
};
|
||||
}
|
||||
|
||||
/** Art. 50: Transparency — models disclosed. */
|
||||
private checkArt50(workspaceId?: string): ArticleStatus & { modelsDisclosed: boolean } {
|
||||
const inventory = this.store.getModelInventory(undefined, undefined, workspaceId);
|
||||
const modelsDisclosed = inventory.length > 0;
|
||||
|
||||
return {
|
||||
status: 'compliant',
|
||||
detail: modelsDisclosed
|
||||
? `${inventory.length} model(s) in use, all identified in StatusBar and interaction logs.`
|
||||
: 'Model identification active. Models will be disclosed on first interaction.',
|
||||
modelsDisclosed,
|
||||
};
|
||||
}
|
||||
}
|
||||
242
packages/core/src/compliance/template-store.ts
Normal file
242
packages/core/src/compliance/template-store.ts
Normal file
@@ -0,0 +1,242 @@
|
||||
/**
|
||||
* ComplianceTemplateStore — CRUD for user-editable compliance report templates.
|
||||
*
|
||||
* M-03: Templates let a user save a named "report shape" (which sections to include,
|
||||
* a risk-class override, and optional org/footer text) and re-apply it to subsequent
|
||||
* audit-report exports. Sections MERGE with the runtime selection (union semantics).
|
||||
*
|
||||
* Logo support is deferred to Bucket 2. See docs/plans/COMPLIANCE-AUDIT-2026-04-20.md.
|
||||
*
|
||||
* DDL is self-contained here (following HarvestRunStore pattern) rather than threaded
|
||||
* through mind/schema.ts — compliance_templates is not part of the Art. 12 audit
|
||||
* substrate and doesn't need append-only triggers or cross-layer migration tracking.
|
||||
*/
|
||||
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
import type {
|
||||
AIActRiskLevel,
|
||||
ComplianceTemplate,
|
||||
ComplianceTemplateSections,
|
||||
CreateComplianceTemplateInput,
|
||||
UpdateComplianceTemplateInput,
|
||||
} from './types.js';
|
||||
|
||||
const CREATE_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS compliance_templates (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
sections_json TEXT NOT NULL DEFAULT '{}',
|
||||
risk_classification TEXT CHECK (risk_classification IN ('minimal', 'limited', 'high-risk', 'unacceptable')),
|
||||
org_name TEXT,
|
||||
footer_text TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
)
|
||||
`;
|
||||
// TODO Bucket 2 (M-05 custom branding): add `logo_blob BLOB` + `logo_mime TEXT`
|
||||
// columns. The `Edit templates` modal will grow a file-picker; renderComplianceReportPdf
|
||||
// will render the blob into the PDF header when present.
|
||||
|
||||
const CREATE_INDEX_SQL = `
|
||||
CREATE INDEX IF NOT EXISTS idx_compliance_templates_name ON compliance_templates (name)
|
||||
`;
|
||||
|
||||
const DEFAULT_SECTIONS: ComplianceTemplateSections = {
|
||||
interactions: true,
|
||||
oversight: true,
|
||||
models: true,
|
||||
provenance: true,
|
||||
riskAssessment: true,
|
||||
fria: false,
|
||||
};
|
||||
|
||||
/**
|
||||
* Canonical name for the seeded KVARK template. Used for idempotency — the
|
||||
* seed only fires when no row with this exact name exists.
|
||||
*/
|
||||
export const KVARK_TEMPLATE_NAME = 'KVARK Enterprise Audit';
|
||||
|
||||
export class ComplianceTemplateStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
const existsRow = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='compliance_templates'",
|
||||
).get();
|
||||
if (existsRow) return;
|
||||
raw.prepare(CREATE_TABLE_SQL).run();
|
||||
raw.prepare(CREATE_INDEX_SQL).run();
|
||||
}
|
||||
|
||||
create(input: CreateComplianceTemplateInput): ComplianceTemplate {
|
||||
const name = input.name.trim();
|
||||
if (!name) throw new Error('Template name is required');
|
||||
|
||||
const raw = this.db.getDatabase();
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO compliance_templates (
|
||||
name, description, sections_json, risk_classification, org_name, footer_text
|
||||
) VALUES (?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
name,
|
||||
input.description ?? null,
|
||||
JSON.stringify(this.normalizeSections(input.sections)),
|
||||
input.riskClassification ?? null,
|
||||
input.orgName ?? null,
|
||||
input.footerText ?? null,
|
||||
);
|
||||
return this.getById(Number(result.lastInsertRowid))!;
|
||||
}
|
||||
|
||||
getById(id: number): ComplianceTemplate | null {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT * FROM compliance_templates WHERE id = ?').get(id) as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
return row ? this.rowToTemplate(row) : null;
|
||||
}
|
||||
|
||||
list(): ComplianceTemplate[] {
|
||||
const raw = this.db.getDatabase();
|
||||
return (
|
||||
raw
|
||||
.prepare('SELECT * FROM compliance_templates ORDER BY updated_at DESC')
|
||||
.all() as Record<string, unknown>[]
|
||||
).map(r => this.rowToTemplate(r));
|
||||
}
|
||||
|
||||
update(id: number, patch: UpdateComplianceTemplateInput): ComplianceTemplate | null {
|
||||
const existing = this.getById(id);
|
||||
if (!existing) return null;
|
||||
|
||||
const next: ComplianceTemplate = {
|
||||
...existing,
|
||||
name: patch.name !== undefined ? patch.name.trim() : existing.name,
|
||||
description: patch.description !== undefined ? patch.description : existing.description,
|
||||
sections: patch.sections !== undefined ? this.normalizeSections(patch.sections) : existing.sections,
|
||||
riskClassification:
|
||||
patch.riskClassification !== undefined ? patch.riskClassification : existing.riskClassification,
|
||||
orgName: patch.orgName !== undefined ? patch.orgName : existing.orgName,
|
||||
footerText: patch.footerText !== undefined ? patch.footerText : existing.footerText,
|
||||
};
|
||||
if (!next.name) throw new Error('Template name is required');
|
||||
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(`
|
||||
UPDATE compliance_templates SET
|
||||
name = ?,
|
||||
description = ?,
|
||||
sections_json = ?,
|
||||
risk_classification = ?,
|
||||
org_name = ?,
|
||||
footer_text = ?,
|
||||
updated_at = datetime('now')
|
||||
WHERE id = ?
|
||||
`).run(
|
||||
next.name,
|
||||
next.description,
|
||||
JSON.stringify(next.sections),
|
||||
next.riskClassification,
|
||||
next.orgName,
|
||||
next.footerText,
|
||||
id,
|
||||
);
|
||||
return this.getById(id);
|
||||
}
|
||||
|
||||
/** Returns true if the row existed and was removed, false otherwise. */
|
||||
delete(id: number): boolean {
|
||||
const raw = this.db.getDatabase();
|
||||
const info = raw.prepare('DELETE FROM compliance_templates WHERE id = ?').run(id);
|
||||
return info.changes > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge a template's section flags with runtime flags. Union semantics: a section
|
||||
* is included if EITHER the template OR the runtime toggle turns it on. This keeps
|
||||
* the template's role "additive" — it can never silently hide a section the user
|
||||
* asked for in the current export.
|
||||
*/
|
||||
static mergeSections(
|
||||
template: ComplianceTemplateSections,
|
||||
runtime: ComplianceTemplateSections,
|
||||
): ComplianceTemplateSections {
|
||||
return {
|
||||
interactions: template.interactions || runtime.interactions,
|
||||
oversight: template.oversight || runtime.oversight,
|
||||
models: template.models || runtime.models,
|
||||
provenance: template.provenance || runtime.provenance,
|
||||
riskAssessment: template.riskAssessment || runtime.riskAssessment,
|
||||
fria: template.fria || runtime.fria,
|
||||
};
|
||||
}
|
||||
|
||||
private normalizeSections(s: Partial<ComplianceTemplateSections> | undefined): ComplianceTemplateSections {
|
||||
return { ...DEFAULT_SECTIONS, ...(s ?? {}) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Idempotent: seed the built-in "KVARK Enterprise Audit" template if no
|
||||
* template with that exact name exists yet (M-06).
|
||||
*
|
||||
* The KVARK template is a sensible-default shape for sovereign enterprise
|
||||
* deployments — every section on (including FRIA, since KVARK serves
|
||||
* high-risk enterprise systems), risk pinned at `high-risk`, and org/footer
|
||||
* text that signals the sovereign-deployment narrative.
|
||||
*
|
||||
* Custom KVARK-specific sections (per-department risk breakdown, IAM
|
||||
* audit columns, data-residency attestation text) are deferred to
|
||||
* Bucket 2 along with the logo field — the current template schema
|
||||
* doesn't support custom sections yet.
|
||||
*
|
||||
* Returns the seeded template, or null if one already existed.
|
||||
*/
|
||||
seedKvarkTemplateIfMissing(): ComplianceTemplate | null {
|
||||
const raw = this.db.getDatabase();
|
||||
const existing = raw.prepare(
|
||||
'SELECT id FROM compliance_templates WHERE name = ? LIMIT 1',
|
||||
).get(KVARK_TEMPLATE_NAME) as { id: number } | undefined;
|
||||
if (existing) return null;
|
||||
return this.create({
|
||||
name: KVARK_TEMPLATE_NAME,
|
||||
description:
|
||||
'Sovereign AI Act audit shape for enterprise on-prem deployments. ' +
|
||||
'Includes all monitored articles (12/14/19/26/50) plus FRIA since KVARK ' +
|
||||
'customers typically operate high-risk systems. Data never leaves the ' +
|
||||
"customer perimeter; this template's org + footer text signal that " +
|
||||
'posture directly on every exported report.',
|
||||
sections: { ...DEFAULT_SECTIONS, fria: true },
|
||||
riskClassification: 'high-risk',
|
||||
orgName: 'KVARK Sovereign — Enterprise Deployment',
|
||||
footerText:
|
||||
'Confidential · EU AI Act attestation · KVARK sovereign deployment · Data remains within customer perimeter',
|
||||
});
|
||||
}
|
||||
|
||||
private rowToTemplate(row: Record<string, unknown>): ComplianceTemplate {
|
||||
let sections: ComplianceTemplateSections;
|
||||
try {
|
||||
sections = this.normalizeSections(JSON.parse((row.sections_json as string) ?? '{}'));
|
||||
} catch {
|
||||
sections = { ...DEFAULT_SECTIONS };
|
||||
}
|
||||
return {
|
||||
id: row.id as number,
|
||||
name: row.name as string,
|
||||
description: (row.description as string | null) ?? null,
|
||||
sections,
|
||||
riskClassification: (row.risk_classification as AIActRiskLevel | null) ?? null,
|
||||
orgName: (row.org_name as string | null) ?? null,
|
||||
footerText: (row.footer_text as string | null) ?? null,
|
||||
createdAt: row.created_at as string,
|
||||
updatedAt: row.updated_at as string,
|
||||
};
|
||||
}
|
||||
}
|
||||
201
packages/core/src/compliance/types.ts
Normal file
201
packages/core/src/compliance/types.ts
Normal file
@@ -0,0 +1,201 @@
|
||||
/**
|
||||
* AI Act Compliance Types — interaction audit, risk classification, report format.
|
||||
*/
|
||||
|
||||
// ── Risk Classification (Art. 26) ──
|
||||
|
||||
export type AIActRiskLevel = 'minimal' | 'limited' | 'high-risk' | 'unacceptable';
|
||||
|
||||
export type HumanAction = 'approved' | 'denied' | 'modified' | 'none';
|
||||
|
||||
// ── Interaction Audit (Art. 12) ──
|
||||
|
||||
export interface AIInteraction {
|
||||
id: number;
|
||||
timestamp: string;
|
||||
workspaceId: string | null;
|
||||
sessionId: string | null;
|
||||
model: string;
|
||||
provider: string;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
costUsd: number;
|
||||
toolsCalled: string[];
|
||||
humanAction: HumanAction;
|
||||
riskContext: string | null;
|
||||
importedFrom: string | null;
|
||||
persona: string | null;
|
||||
// Review Critical #3 (compliance): EU AI Act Art. 12.1(a) requires recording the
|
||||
// actual inputs and outputs of the system, not just token counts. Nullable because
|
||||
// pre-existing DBs may have rows from before the columns were added.
|
||||
inputText: string | null;
|
||||
outputText: string | null;
|
||||
}
|
||||
|
||||
export interface RecordInteractionInput {
|
||||
workspaceId?: string;
|
||||
sessionId?: string;
|
||||
model: string;
|
||||
provider: string;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
costUsd: number;
|
||||
toolsCalled?: string[];
|
||||
humanAction?: HumanAction;
|
||||
riskContext?: string;
|
||||
importedFrom?: string;
|
||||
persona?: string;
|
||||
// Review Critical #3: optional because echo-mode interactions may not have meaningful
|
||||
// content. Callers on the live agent path SHOULD pass these; record() does not reject
|
||||
// absent values but the compliance status checker flags them as a gap.
|
||||
inputText?: string;
|
||||
outputText?: string;
|
||||
}
|
||||
|
||||
// ── Compliance Status ──
|
||||
|
||||
export interface ArticleStatus {
|
||||
status: 'compliant' | 'warning' | 'non-compliant';
|
||||
detail: string;
|
||||
}
|
||||
|
||||
export interface ComplianceStatus {
|
||||
overall: 'compliant' | 'warning' | 'non-compliant';
|
||||
art12Logging: ArticleStatus & { totalInteractions: number };
|
||||
art14Oversight: ArticleStatus & { humanActions: number; approvalRate: number };
|
||||
art19Retention: ArticleStatus & { oldestLogDate: string | null; retentionDays: number };
|
||||
art26Monitoring: ArticleStatus & { activeMonitors: string[] };
|
||||
art50Transparency: ArticleStatus & { modelsDisclosed: boolean };
|
||||
}
|
||||
|
||||
// ── Audit Report ──
|
||||
|
||||
export interface ModelInventoryEntry {
|
||||
model: string;
|
||||
provider: string;
|
||||
calls: number;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
costUsd: number;
|
||||
}
|
||||
|
||||
export interface OversightLogEntry {
|
||||
timestamp: string;
|
||||
action: HumanAction;
|
||||
tool: string;
|
||||
detail: string;
|
||||
user?: string;
|
||||
}
|
||||
|
||||
export interface HarvestProvenanceEntry {
|
||||
source: string;
|
||||
importedAt: string;
|
||||
itemsImported: number;
|
||||
framesCreated: number;
|
||||
}
|
||||
|
||||
export interface AuditReportRequest {
|
||||
workspaceId?: string;
|
||||
from: string;
|
||||
to: string;
|
||||
format: 'json' | 'pdf' | 'both';
|
||||
include: {
|
||||
interactions: boolean;
|
||||
oversight: boolean;
|
||||
models: boolean;
|
||||
provenance: boolean;
|
||||
riskAssessment: boolean;
|
||||
fria: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
export interface AuditReport {
|
||||
report: {
|
||||
version: string;
|
||||
generatedAt: string;
|
||||
period: { from: string; to: string };
|
||||
generatedBy: string;
|
||||
};
|
||||
workspace: {
|
||||
id: string | null;
|
||||
name: string;
|
||||
riskLevel: AIActRiskLevel;
|
||||
riskClassifiedAt: string | null;
|
||||
} | null;
|
||||
complianceStatus: ComplianceStatus;
|
||||
modelInventory: ModelInventoryEntry[];
|
||||
humanOversightLog: OversightLogEntry[];
|
||||
harvestProvenance: HarvestProvenanceEntry[];
|
||||
interactionCount: number;
|
||||
}
|
||||
|
||||
// ── Compliance Templates (M-03) ──
|
||||
//
|
||||
// Templates let a user save a preferred audit-report shape (sections + risk class +
|
||||
// org/footer overrides) and re-apply it to exports. Sections MERGE (union) with the
|
||||
// user's runtime section toggles — a template only turns sections ON, never off.
|
||||
// `riskClassification` overrides the workspace's risk level for that one report.
|
||||
// Logo support is deferred to Bucket 2 (post-launch).
|
||||
|
||||
export interface ComplianceTemplateSections {
|
||||
interactions: boolean;
|
||||
oversight: boolean;
|
||||
models: boolean;
|
||||
provenance: boolean;
|
||||
riskAssessment: boolean;
|
||||
fria: boolean;
|
||||
}
|
||||
|
||||
export interface ComplianceTemplate {
|
||||
id: number;
|
||||
name: string;
|
||||
description: string | null;
|
||||
sections: ComplianceTemplateSections;
|
||||
riskClassification: AIActRiskLevel | null;
|
||||
orgName: string | null;
|
||||
footerText: string | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface CreateComplianceTemplateInput {
|
||||
name: string;
|
||||
description?: string | null;
|
||||
sections: ComplianceTemplateSections;
|
||||
riskClassification?: AIActRiskLevel | null;
|
||||
orgName?: string | null;
|
||||
footerText?: string | null;
|
||||
}
|
||||
|
||||
export interface UpdateComplianceTemplateInput {
|
||||
name?: string;
|
||||
description?: string | null;
|
||||
sections?: ComplianceTemplateSections;
|
||||
riskClassification?: AIActRiskLevel | null;
|
||||
orgName?: string | null;
|
||||
footerText?: string | null;
|
||||
}
|
||||
|
||||
// ── Template auto-suggestion ──
|
||||
|
||||
export const TEMPLATE_RISK_MAP: Record<string, AIActRiskLevel> = {
|
||||
'legal-review': 'high-risk',
|
||||
'hr-management': 'high-risk',
|
||||
'recruiting': 'high-risk',
|
||||
'finance': 'high-risk',
|
||||
'insurance': 'high-risk',
|
||||
'credit-scoring': 'high-risk',
|
||||
'sales-pipeline': 'limited',
|
||||
'marketing-campaign': 'limited',
|
||||
'research-project': 'minimal',
|
||||
'code-review': 'minimal',
|
||||
'product-launch': 'limited',
|
||||
'agency-consulting': 'limited',
|
||||
'blank': 'minimal',
|
||||
'content-creation': 'minimal',
|
||||
'customer-support': 'limited',
|
||||
'data-analysis': 'limited',
|
||||
'project-management': 'minimal',
|
||||
'education': 'limited',
|
||||
'healthcare': 'high-risk',
|
||||
};
|
||||
281
packages/core/src/config.ts
Normal file
281
packages/core/src/config.ts
Normal file
@@ -0,0 +1,281 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import type { EmbeddingProviderConfig, EmbeddingProviderType } from '@waggle/hive-mind-core';
|
||||
|
||||
export interface ProviderEntry {
|
||||
apiKey: string;
|
||||
models: string[];
|
||||
baseUrl?: string;
|
||||
}
|
||||
|
||||
export interface TeamServerConfig {
|
||||
url: string;
|
||||
token?: string;
|
||||
userId?: string;
|
||||
displayName?: string;
|
||||
}
|
||||
|
||||
export interface CliConfig {
|
||||
allowlist?: string[];
|
||||
}
|
||||
|
||||
interface ConfigData {
|
||||
defaultModel: string;
|
||||
providers: Record<string, ProviderEntry>;
|
||||
mindPath?: string;
|
||||
teamServer?: TeamServerConfig;
|
||||
/** Governed CLI programs the agent may execute. */
|
||||
cli?: CliConfig;
|
||||
/** F8: Daily cost budget in dollars. null = no limit. */
|
||||
dailyBudget?: number | null;
|
||||
/** When true, exceeding dailyBudget blocks agent. When false, warns only. */
|
||||
budgetHardCap?: boolean;
|
||||
/** Model Pilot: fallback model when primary fails (429/500/timeout) */
|
||||
fallbackModel?: string;
|
||||
/** Model Pilot: budget-saver model when daily spend hits threshold */
|
||||
budgetModel?: string;
|
||||
/** Model Pilot: budget threshold as 0.0-1.0 fraction. Default 0.8 */
|
||||
budgetThreshold?: number;
|
||||
/** Agent Intelligence: max LLM iterations per conversation. Default 90. */
|
||||
maxIterations?: number;
|
||||
/** M2-7: Telemetry opt-in (default: false — privacy first) */
|
||||
telemetryEnabled?: boolean;
|
||||
/** M2-1: Embedding provider configuration */
|
||||
embedding?: {
|
||||
provider?: EmbeddingProviderType | 'auto';
|
||||
ollamaUrl?: string;
|
||||
ollamaModel?: string;
|
||||
inprocessModel?: string;
|
||||
};
|
||||
/** Steal #6: on-demand relevance gating for MCP tools. */
|
||||
mcpToolRetrieval?: {
|
||||
enabled?: boolean;
|
||||
threshold?: number;
|
||||
topK?: number;
|
||||
};
|
||||
}
|
||||
|
||||
/** Resolved MCP tool-retrieval config (all fields present). */
|
||||
export interface McpToolRetrievalSettings {
|
||||
enabled: boolean;
|
||||
threshold: number;
|
||||
topK: number;
|
||||
}
|
||||
|
||||
const DEFAULT_MODEL = 'claude-sonnet-4-6';
|
||||
|
||||
function getDefaultConfigDir(): string {
|
||||
const home = process.env.HOME || process.env.USERPROFILE || os.homedir();
|
||||
return path.join(home, '.waggle');
|
||||
}
|
||||
|
||||
export class WaggleConfig {
|
||||
private readonly configDir: string;
|
||||
private readonly configPath: string;
|
||||
private data: ConfigData;
|
||||
|
||||
constructor(configDir?: string) {
|
||||
this.configDir = configDir ?? getDefaultConfigDir();
|
||||
this.configPath = path.join(this.configDir, 'config.json');
|
||||
|
||||
// Ensure config directory exists
|
||||
if (!fs.existsSync(this.configDir)) {
|
||||
fs.mkdirSync(this.configDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Load existing config or use defaults
|
||||
this.data = this.load();
|
||||
}
|
||||
|
||||
private load(): ConfigData {
|
||||
if (fs.existsSync(this.configPath)) {
|
||||
const raw = fs.readFileSync(this.configPath, 'utf-8');
|
||||
return JSON.parse(raw) as ConfigData;
|
||||
}
|
||||
return {
|
||||
defaultModel: DEFAULT_MODEL,
|
||||
providers: {},
|
||||
};
|
||||
}
|
||||
|
||||
save(): void {
|
||||
fs.writeFileSync(this.configPath, JSON.stringify(this.data, null, 2), 'utf-8');
|
||||
}
|
||||
|
||||
getDefaultModel(): string {
|
||||
return this.data.defaultModel;
|
||||
}
|
||||
|
||||
setDefaultModel(model: string): void {
|
||||
this.data.defaultModel = model;
|
||||
}
|
||||
|
||||
getProviders(): Record<string, ProviderEntry> {
|
||||
return { ...(this.data.providers ?? {}) };
|
||||
}
|
||||
|
||||
setProvider(name: string, entry: ProviderEntry): void {
|
||||
if (!this.data.providers) this.data.providers = {};
|
||||
this.data.providers[name] = entry;
|
||||
}
|
||||
|
||||
removeProvider(name: string): void {
|
||||
delete this.data.providers[name];
|
||||
}
|
||||
|
||||
getMindPath(): string {
|
||||
return this.data.mindPath ?? path.join(this.configDir, 'default.mind');
|
||||
}
|
||||
|
||||
getConfigDir(): string {
|
||||
return this.configDir;
|
||||
}
|
||||
|
||||
// F8: Daily cost budget
|
||||
getDailyBudget(): number | null {
|
||||
return this.data.dailyBudget ?? null;
|
||||
}
|
||||
|
||||
setDailyBudget(budget: number | null): void {
|
||||
this.data.dailyBudget = budget;
|
||||
}
|
||||
|
||||
getBudgetHardCap(): boolean {
|
||||
return this.data.budgetHardCap ?? false;
|
||||
}
|
||||
|
||||
setBudgetHardCap(enabled: boolean): void {
|
||||
this.data.budgetHardCap = enabled;
|
||||
}
|
||||
|
||||
// --- Model Pilot ---
|
||||
|
||||
getFallbackModel(): string | null {
|
||||
return this.data.fallbackModel ?? null;
|
||||
}
|
||||
|
||||
setFallbackModel(model: string): void {
|
||||
this.data.fallbackModel = model;
|
||||
}
|
||||
|
||||
clearFallbackModel(): void {
|
||||
delete this.data.fallbackModel;
|
||||
}
|
||||
|
||||
getBudgetModel(): string | null {
|
||||
return this.data.budgetModel ?? null;
|
||||
}
|
||||
|
||||
setBudgetModel(model: string): void {
|
||||
this.data.budgetModel = model;
|
||||
}
|
||||
|
||||
clearBudgetModel(): void {
|
||||
delete this.data.budgetModel;
|
||||
}
|
||||
|
||||
getBudgetThreshold(): number {
|
||||
return this.data.budgetThreshold ?? 0.8;
|
||||
}
|
||||
|
||||
setBudgetThreshold(threshold: number): void {
|
||||
this.data.budgetThreshold = Math.max(0.5, Math.min(0.95, threshold));
|
||||
}
|
||||
|
||||
// --- Agent Intelligence ---
|
||||
|
||||
getMaxIterations(): number {
|
||||
return this.data.maxIterations ?? 90;
|
||||
}
|
||||
|
||||
setMaxIterations(max: number): void {
|
||||
this.data.maxIterations = Math.max(5, Math.min(500, max));
|
||||
}
|
||||
|
||||
// --- Team Server (Phase 5) ---
|
||||
|
||||
getTeamServer(): TeamServerConfig | null {
|
||||
return this.data.teamServer ?? null;
|
||||
}
|
||||
|
||||
setTeamServer(config: TeamServerConfig): void {
|
||||
this.data.teamServer = config;
|
||||
}
|
||||
|
||||
clearTeamServer(): void {
|
||||
delete this.data.teamServer;
|
||||
}
|
||||
|
||||
isTeamConnected(): boolean {
|
||||
return this.data.teamServer !== null && this.data.teamServer !== undefined && typeof this.data.teamServer.url === 'string' && this.data.teamServer.url.length > 0;
|
||||
}
|
||||
|
||||
// --- Governed CLI access ---
|
||||
|
||||
getCliAllowlist(): string[] {
|
||||
return [...(this.data.cli?.allowlist ?? [])];
|
||||
}
|
||||
|
||||
setCliAllowlist(allowlist: string[]): void {
|
||||
const seen = new Set<string>();
|
||||
const next = allowlist.reduce<string[]>((result, entry) => {
|
||||
const value = entry.trim();
|
||||
const key = value.toLowerCase();
|
||||
if (value && !seen.has(key)) {
|
||||
seen.add(key);
|
||||
result.push(value);
|
||||
}
|
||||
return result;
|
||||
}, []);
|
||||
this.data.cli = { ...(this.data.cli ?? {}), allowlist: next };
|
||||
}
|
||||
|
||||
// --- Telemetry (M2-7) ---
|
||||
|
||||
getTelemetryEnabled(): boolean {
|
||||
return this.data.telemetryEnabled ?? false;
|
||||
}
|
||||
|
||||
setTelemetryEnabled(enabled: boolean): void {
|
||||
this.data.telemetryEnabled = enabled;
|
||||
this.save();
|
||||
}
|
||||
|
||||
// --- Embedding Provider (M2-1) ---
|
||||
|
||||
getEmbeddingConfig(): EmbeddingProviderConfig {
|
||||
const emb = this.data.embedding;
|
||||
const config: EmbeddingProviderConfig = {
|
||||
provider: (process.env.EMBEDDING_PROVIDER as EmbeddingProviderType | 'auto' | undefined) ?? emb?.provider ?? 'auto',
|
||||
targetDimensions: 1024,
|
||||
inprocess: {
|
||||
model: process.env.EMBEDDING_MODEL ?? emb?.inprocessModel,
|
||||
cacheDir: path.join(this.configDir, 'models'),
|
||||
},
|
||||
ollama: {
|
||||
baseUrl: process.env.OLLAMA_HOST ?? emb?.ollamaUrl,
|
||||
model: process.env.OLLAMA_EMBED_MODEL ?? emb?.ollamaModel,
|
||||
},
|
||||
// API keys injected separately from Vault — not stored in config.json
|
||||
};
|
||||
return config;
|
||||
}
|
||||
|
||||
setEmbeddingProvider(provider: EmbeddingProviderType | 'auto'): void {
|
||||
if (!this.data.embedding) this.data.embedding = {};
|
||||
this.data.embedding.provider = provider;
|
||||
}
|
||||
|
||||
// --- MCP tool retrieval (Steal #6) ---
|
||||
|
||||
/** Resolve MCP tool-retrieval settings, filling defaults (ON, threshold 20, top-k 10). */
|
||||
getMcpToolRetrieval(): McpToolRetrievalSettings {
|
||||
const cfg = this.data.mcpToolRetrieval;
|
||||
return {
|
||||
enabled: cfg?.enabled ?? true,
|
||||
threshold: cfg?.threshold ?? 20,
|
||||
topK: cfg?.topK ?? 10,
|
||||
};
|
||||
}
|
||||
}
|
||||
588
packages/core/src/cron-store.ts
Normal file
588
packages/core/src/cron-store.ts
Normal file
@@ -0,0 +1,588 @@
|
||||
/**
|
||||
* Cron Store — SQLite persistence for solo cron schedules.
|
||||
*
|
||||
* Stores cron job definitions with expression validation via cron-parser,
|
||||
* due-job queries, and run tracking. Follows the same pattern as
|
||||
* InstallAuditStore — operates on the .mind DB with lazy table creation.
|
||||
*/
|
||||
|
||||
import cronParser from 'cron-parser';
|
||||
const { parseExpression } = cronParser;
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
export type CronJobType = 'agent_task' | 'memory_consolidation' | 'workspace_health' | 'proactive' | 'prompt_optimization' | 'monthly_assessment' | 'connector_fetch' | 'loop';
|
||||
|
||||
export const VALID_JOB_TYPES: Set<string> = new Set([
|
||||
'agent_task',
|
||||
'memory_consolidation',
|
||||
'workspace_health',
|
||||
'proactive',
|
||||
'prompt_optimization',
|
||||
'monthly_assessment',
|
||||
'connector_fetch',
|
||||
// Loop v0: a stateful, memory-powered, report-only (L1) scheduled automation.
|
||||
// Composes recall + maker (toolless LLM) + checker (LLMJudge) + memory write.
|
||||
// job_type TEXT has no CHECK constraint, so this is additive — no migration.
|
||||
'loop',
|
||||
]);
|
||||
|
||||
export interface CronSchedule {
|
||||
id: number;
|
||||
name: string;
|
||||
cron_expr: string;
|
||||
job_type: CronJobType;
|
||||
job_config: string;
|
||||
workspace_id: string | null;
|
||||
enabled: number; // SQLite integer boolean
|
||||
last_run_at: string | null;
|
||||
next_run_at: string | null;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export interface CreateScheduleInput {
|
||||
name: string;
|
||||
cronExpr: string;
|
||||
jobType: CronJobType;
|
||||
jobConfig?: Record<string, unknown>;
|
||||
workspaceId?: string;
|
||||
enabled?: boolean;
|
||||
}
|
||||
|
||||
export interface CronExecutionRow {
|
||||
id: number;
|
||||
schedule_id: number;
|
||||
schedule_name: string;
|
||||
executed_at: string;
|
||||
duration_ms: number | null;
|
||||
success: number; // SQLite integer boolean
|
||||
result_summary: string | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export interface CronRunLeaseRow {
|
||||
id: number;
|
||||
schedule_id: number;
|
||||
schedule_name: string | null;
|
||||
started_at: string;
|
||||
pid: number | null;
|
||||
}
|
||||
|
||||
export interface NotificationRow {
|
||||
id: number;
|
||||
title: string;
|
||||
body: string;
|
||||
category: string;
|
||||
action_url: string | null;
|
||||
read: number; // SQLite integer boolean
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
// ── L2 assisted loops: durable "held action" approval queue ──────────────
|
||||
// A held action is a self-contained proposed tool call drafted by a headless
|
||||
// run (e.g. an assist-mode Loop) that needs a human's one-click approval before
|
||||
// it executes. Durable so it survives a sidecar restart (the in-memory live
|
||||
// approval Promise does not). Execute-on-approve, never mid-run suspend/resume.
|
||||
|
||||
export type PendingActionStatus = 'held' | 'approved' | 'denied' | 'executed' | 'failed' | 'expired';
|
||||
|
||||
export interface PendingActionRow {
|
||||
id: string; // requestId (uuid)
|
||||
workspace_id: string | null;
|
||||
source: string; // e.g. 'loop:<scheduleId>'
|
||||
tool_name: string;
|
||||
args_json: string;
|
||||
summary: string | null; // the maker's plain-language rationale
|
||||
risk_level: string; // low | medium | high | critical
|
||||
approval_class: string;
|
||||
status: PendingActionStatus;
|
||||
result_summary: string | null;
|
||||
error: string | null;
|
||||
created_at: string;
|
||||
decided_at: string | null;
|
||||
executed_at: string | null;
|
||||
expires_at: string | null;
|
||||
}
|
||||
|
||||
export interface SavePendingActionInput {
|
||||
id: string;
|
||||
workspaceId: string | null;
|
||||
source: string;
|
||||
toolName: string;
|
||||
argsJson: string;
|
||||
summary?: string;
|
||||
riskLevel: string;
|
||||
approvalClass: string;
|
||||
expiresAt?: string;
|
||||
}
|
||||
|
||||
// ── Table DDL ──────────────────────────────────────────────────────────
|
||||
|
||||
export const CRON_SCHEDULES_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS cron_schedules (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
cron_expr TEXT NOT NULL,
|
||||
job_type TEXT NOT NULL,
|
||||
job_config TEXT NOT NULL DEFAULT '{}',
|
||||
workspace_id TEXT,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
last_run_at TEXT,
|
||||
next_run_at TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_cron_enabled_next ON cron_schedules (enabled, next_run_at);
|
||||
`;
|
||||
|
||||
// W5.12: Cron execution history table
|
||||
export const CRON_HISTORY_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS cron_execution_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
schedule_id INTEGER NOT NULL,
|
||||
schedule_name TEXT NOT NULL,
|
||||
executed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
duration_ms INTEGER,
|
||||
success INTEGER NOT NULL DEFAULT 1,
|
||||
result_summary TEXT,
|
||||
error TEXT,
|
||||
FOREIGN KEY (schedule_id) REFERENCES cron_schedules(id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_cron_history_schedule ON cron_execution_history (schedule_id, executed_at);
|
||||
`;
|
||||
|
||||
export const CRON_RUN_LEASES_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS cron_run_leases (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
schedule_id INTEGER NOT NULL,
|
||||
schedule_name TEXT,
|
||||
started_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
pid INTEGER
|
||||
);
|
||||
`;
|
||||
|
||||
// W5.10: Notification persistence table
|
||||
export const NOTIFICATIONS_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS notifications (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
title TEXT NOT NULL,
|
||||
body TEXT NOT NULL,
|
||||
category TEXT NOT NULL DEFAULT 'system',
|
||||
action_url TEXT,
|
||||
read INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_notifications_created ON notifications (created_at);
|
||||
`;
|
||||
|
||||
// L2: durable held-action approval queue
|
||||
export const PENDING_ACTIONS_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS pending_actions (
|
||||
id TEXT PRIMARY KEY,
|
||||
workspace_id TEXT,
|
||||
source TEXT NOT NULL,
|
||||
tool_name TEXT NOT NULL,
|
||||
args_json TEXT NOT NULL,
|
||||
summary TEXT,
|
||||
risk_level TEXT NOT NULL,
|
||||
approval_class TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'held',
|
||||
result_summary TEXT,
|
||||
error TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
decided_at TEXT,
|
||||
executed_at TEXT,
|
||||
expires_at TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_pending_actions_status ON pending_actions (status, created_at);
|
||||
`;
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
/** Parse a cron expression and return the next run time as ISO string. Throws on invalid expr. */
|
||||
function computeNextRun(cronExpr: string): string {
|
||||
const interval = parseExpression(cronExpr);
|
||||
return interval.next().toISOString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a cron expression with the SAME parser create()/update() use.
|
||||
* Returns null when parseable, else the parser's error message. Lets route
|
||||
* layers (e.g. the automations /test preview) reject an expression the store
|
||||
* would refuse to persist, without duplicating the parser dependency.
|
||||
*/
|
||||
export function cronExprError(cronExpr: string): string | null {
|
||||
try {
|
||||
parseExpression(cronExpr);
|
||||
return null;
|
||||
} catch (err) {
|
||||
return err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Store ──────────────────────────────────────────────────────────────
|
||||
|
||||
export class CronStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
const exists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='cron_schedules'",
|
||||
).get();
|
||||
if (!exists) {
|
||||
raw.exec(CRON_SCHEDULES_TABLE_SQL);
|
||||
}
|
||||
// W5.12: Ensure cron execution history table
|
||||
const histExists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='cron_execution_history'",
|
||||
).get();
|
||||
if (!histExists) {
|
||||
raw.exec(CRON_HISTORY_TABLE_SQL);
|
||||
}
|
||||
const leaseExists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='cron_run_leases'",
|
||||
).get();
|
||||
if (!leaseExists) {
|
||||
raw.exec(CRON_RUN_LEASES_TABLE_SQL);
|
||||
}
|
||||
// W5.10: Ensure notifications table
|
||||
const notifExists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='notifications'",
|
||||
).get();
|
||||
if (!notifExists) {
|
||||
raw.exec(NOTIFICATIONS_TABLE_SQL);
|
||||
}
|
||||
// L2: Ensure pending_actions (held-action approval queue) table
|
||||
const pendingExists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='pending_actions'",
|
||||
).get();
|
||||
if (!pendingExists) {
|
||||
raw.exec(PENDING_ACTIONS_TABLE_SQL);
|
||||
}
|
||||
}
|
||||
|
||||
/** Create a new cron schedule. Validates cron expression and job type. */
|
||||
create(input: CreateScheduleInput): CronSchedule {
|
||||
// Validate job type
|
||||
if (!VALID_JOB_TYPES.has(input.jobType)) {
|
||||
throw new Error(`Invalid job type: "${input.jobType}". Must be one of: ${[...VALID_JOB_TYPES].join(', ')}`);
|
||||
}
|
||||
|
||||
// agent_task requires workspaceId
|
||||
if (input.jobType === 'agent_task' && !input.workspaceId) {
|
||||
throw new Error('agent_task jobs require a workspace ID');
|
||||
}
|
||||
|
||||
// Validate cron expression (throws on invalid)
|
||||
const nextRun = computeNextRun(input.cronExpr);
|
||||
|
||||
const raw = this.db.getDatabase();
|
||||
const enabled = input.enabled === false ? 0 : 1;
|
||||
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO cron_schedules (name, cron_expr, job_type, job_config, workspace_id, enabled, next_run_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
input.name,
|
||||
input.cronExpr,
|
||||
input.jobType,
|
||||
JSON.stringify(input.jobConfig ?? {}),
|
||||
input.workspaceId ?? null,
|
||||
enabled,
|
||||
nextRun,
|
||||
);
|
||||
|
||||
return raw.prepare(
|
||||
'SELECT * FROM cron_schedules WHERE id = ?',
|
||||
).get(result.lastInsertRowid) as CronSchedule;
|
||||
}
|
||||
|
||||
/** List all schedules ordered by name. */
|
||||
list(): CronSchedule[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM cron_schedules ORDER BY name ASC',
|
||||
).all() as CronSchedule[];
|
||||
}
|
||||
|
||||
/** Get a schedule by ID. Returns undefined if not found. */
|
||||
getById(id: number): CronSchedule | undefined {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM cron_schedules WHERE id = ?',
|
||||
).get(id) as CronSchedule | undefined;
|
||||
}
|
||||
|
||||
/** Update a schedule. Recomputes next_run_at if cronExpr changes. Returns updated schedule. */
|
||||
update(id: number, changes: Partial<Pick<CreateScheduleInput, 'name' | 'cronExpr' | 'jobConfig' | 'workspaceId' | 'enabled'>>): CronSchedule {
|
||||
const setClauses: string[] = [];
|
||||
const values: unknown[] = [];
|
||||
|
||||
if (changes.name !== undefined) {
|
||||
setClauses.push('name = ?');
|
||||
values.push(changes.name);
|
||||
}
|
||||
if (changes.cronExpr !== undefined) {
|
||||
const nextRun = computeNextRun(changes.cronExpr);
|
||||
setClauses.push('cron_expr = ?');
|
||||
values.push(changes.cronExpr);
|
||||
setClauses.push('next_run_at = ?');
|
||||
values.push(nextRun);
|
||||
}
|
||||
if (changes.jobConfig !== undefined) {
|
||||
setClauses.push('job_config = ?');
|
||||
values.push(JSON.stringify(changes.jobConfig));
|
||||
}
|
||||
if (changes.workspaceId !== undefined) {
|
||||
setClauses.push('workspace_id = ?');
|
||||
values.push(changes.workspaceId);
|
||||
}
|
||||
if (changes.enabled !== undefined) {
|
||||
setClauses.push('enabled = ?');
|
||||
values.push(changes.enabled ? 1 : 0);
|
||||
}
|
||||
|
||||
if (setClauses.length > 0) {
|
||||
values.push(id);
|
||||
this.db.getDatabase().prepare(
|
||||
`UPDATE cron_schedules SET ${setClauses.join(', ')} WHERE id = ?`,
|
||||
).run(...values);
|
||||
}
|
||||
|
||||
return this.getById(id)!;
|
||||
}
|
||||
|
||||
/** Delete a schedule by ID. */
|
||||
delete(id: number): void {
|
||||
this.db.getDatabase().prepare(
|
||||
'DELETE FROM cron_schedules WHERE id = ?',
|
||||
).run(id);
|
||||
}
|
||||
|
||||
/** Get all enabled schedules whose next_run_at is in the past. */
|
||||
getDue(): CronSchedule[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
"SELECT * FROM cron_schedules WHERE enabled = 1 AND next_run_at <= datetime('now')",
|
||||
).all() as CronSchedule[];
|
||||
}
|
||||
|
||||
/** Mark a schedule as having just run. Updates last_run_at and recomputes next_run_at. */
|
||||
markRun(id: number): void {
|
||||
const schedule = this.getById(id);
|
||||
if (!schedule) return;
|
||||
|
||||
const nextRun = computeNextRun(schedule.cron_expr);
|
||||
this.db.getDatabase().prepare(
|
||||
"UPDATE cron_schedules SET last_run_at = datetime('now'), next_run_at = ? WHERE id = ?",
|
||||
).run(nextRun, id);
|
||||
}
|
||||
|
||||
/** Clear all schedules (for testing). */
|
||||
clear(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM cron_schedules').run();
|
||||
}
|
||||
|
||||
// ── W5.12: Cron Execution History ──────────────────────────────────
|
||||
|
||||
/** Record a cron job execution result. */
|
||||
recordExecution(scheduleId: number, scheduleName: string, opts: {
|
||||
executedAt?: string;
|
||||
durationMs?: number;
|
||||
success: boolean;
|
||||
resultSummary?: string;
|
||||
error?: string;
|
||||
}): void {
|
||||
this.db.getDatabase().prepare(
|
||||
`INSERT INTO cron_execution_history (schedule_id, schedule_name, executed_at, duration_ms, success, result_summary, error)
|
||||
VALUES (?, ?, COALESCE(?, datetime('now')), ?, ?, ?, ?)`,
|
||||
).run(scheduleId, scheduleName, opts.executedAt ?? null, opts.durationMs ?? null, opts.success ? 1 : 0, opts.resultSummary ?? null, opts.error ?? null);
|
||||
}
|
||||
|
||||
/** #17: count today's (UTC) executions for a schedule — ai_task daily cap. */
|
||||
countExecutionsToday(scheduleId: number): number {
|
||||
const row = this.db.getDatabase().prepare(
|
||||
"SELECT COUNT(*) AS n FROM cron_execution_history WHERE schedule_id = ? AND executed_at >= date('now')",
|
||||
).get(scheduleId) as { n: number };
|
||||
return row.n;
|
||||
}
|
||||
|
||||
/** Get execution history for a schedule (most recent first). */
|
||||
getExecutionHistory(scheduleId: number, limit = 20): CronExecutionRow[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM cron_execution_history WHERE schedule_id = ? ORDER BY executed_at DESC LIMIT ?',
|
||||
).all(scheduleId, limit) as CronExecutionRow[];
|
||||
}
|
||||
|
||||
/** Get the most recent executions for boot-time failure-state recovery. */
|
||||
getRecentExecutions(scheduleId: number, limit = 5): CronExecutionRow[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM cron_execution_history WHERE schedule_id = ? ORDER BY executed_at DESC, id DESC LIMIT ?',
|
||||
).all(scheduleId, limit) as CronExecutionRow[];
|
||||
}
|
||||
|
||||
/** Prune execution-history rows older than N days. recordExecution writes a
|
||||
* row per tick (UX-Refactor Phase 3, Journey 16), so without retention the
|
||||
* table grows unbounded (a per-minute job ≈ 525k rows/year). Mirrors
|
||||
* optStore.pruneOlderThan(30). Returns the number of rows deleted. */
|
||||
pruneExecutionHistory(olderThanDays: number): number {
|
||||
const days = Math.max(1, Math.floor(olderThanDays));
|
||||
const result = this.db.getDatabase().prepare(
|
||||
"DELETE FROM cron_execution_history WHERE executed_at < datetime('now', ?)",
|
||||
).run(`-${days} days`);
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
// ── Interrupted-run leases ────────────────────────────────────────
|
||||
|
||||
/** Acquire a durable lease before a scheduled job begins execution. */
|
||||
acquireRunLease(scheduleId: number, name: string, pid: number): number {
|
||||
const result = this.db.getDatabase().prepare(
|
||||
'INSERT INTO cron_run_leases (schedule_id, schedule_name, pid) VALUES (?, ?, ?)',
|
||||
).run(scheduleId, name, pid);
|
||||
return Number(result.lastInsertRowid);
|
||||
}
|
||||
|
||||
/** Release a run lease after its scheduled job finishes. */
|
||||
releaseRunLease(leaseId: number): void {
|
||||
this.db.getDatabase().prepare(
|
||||
'DELETE FROM cron_run_leases WHERE id = ?',
|
||||
).run(leaseId);
|
||||
}
|
||||
|
||||
/** List leases left behind by an interrupted process. */
|
||||
listStaleRunLeases(): CronRunLeaseRow[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM cron_run_leases ORDER BY started_at ASC, id ASC',
|
||||
).all() as CronRunLeaseRow[];
|
||||
}
|
||||
|
||||
/** Clear all interrupted-run leases after boot recovery. */
|
||||
clearRunLeases(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM cron_run_leases').run();
|
||||
}
|
||||
|
||||
// ── W5.10: Notification Persistence ────────────────────────────────
|
||||
|
||||
/** Save a notification. */
|
||||
saveNotification(title: string, body: string, category = 'system', actionUrl?: string): number {
|
||||
const result = this.db.getDatabase().prepare(
|
||||
'INSERT INTO notifications (title, body, category, action_url) VALUES (?, ?, ?, ?)',
|
||||
).run(title, body, category, actionUrl ?? null);
|
||||
return Number(result.lastInsertRowid);
|
||||
}
|
||||
|
||||
/** Get recent notifications (newest first). */
|
||||
getNotifications(opts?: { since?: string; limit?: number; unreadOnly?: boolean }): NotificationRow[] {
|
||||
const limit = opts?.limit ?? 50;
|
||||
let sql = 'SELECT * FROM notifications';
|
||||
const conditions: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
if (opts?.since) { conditions.push('created_at > ?'); params.push(opts.since); }
|
||||
if (opts?.unreadOnly) { conditions.push('read = 0'); }
|
||||
if (conditions.length > 0) sql += ' WHERE ' + conditions.join(' AND ');
|
||||
sql += ' ORDER BY created_at DESC LIMIT ?';
|
||||
params.push(limit);
|
||||
return this.db.getDatabase().prepare(sql).all(...params) as NotificationRow[];
|
||||
}
|
||||
|
||||
/** Mark a notification as read. */
|
||||
markNotificationRead(id: number): void {
|
||||
this.db.getDatabase().prepare('UPDATE notifications SET read = 1 WHERE id = ?').run(id);
|
||||
}
|
||||
|
||||
/** Mark all notifications as read. Returns the number of rows updated. */
|
||||
markAllRead(): number {
|
||||
const result = this.db.getDatabase().prepare('UPDATE notifications SET read = 1 WHERE read = 0').run();
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
/** Count unread notifications. */
|
||||
countUnread(): number {
|
||||
const row = this.db.getDatabase().prepare('SELECT COUNT(*) as cnt FROM notifications WHERE read = 0').get() as { cnt: number };
|
||||
return row.cnt;
|
||||
}
|
||||
|
||||
// ── L2: Held-action approval queue ─────────────────────────────────
|
||||
|
||||
/** Enqueue a held action (status 'held'). Returns the persisted row. */
|
||||
savePendingAction(input: SavePendingActionInput): PendingActionRow {
|
||||
this.db.getDatabase().prepare(`
|
||||
INSERT INTO pending_actions (id, workspace_id, source, tool_name, args_json, summary, risk_level, approval_class, status, expires_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'held', ?)
|
||||
`).run(
|
||||
input.id,
|
||||
input.workspaceId,
|
||||
input.source,
|
||||
input.toolName,
|
||||
input.argsJson,
|
||||
input.summary ?? null,
|
||||
input.riskLevel,
|
||||
input.approvalClass,
|
||||
input.expiresAt ?? null,
|
||||
);
|
||||
return this.getPendingAction(input.id)!;
|
||||
}
|
||||
|
||||
/** List held actions (newest first) — defaults to the 'held' queue. */
|
||||
listPendingActions(status: PendingActionStatus = 'held'): PendingActionRow[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM pending_actions WHERE status = ? ORDER BY created_at DESC',
|
||||
).all(status) as PendingActionRow[];
|
||||
}
|
||||
|
||||
/** Get one held action by id. */
|
||||
getPendingAction(id: string): PendingActionRow | undefined {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM pending_actions WHERE id = ?',
|
||||
).get(id) as PendingActionRow | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomically claim a held action — transition 'held' → 'approved' | 'denied'.
|
||||
* This is the idempotency gate: exactly ONE caller wins (the `WHERE status =
|
||||
* 'held'` makes it atomic in SQLite), so a double-approve / approve-after-deny
|
||||
* is a no-op. Returns the updated row if THIS call won the claim, else
|
||||
* undefined (already decided or missing).
|
||||
*/
|
||||
claimPendingAction(id: string, status: 'approved' | 'denied', decidedAt: string): PendingActionRow | undefined {
|
||||
const result = this.db.getDatabase().prepare(
|
||||
"UPDATE pending_actions SET status = ?, decided_at = ? WHERE id = ? AND status = 'held'",
|
||||
).run(status, decidedAt, id);
|
||||
if (result.changes === 0) return undefined;
|
||||
return this.getPendingAction(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Record the terminal outcome of a claimed action ('approved' → 'executed' |
|
||||
* 'failed'). Unconditional — the caller already won claimPendingAction(), so
|
||||
* it owns the row and no further guard is needed.
|
||||
*/
|
||||
updatePendingActionResult(id: string, changes: { status: 'executed' | 'failed'; resultSummary?: string; error?: string; executedAt?: string }): void {
|
||||
const sets: string[] = ['status = ?'];
|
||||
const vals: unknown[] = [changes.status];
|
||||
if (changes.resultSummary !== undefined) { sets.push('result_summary = ?'); vals.push(changes.resultSummary); }
|
||||
if (changes.error !== undefined) { sets.push('error = ?'); vals.push(changes.error); }
|
||||
if (changes.executedAt !== undefined) { sets.push('executed_at = ?'); vals.push(changes.executedAt); }
|
||||
vals.push(id);
|
||||
this.db.getDatabase().prepare(
|
||||
`UPDATE pending_actions SET ${sets.join(', ')} WHERE id = ?`,
|
||||
).run(...vals);
|
||||
}
|
||||
|
||||
/** Flip held actions past their expires_at to 'expired'. Returns rows changed. */
|
||||
expireStalePendingActions(): number {
|
||||
const result = this.db.getDatabase().prepare(
|
||||
"UPDATE pending_actions SET status = 'expired' WHERE status = 'held' AND expires_at IS NOT NULL AND expires_at < datetime('now')",
|
||||
).run();
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
/** Clear all pending actions (for testing). */
|
||||
clearPendingActions(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM pending_actions').run();
|
||||
}
|
||||
}
|
||||
259
packages/core/src/file-indexer.ts
Normal file
259
packages/core/src/file-indexer.ts
Normal file
@@ -0,0 +1,259 @@
|
||||
/**
|
||||
* FileIndexer — auto-index workspace files into their workspace mind (L-20).
|
||||
*
|
||||
* Design decisions (see docs/plans/FILE-TOOLS-AUDIT-2026-04-20.md):
|
||||
* • Trigger: on-upload / on-overwrite (called from route handlers)
|
||||
* • Target mind: per-workspace (the mind you pass in the constructor)
|
||||
* • Granularity: 1 frame per file
|
||||
* • Formats: .md, .markdown, .txt only for v1 (Bucket 2 adds PDF/DOCX)
|
||||
* • Cleanup: re-index on overwrite, delete index on file remove
|
||||
*
|
||||
* A `file_index` table tracks the file_path → frame_id link so overwrites can
|
||||
* swap the old frame cleanly and moves/deletes can locate the right frame
|
||||
* without scanning every frame in the workspace.
|
||||
*
|
||||
* Large files are truncated to MAX_CONTENT_BYTES with a marker appended; the
|
||||
* full file is still on disk, only the indexed frame is shortened. Keeps the
|
||||
* mind DB size bounded even if a user drops a 5MB markdown dump.
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
import { FrameStore } from '@waggle/hive-mind-core';
|
||||
import { SessionStore } from '@waggle/hive-mind-core';
|
||||
|
||||
const FILE_INDEX_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS file_index (
|
||||
file_path TEXT PRIMARY KEY,
|
||||
frame_id INTEGER NOT NULL,
|
||||
mime_type TEXT,
|
||||
size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
content_hash TEXT NOT NULL,
|
||||
indexed_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
)
|
||||
`;
|
||||
const FILE_INDEX_IDX_SQL = `
|
||||
CREATE INDEX IF NOT EXISTS idx_file_index_frame ON file_index (frame_id)
|
||||
`;
|
||||
|
||||
const INDEX_SESSION_GOP_ID = 'session:file-index';
|
||||
const INDEX_SESSION_PROJECT = 'file-indexer';
|
||||
|
||||
/** Maximum bytes of file content we store in a single frame. Beyond this, the
|
||||
* content is truncated with a marker. Keeps mind size bounded on large dumps. */
|
||||
export const MAX_CONTENT_BYTES = 64 * 1024;
|
||||
|
||||
/** Extensions we index for v1. PDF/DOCX/XLSX land in Bucket 2. */
|
||||
const INDEXABLE_EXTENSIONS = new Set(['.md', '.markdown', '.txt']);
|
||||
|
||||
export interface FileIndexRow {
|
||||
filePath: string;
|
||||
frameId: number;
|
||||
mimeType: string | null;
|
||||
sizeBytes: number;
|
||||
contentHash: string;
|
||||
indexedAt: string;
|
||||
}
|
||||
|
||||
export type FileIndexResult =
|
||||
| { skipped: true; reason: 'unsupported_format' | 'unchanged' | 'empty' }
|
||||
| { skipped: false; frameId: number; truncated: boolean };
|
||||
|
||||
export class FileIndexer {
|
||||
private readonly db: MindDB;
|
||||
private readonly frames: FrameStore;
|
||||
private readonly sessions: SessionStore;
|
||||
private cachedGopId: string | null = null;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.frames = new FrameStore(db);
|
||||
this.sessions = new SessionStore(db);
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
const existsRow = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='file_index'",
|
||||
).get();
|
||||
if (existsRow) return;
|
||||
raw.prepare(FILE_INDEX_TABLE_SQL).run();
|
||||
raw.prepare(FILE_INDEX_IDX_SQL).run();
|
||||
}
|
||||
|
||||
/** True iff the path's extension is one we index in v1. */
|
||||
static shouldIndex(filePath: string): boolean {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
return INDEXABLE_EXTENSIONS.has(ext);
|
||||
}
|
||||
|
||||
private ensureSession(): string {
|
||||
if (this.cachedGopId) return this.cachedGopId;
|
||||
const session = this.sessions.ensure(
|
||||
INDEX_SESSION_GOP_ID,
|
||||
INDEX_SESSION_PROJECT,
|
||||
'Auto-indexed workspace files',
|
||||
);
|
||||
this.cachedGopId = session.gop_id;
|
||||
return session.gop_id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Index a file. Idempotent on unchanged content (same hash → skip). On
|
||||
* overwrite with new content, deletes the old frame and inserts a fresh one.
|
||||
*/
|
||||
indexFile(filePath: string, content: Buffer, mime?: string): FileIndexResult {
|
||||
if (!FileIndexer.shouldIndex(filePath)) {
|
||||
return { skipped: true, reason: 'unsupported_format' };
|
||||
}
|
||||
if (content.length === 0) {
|
||||
// Still remove any stale index so an emptied file doesn't leave an
|
||||
// orphan frame pointing at old content.
|
||||
this.removeFile(filePath);
|
||||
return { skipped: true, reason: 'empty' };
|
||||
}
|
||||
|
||||
const hash = createHash('sha256').update(content).digest('hex');
|
||||
const raw = this.db.getDatabase();
|
||||
const existingRow = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(filePath) as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
|
||||
if (existingRow && existingRow.content_hash === hash) {
|
||||
return { skipped: true, reason: 'unchanged' };
|
||||
}
|
||||
|
||||
const normalized = this.normalizeText(content);
|
||||
const truncated = content.length > MAX_CONTENT_BYTES;
|
||||
const frameBody = this.buildFrameContent(filePath, normalized, truncated, mime);
|
||||
const gopId = this.ensureSession();
|
||||
// Atomicity contract (L-20 BLOCKER-1 fix): createIFrame → (conditional
|
||||
// old-frame delete) → UPDATE/INSERT on file_index must commit as one unit.
|
||||
// If any step throws mid-callback, better-sqlite3 rolls back the new frame
|
||||
// and the index-row write, leaving the table in its pre-call state. Without
|
||||
// this, a crash between the delete and the UPDATE would leave file_index
|
||||
// pointing at a deleted frame_id (dangling) or an uncommitted new frame
|
||||
// orphaned in the frames table.
|
||||
const mutate = raw.transaction(() => {
|
||||
const frame = this.frames.createIFrame(gopId, frameBody, 'normal', 'system');
|
||||
|
||||
if (existingRow) {
|
||||
// Overwrite path: delete the old frame before we swap the row, unless
|
||||
// the dedup coalesced two paths onto the same frame (leave that frame
|
||||
// alone in that case — the other path still references it).
|
||||
const oldFrameId = Number(existingRow.frame_id);
|
||||
if (oldFrameId !== frame.id) {
|
||||
const otherRef = raw.prepare('SELECT 1 FROM file_index WHERE frame_id = ? AND file_path != ? LIMIT 1').get(oldFrameId, filePath);
|
||||
if (!otherRef) {
|
||||
this.frames.delete(oldFrameId);
|
||||
}
|
||||
}
|
||||
raw.prepare(`
|
||||
UPDATE file_index SET
|
||||
frame_id = ?,
|
||||
mime_type = ?,
|
||||
size_bytes = ?,
|
||||
content_hash = ?,
|
||||
indexed_at = datetime('now')
|
||||
WHERE file_path = ?
|
||||
`).run(frame.id, mime ?? null, content.length, hash, filePath);
|
||||
} else {
|
||||
raw.prepare(`
|
||||
INSERT INTO file_index (file_path, frame_id, mime_type, size_bytes, content_hash)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`).run(filePath, frame.id, mime ?? null, content.length, hash);
|
||||
}
|
||||
|
||||
return frame.id;
|
||||
});
|
||||
|
||||
const frameId = mutate();
|
||||
return { skipped: false, frameId, truncated };
|
||||
}
|
||||
|
||||
/** Remove a file's frame + index row. Returns true if anything was removed. */
|
||||
removeFile(filePath: string): boolean {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT frame_id FROM file_index WHERE file_path = ?').get(filePath) as
|
||||
| { frame_id: number }
|
||||
| undefined;
|
||||
if (!row) return false;
|
||||
|
||||
const otherRef = raw.prepare('SELECT 1 FROM file_index WHERE frame_id = ? AND file_path != ? LIMIT 1').get(row.frame_id, filePath);
|
||||
if (!otherRef) {
|
||||
this.frames.delete(row.frame_id);
|
||||
}
|
||||
raw.prepare('DELETE FROM file_index WHERE file_path = ?').run(filePath);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the recorded path for a moved file. The frame content itself still
|
||||
* carries the old path in its header — that's acceptable for v1 since the
|
||||
* frame body is already the file's own content (not a description). A later
|
||||
* reader sees the stale header only if they dig into raw frame content; the
|
||||
* index table, which is what every other code path queries, is correct.
|
||||
*/
|
||||
moveFile(from: string, to: string): boolean {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(from) as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
if (!row) return false;
|
||||
|
||||
// If `to` already indexed, drop its entry (file was replaced by the move).
|
||||
raw.prepare('DELETE FROM file_index WHERE file_path = ?').run(to);
|
||||
raw.prepare('UPDATE file_index SET file_path = ?, indexed_at = datetime(\'now\') WHERE file_path = ?').run(to, from);
|
||||
return true;
|
||||
}
|
||||
|
||||
getRow(filePath: string): FileIndexRow | null {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(filePath) as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
if (!row) return null;
|
||||
return {
|
||||
filePath: row.file_path as string,
|
||||
frameId: row.frame_id as number,
|
||||
mimeType: (row.mime_type as string | null) ?? null,
|
||||
sizeBytes: row.size_bytes as number,
|
||||
contentHash: row.content_hash as string,
|
||||
indexedAt: row.indexed_at as string,
|
||||
};
|
||||
}
|
||||
|
||||
listAll(): FileIndexRow[] {
|
||||
const raw = this.db.getDatabase();
|
||||
const rows = raw.prepare('SELECT * FROM file_index ORDER BY indexed_at DESC').all() as Record<string, unknown>[];
|
||||
return rows.map(r => ({
|
||||
filePath: r.file_path as string,
|
||||
frameId: r.frame_id as number,
|
||||
mimeType: (r.mime_type as string | null) ?? null,
|
||||
sizeBytes: r.size_bytes as number,
|
||||
contentHash: r.content_hash as string,
|
||||
indexedAt: r.indexed_at as string,
|
||||
}));
|
||||
}
|
||||
|
||||
private normalizeText(content: Buffer): string {
|
||||
// UTF-8 decode. Malformed bytes become U+FFFD; acceptable for index use.
|
||||
const full = content.toString('utf-8');
|
||||
if (Buffer.byteLength(full, 'utf-8') <= MAX_CONTENT_BYTES) return full;
|
||||
// Truncate in character-safe chunks. Walk down until we're under the byte
|
||||
// budget; avoids cutting a multi-byte character in half.
|
||||
let candidate = full.slice(0, MAX_CONTENT_BYTES);
|
||||
while (Buffer.byteLength(candidate, 'utf-8') > MAX_CONTENT_BYTES && candidate.length > 0) {
|
||||
candidate = candidate.slice(0, candidate.length - 1);
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
private buildFrameContent(filePath: string, body: string, truncated: boolean, mime?: string): string {
|
||||
const header = `[FILE: ${filePath}${mime ? ` · ${mime}` : ''}]`;
|
||||
const suffix = truncated ? `\n\n[…truncated at ${MAX_CONTENT_BYTES} bytes for indexing]` : '';
|
||||
return `${header}\n\n${body}${suffix}`;
|
||||
}
|
||||
}
|
||||
639
packages/core/src/file-store.ts
Normal file
639
packages/core/src/file-store.ts
Normal file
@@ -0,0 +1,639 @@
|
||||
/**
|
||||
* FileStore — workspace file storage abstraction.
|
||||
*
|
||||
* Provides a clean interface for reading/writing workspace files that abstracts
|
||||
* over the underlying storage backend. Two implementations:
|
||||
*
|
||||
* - LocalFileStore: manages files in ~/.waggle/workspaces/{id}/files/ (virtual storage)
|
||||
* - LinkedDirStore: reads/writes to workspace.directory (linked to external folder)
|
||||
* - S3FileStore: MinIO/S3-backed storage for team/cloud deployments (lazy SDK import)
|
||||
*
|
||||
* All operations enforce path traversal protection — files cannot escape
|
||||
* the workspace boundary.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { glob } from 'glob';
|
||||
// Type-only import: erased at compile time, so it does not force @aws-sdk/client-s3
|
||||
// (a devDependency) into the runtime bundle for local-only users.
|
||||
import type { S3Client } from '@aws-sdk/client-s3';
|
||||
|
||||
// ── Interface ───────────────────────────────────────────────────────
|
||||
|
||||
export interface FileEntry {
|
||||
name: string;
|
||||
path: string; // relative to workspace root
|
||||
size: number;
|
||||
modified: string; // ISO timestamp
|
||||
isDirectory: boolean;
|
||||
}
|
||||
|
||||
export interface StorageInfo {
|
||||
usedBytes: number;
|
||||
fileCount: number;
|
||||
storageType: 'virtual' | 'linked';
|
||||
}
|
||||
|
||||
export interface FileStore {
|
||||
// Read
|
||||
readFile(relativePath: string): Promise<Buffer>;
|
||||
listFiles(directory?: string): Promise<FileEntry[]>;
|
||||
searchFiles(pattern: string): Promise<FileEntry[]>;
|
||||
|
||||
// Write
|
||||
writeFile(relativePath: string, content: Buffer | string): Promise<void>;
|
||||
deleteFile(relativePath: string): Promise<void>;
|
||||
moveFile(from: string, to: string): Promise<void>;
|
||||
|
||||
// Meta
|
||||
getStorageInfo(): Promise<StorageInfo>;
|
||||
getRootPath(): string;
|
||||
getStorageType(): 'virtual' | 'linked';
|
||||
}
|
||||
|
||||
// ── Path safety ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Directory segments that, anywhere in a path, almost always hold secrets.
|
||||
* Matched (normalized, case-insensitive) against each path segment.
|
||||
*/
|
||||
const SENSITIVE_DIR_SEGMENTS = new Set([
|
||||
'.ssh', '.aws', '.gnupg', '.gpg', '.docker', '.kube', '.azure', '.terraform', '.terraform.d',
|
||||
]);
|
||||
|
||||
/** Exact basenames (normalized) that are secret material. */
|
||||
const SENSITIVE_BASENAMES = new Set([
|
||||
'id_rsa', 'id_dsa', 'id_ecdsa', 'id_ed25519', 'authorized_keys', 'known_hosts',
|
||||
'.netrc', '.pgpass', '.npmrc', '.pypirc', '.git-credentials',
|
||||
'credentials', 'credentials.json', 'service-account.json',
|
||||
'terraform.tfstate', 'terraform.tfstate.backup',
|
||||
]);
|
||||
|
||||
/** Extensions that are (almost always) private-key material. */
|
||||
const SENSITIVE_EXTENSIONS = new Set(['.pem']);
|
||||
|
||||
/** Backup/copy suffixes — strip and re-test the base (id_rsa.bak → id_rsa). */
|
||||
const BACKUP_SUFFIX_RE = /\.(bak|old|backup|orig|copy|save|swp)$/i;
|
||||
|
||||
/** `.env` files are secrets — but the documented, checked-in templates are not. */
|
||||
const ENV_TEMPLATE_ALLOW = new Set(['.env.example', '.env.sample', '.env.template', '.env.dist', '.env.defaults']);
|
||||
|
||||
/**
|
||||
* Normalize one path segment to the name the OS will actually open: lower-case
|
||||
* (case-insensitive FS), strip a Windows NTFS alternate-data-stream suffix
|
||||
* (`id_rsa::$DATA` → `id_rsa`) and any trailing dots/spaces (`id_rsa.`, `.env `
|
||||
* → the base) which Windows silently removes when opening.
|
||||
*/
|
||||
function normalizeSegment(seg: string): string {
|
||||
return seg.toLowerCase().replace(/::.*$/, '').replace(/[. ]+$/, '');
|
||||
}
|
||||
|
||||
function isSensitiveBase(base: string): boolean {
|
||||
if (SENSITIVE_BASENAMES.has(base)) return true;
|
||||
const dot = base.lastIndexOf('.');
|
||||
if (dot > 0 && SENSITIVE_EXTENSIONS.has(base.slice(dot))) return true;
|
||||
if (base === '.env' || base.startsWith('.env.')) return !ENV_TEMPLATE_ALLOW.has(base);
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a path points at well-known secret material (SSH/GPG keys, cloud
|
||||
* credentials, dotenv files, terraform state, …). Used to deny reads/writes
|
||||
* inside LINKED external folders so an agent given a project directory cannot
|
||||
* exfiltrate or clobber the user's secrets.
|
||||
*
|
||||
* It is a BLOCKLIST (defense-in-depth), not a sandbox: it raises the bar against
|
||||
* obvious secrets but cannot enumerate every secret a home dir holds. Conservative
|
||||
* on extensions (only *.pem) to avoid denying legitimate files. Path-separator
|
||||
* agnostic; segments are normalized for case + Windows ADS/trailing-char tricks.
|
||||
*/
|
||||
export function isSensitiveFilePath(relativePath: string): boolean {
|
||||
const segments = relativePath.replace(/\\/g, '/').split('/').map(normalizeSegment).filter(Boolean);
|
||||
if (segments.length === 0) return false;
|
||||
for (const seg of segments) {
|
||||
if (SENSITIVE_DIR_SEGMENTS.has(seg)) return true;
|
||||
}
|
||||
const base = segments[segments.length - 1];
|
||||
if (isSensitiveBase(base)) return true;
|
||||
if (BACKUP_SUFFIX_RE.test(base) && isSensitiveBase(base.replace(BACKUP_SUFFIX_RE, ''))) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
interface ResolveSafeOptions {
|
||||
/** Reject paths flagged by isSensitiveFilePath (linked external dirs only). */
|
||||
denySensitive?: boolean;
|
||||
}
|
||||
|
||||
/** realpathSync, falling back to the input if it can't be resolved (e.g. not created yet). */
|
||||
function safeRealpath(p: string): string {
|
||||
try { return fs.realpathSync(p); } catch { return p; }
|
||||
}
|
||||
|
||||
/** Walk up to the deepest ancestor of `p` that exists on disk. */
|
||||
function deepestExisting(p: string): string {
|
||||
let cur = p;
|
||||
while (cur !== path.dirname(cur) && !fs.existsSync(cur)) cur = path.dirname(cur);
|
||||
return cur;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve `relativePath` under `root` and assert it cannot escape the boundary.
|
||||
*
|
||||
* Two layers:
|
||||
* 1. LEXICAL segment-boundary containment — the resolved path is the root or
|
||||
* sits under `root + sep` (a bare `startsWith(root)` wrongly admits a sibling
|
||||
* like `${root}-evil/secret`).
|
||||
* 2. SYMLINK-aware containment — `path.resolve` is lexical but `fs.*` follows
|
||||
* symlinks, so we realpath the deepest EXISTING ancestor of the target and
|
||||
* re-check it is still under the realpath'd root. This blocks a benign-named
|
||||
* symlink that points outside the boundary, while still permitting in-root
|
||||
* symlinks (e.g. monorepo package links). The sensitive-file deny then runs
|
||||
* on BOTH the lexical and the real in-root path, so an in-root symlink to a
|
||||
* secret (`alias → ./.env`) cannot launder it past a benign basename.
|
||||
*/
|
||||
function resolveSafe(root: string, relativePath: string, opts: ResolveSafeOptions = {}): string {
|
||||
const resolvedRoot = path.resolve(root);
|
||||
const resolved = path.resolve(resolvedRoot, relativePath);
|
||||
if (resolved !== resolvedRoot && !resolved.startsWith(resolvedRoot + path.sep)) {
|
||||
throw new Error(`Path traversal denied: ${relativePath}`);
|
||||
}
|
||||
|
||||
// Symlink-aware containment only applies once the root exists on disk — if it
|
||||
// doesn't, nothing inside it exists to be a symlink, and walking the deepest
|
||||
// existing ancestor above the (not-yet-created) root would compare against an
|
||||
// unrelated real dir (e.g. an OS temp-dir symlink). Lexical containment holds.
|
||||
const realRoot = safeRealpath(resolvedRoot);
|
||||
let realTarget = realRoot;
|
||||
if (fs.existsSync(resolvedRoot)) {
|
||||
realTarget = safeRealpath(deepestExisting(resolved));
|
||||
if (realTarget !== realRoot && !realTarget.startsWith(realRoot + path.sep)) {
|
||||
throw new Error(`Path traversal denied (symlink): ${relativePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (opts.denySensitive) {
|
||||
const lexicalRel = path.relative(resolvedRoot, resolved);
|
||||
const realRel = path.relative(realRoot, realTarget);
|
||||
if (isSensitiveFilePath(lexicalRel) || isSensitiveFilePath(realRel)) {
|
||||
throw new Error(`Access to sensitive file denied: ${relativePath}`);
|
||||
}
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
/** Keep only glob matches that resolve back inside `root` (glob `../` patterns can escape cwd). */
|
||||
function containGlobMatches(root: string, matches: string[]): string[] {
|
||||
const resolvedRoot = path.resolve(root);
|
||||
return matches.filter(m => {
|
||||
const abs = path.resolve(resolvedRoot, m);
|
||||
return abs === resolvedRoot || abs.startsWith(resolvedRoot + path.sep);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a `*`/`?` glob to a RegExp that is SAFE against catastrophic
|
||||
* backtracking (ReDoS). Every regex metacharacter is escaped first, so the only
|
||||
* specials left are the two wildcards we re-introduce (`.*`, `.`) — neither can
|
||||
* nest a quantifier, so a hostile pattern like `(a+)+x` becomes a harmless
|
||||
* literal. Used by S3FileStore.searchFiles where the pattern is caller-controlled.
|
||||
* Unanchored (substring match) to preserve the prior behavior.
|
||||
*/
|
||||
function globToRegExp(pattern: string): RegExp {
|
||||
const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const body = escaped.replace(/\\\*/g, '.*').replace(/\\\?/g, '.');
|
||||
return new RegExp(body, 'i');
|
||||
}
|
||||
|
||||
/**
|
||||
* Reject an S3 key relative-path that would escape its workspace prefix. S3 keys
|
||||
* are literal, but MinIO (path-style, filesystem-backed) can normalize `..`, so a
|
||||
* `..` segment could cross into a sibling workspace's prefix. Makes the file
|
||||
* header's "all operations enforce path traversal protection" true for S3 too.
|
||||
*/
|
||||
function assertSafeRelativeKey(relativePath: string): void {
|
||||
const segments = relativePath.replace(/\\/g, '/').split('/');
|
||||
if (segments.some(s => s === '..')) {
|
||||
throw new Error(`Path traversal denied: ${relativePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
function ensureDir(dirPath: string): void {
|
||||
if (!fs.existsSync(dirPath)) {
|
||||
fs.mkdirSync(dirPath, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
// ── LocalFileStore (virtual workspace storage) ──────────────────────
|
||||
|
||||
export class LocalFileStore implements FileStore {
|
||||
private readonly root: string;
|
||||
|
||||
constructor(dataDir: string, workspaceId: string) {
|
||||
this.root = path.join(dataDir, 'workspaces', workspaceId, 'files');
|
||||
}
|
||||
|
||||
getRootPath(): string { return this.root; }
|
||||
getStorageType(): 'virtual' { return 'virtual'; }
|
||||
|
||||
async readFile(relativePath: string): Promise<Buffer> {
|
||||
const fullPath = resolveSafe(this.root, relativePath);
|
||||
return fs.readFileSync(fullPath);
|
||||
}
|
||||
|
||||
async listFiles(directory?: string): Promise<FileEntry[]> {
|
||||
const dir = directory ? resolveSafe(this.root, directory) : this.root;
|
||||
if (!fs.existsSync(dir)) return [];
|
||||
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
return entries.map(entry => {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
const stat = fs.statSync(fullPath);
|
||||
return {
|
||||
name: entry.name,
|
||||
path: path.relative(this.root, fullPath).replace(/\\/g, '/'),
|
||||
size: stat.size,
|
||||
modified: stat.mtime.toISOString(),
|
||||
isDirectory: entry.isDirectory(),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async searchFiles(pattern: string): Promise<FileEntry[]> {
|
||||
if (!fs.existsSync(this.root)) return [];
|
||||
const matches = await glob(pattern, {
|
||||
cwd: this.root,
|
||||
nodir: true,
|
||||
ignore: ['node_modules/**', '.git/**'],
|
||||
});
|
||||
// A glob pattern with `../` can escape cwd — keep only matches inside root.
|
||||
return containGlobMatches(this.root, matches).slice(0, 200).map(match => {
|
||||
const fullPath = path.join(this.root, match);
|
||||
try {
|
||||
const stat = fs.statSync(fullPath);
|
||||
return {
|
||||
name: path.basename(match),
|
||||
path: match.replace(/\\/g, '/'),
|
||||
size: stat.size,
|
||||
modified: stat.mtime.toISOString(),
|
||||
isDirectory: false,
|
||||
};
|
||||
} catch {
|
||||
return { name: path.basename(match), path: match, size: 0, modified: '', isDirectory: false };
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async writeFile(relativePath: string, content: Buffer | string): Promise<void> {
|
||||
const fullPath = resolveSafe(this.root, relativePath);
|
||||
// Lazy directory creation
|
||||
ensureDir(path.dirname(fullPath));
|
||||
fs.writeFileSync(fullPath, content);
|
||||
}
|
||||
|
||||
async deleteFile(relativePath: string): Promise<void> {
|
||||
const fullPath = resolveSafe(this.root, relativePath);
|
||||
if (fs.existsSync(fullPath)) {
|
||||
const stat = fs.statSync(fullPath);
|
||||
if (stat.isDirectory()) {
|
||||
fs.rmSync(fullPath, { recursive: true });
|
||||
} else {
|
||||
fs.unlinkSync(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async moveFile(from: string, to: string): Promise<void> {
|
||||
const fromPath = resolveSafe(this.root, from);
|
||||
const toPath = resolveSafe(this.root, to);
|
||||
ensureDir(path.dirname(toPath));
|
||||
fs.renameSync(fromPath, toPath);
|
||||
}
|
||||
|
||||
async getStorageInfo(): Promise<StorageInfo> {
|
||||
if (!fs.existsSync(this.root)) {
|
||||
return { usedBytes: 0, fileCount: 0, storageType: 'virtual' };
|
||||
}
|
||||
let usedBytes = 0;
|
||||
let fileCount = 0;
|
||||
const walk = (dir: string) => {
|
||||
try {
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
walk(fullPath);
|
||||
} else {
|
||||
try {
|
||||
usedBytes += fs.statSync(fullPath).size;
|
||||
fileCount++;
|
||||
} catch { /* skip unreadable */ }
|
||||
}
|
||||
}
|
||||
} catch { /* skip unreadable dirs */ }
|
||||
};
|
||||
walk(this.root);
|
||||
return { usedBytes, fileCount, storageType: 'virtual' };
|
||||
}
|
||||
}
|
||||
|
||||
// ── LinkedDirStore (linked to external directory) ───────────────────
|
||||
|
||||
export class LinkedDirStore implements FileStore {
|
||||
private readonly root: string;
|
||||
|
||||
constructor(directory: string) {
|
||||
this.root = directory;
|
||||
}
|
||||
|
||||
getRootPath(): string { return this.root; }
|
||||
getStorageType(): 'linked' { return 'linked'; }
|
||||
|
||||
// Linked stores point at a REAL external folder (a code project, even the home
|
||||
// dir), so every path op denies well-known secret files — the agent cannot
|
||||
// read or clobber ~/.ssh, .env, cloud credentials, etc. (LocalFileStore is a
|
||||
// sandboxed virtual dir and needs no such deny.)
|
||||
private static readonly DENY = { denySensitive: true } as const;
|
||||
|
||||
async readFile(relativePath: string): Promise<Buffer> {
|
||||
const fullPath = resolveSafe(this.root, relativePath, LinkedDirStore.DENY);
|
||||
return fs.readFileSync(fullPath);
|
||||
}
|
||||
|
||||
async listFiles(directory?: string): Promise<FileEntry[]> {
|
||||
const dir = directory ? resolveSafe(this.root, directory, LinkedDirStore.DENY) : this.root;
|
||||
if (!fs.existsSync(dir)) return [];
|
||||
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
return entries
|
||||
.filter(e => !e.name.startsWith('.') && e.name !== 'node_modules')
|
||||
// Don't disclose the existence/size/mtime of non-dot secrets (credentials.json,
|
||||
// id_rsa, known_hosts, …) — symmetric with searchFiles + the read deny.
|
||||
.filter(e => !isSensitiveFilePath(e.name))
|
||||
.map(entry => {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
try {
|
||||
const stat = fs.statSync(fullPath);
|
||||
return {
|
||||
name: entry.name,
|
||||
path: path.relative(this.root, fullPath).replace(/\\/g, '/'),
|
||||
size: stat.size,
|
||||
modified: stat.mtime.toISOString(),
|
||||
isDirectory: entry.isDirectory(),
|
||||
};
|
||||
} catch {
|
||||
return { name: entry.name, path: entry.name, size: 0, modified: '', isDirectory: false };
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async searchFiles(pattern: string): Promise<FileEntry[]> {
|
||||
const matches = await glob(pattern, {
|
||||
cwd: this.root,
|
||||
nodir: true,
|
||||
ignore: ['node_modules/**', '.git/**', 'dist/**', 'build/**', '**/.ssh/**', '**/.aws/**', '**/.gnupg/**'],
|
||||
});
|
||||
// Contain `../`-escaping globs to the root, then drop any secret a creative
|
||||
// pattern still matched — search never discloses the existence/path of secrets.
|
||||
return containGlobMatches(this.root, matches).filter(m => !isSensitiveFilePath(m)).slice(0, 200).map(match => {
|
||||
const fullPath = path.join(this.root, match);
|
||||
try {
|
||||
const stat = fs.statSync(fullPath);
|
||||
return {
|
||||
name: path.basename(match),
|
||||
path: match.replace(/\\/g, '/'),
|
||||
size: stat.size,
|
||||
modified: stat.mtime.toISOString(),
|
||||
isDirectory: false,
|
||||
};
|
||||
} catch {
|
||||
return { name: path.basename(match), path: match, size: 0, modified: '', isDirectory: false };
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async writeFile(relativePath: string, content: Buffer | string): Promise<void> {
|
||||
const fullPath = resolveSafe(this.root, relativePath, LinkedDirStore.DENY);
|
||||
ensureDir(path.dirname(fullPath));
|
||||
fs.writeFileSync(fullPath, content);
|
||||
}
|
||||
|
||||
async deleteFile(relativePath: string): Promise<void> {
|
||||
const fullPath = resolveSafe(this.root, relativePath, LinkedDirStore.DENY);
|
||||
if (fs.existsSync(fullPath)) {
|
||||
fs.unlinkSync(fullPath);
|
||||
}
|
||||
}
|
||||
|
||||
async moveFile(from: string, to: string): Promise<void> {
|
||||
const fromPath = resolveSafe(this.root, from, LinkedDirStore.DENY);
|
||||
const toPath = resolveSafe(this.root, to, LinkedDirStore.DENY);
|
||||
ensureDir(path.dirname(toPath));
|
||||
fs.renameSync(fromPath, toPath);
|
||||
}
|
||||
|
||||
async getStorageInfo(): Promise<StorageInfo> {
|
||||
if (!fs.existsSync(this.root)) {
|
||||
return { usedBytes: 0, fileCount: 0, storageType: 'linked' };
|
||||
}
|
||||
let usedBytes = 0;
|
||||
let fileCount = 0;
|
||||
const walk = (dir: string, depth = 0) => {
|
||||
if (depth > 5) return; // Don't walk too deep in linked dirs
|
||||
try {
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (entry.name === 'node_modules' || entry.name === '.git') continue;
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
walk(fullPath, depth + 1);
|
||||
} else {
|
||||
try {
|
||||
usedBytes += fs.statSync(fullPath).size;
|
||||
fileCount++;
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
}
|
||||
} catch { /* skip */ }
|
||||
};
|
||||
walk(this.root);
|
||||
return { usedBytes, fileCount, storageType: 'linked' };
|
||||
}
|
||||
}
|
||||
|
||||
// ── S3FileStore (MinIO/S3 for team workspace storage) ───────────────
|
||||
|
||||
export interface S3Config {
|
||||
endpoint: string;
|
||||
bucket: string;
|
||||
accessKey: string;
|
||||
secretKey: string;
|
||||
prefix: string; // e.g., "workspaces/{id}/"
|
||||
region?: string;
|
||||
}
|
||||
|
||||
export class S3FileStore implements FileStore {
|
||||
private config: S3Config;
|
||||
// Lazily constructed S3 client; the SDK is imported on first use to avoid
|
||||
// bundling it for local-only users.
|
||||
private client: S3Client | undefined;
|
||||
|
||||
constructor(config: S3Config) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
private async getClient(): Promise<S3Client> {
|
||||
if (this.client) return this.client;
|
||||
const { S3Client } = await import('@aws-sdk/client-s3');
|
||||
this.client = new S3Client({
|
||||
endpoint: `http://${this.config.endpoint}`,
|
||||
region: this.config.region ?? 'us-east-1',
|
||||
credentials: {
|
||||
accessKeyId: this.config.accessKey,
|
||||
secretAccessKey: this.config.secretKey,
|
||||
},
|
||||
forcePathStyle: true, // Required for MinIO
|
||||
});
|
||||
return this.client;
|
||||
}
|
||||
|
||||
getRootPath(): string { return `s3://${this.config.bucket}/${this.config.prefix}`; }
|
||||
getStorageType(): 'virtual' | 'linked' { return 'virtual'; }
|
||||
|
||||
async readFile(relativePath: string): Promise<Buffer> {
|
||||
assertSafeRelativeKey(relativePath);
|
||||
const client = await this.getClient();
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
const key = this.config.prefix + relativePath;
|
||||
const response = await client.send(new GetObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
Key: key,
|
||||
}));
|
||||
const body = response.Body;
|
||||
if (!body) throw new Error(`S3 object has no body: ${key}`);
|
||||
const chunks: Uint8Array[] = [];
|
||||
// In Node.js the S3 streaming body is an async-iterable readable stream.
|
||||
for await (const chunk of body as AsyncIterable<Uint8Array>) chunks.push(chunk);
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
async writeFile(relativePath: string, content: Buffer | string): Promise<void> {
|
||||
assertSafeRelativeKey(relativePath);
|
||||
const client = await this.getClient();
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
const key = this.config.prefix + relativePath;
|
||||
await client.send(new PutObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
Key: key,
|
||||
Body: typeof content === 'string' ? Buffer.from(content) : content,
|
||||
}));
|
||||
}
|
||||
|
||||
async deleteFile(relativePath: string): Promise<void> {
|
||||
assertSafeRelativeKey(relativePath);
|
||||
const client = await this.getClient();
|
||||
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
const key = this.config.prefix + relativePath;
|
||||
await client.send(new DeleteObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
Key: key,
|
||||
}));
|
||||
}
|
||||
|
||||
async listFiles(directory?: string): Promise<FileEntry[]> {
|
||||
if (directory) assertSafeRelativeKey(directory);
|
||||
const client = await this.getClient();
|
||||
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
|
||||
const pfx = this.config.prefix + (directory ? directory + '/' : '');
|
||||
const response = await client.send(new ListObjectsV2Command({
|
||||
Bucket: this.config.bucket,
|
||||
Prefix: pfx,
|
||||
Delimiter: '/',
|
||||
}));
|
||||
const entries: FileEntry[] = [];
|
||||
for (const obj of response.Contents ?? []) {
|
||||
if (!obj.Key) continue;
|
||||
const relPath = obj.Key.slice(this.config.prefix.length);
|
||||
entries.push({
|
||||
name: relPath.split('/').pop() ?? relPath,
|
||||
path: relPath,
|
||||
size: obj.Size ?? 0,
|
||||
modified: obj.LastModified?.toISOString() ?? '',
|
||||
isDirectory: false,
|
||||
});
|
||||
}
|
||||
for (const cpfx of response.CommonPrefixes ?? []) {
|
||||
if (!cpfx.Prefix) continue;
|
||||
const relPath = cpfx.Prefix.slice(this.config.prefix.length).replace(/\/$/, '');
|
||||
entries.push({
|
||||
name: relPath.split('/').pop() ?? relPath,
|
||||
path: relPath,
|
||||
size: 0,
|
||||
modified: '',
|
||||
isDirectory: true,
|
||||
});
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
async searchFiles(pattern: string): Promise<FileEntry[]> {
|
||||
const all = await this.listFiles();
|
||||
const regex = globToRegExp(pattern); // ReDoS-safe glob→regex (pattern is caller-controlled)
|
||||
return all.filter(f => regex.test(f.path) || regex.test(f.name));
|
||||
}
|
||||
|
||||
async moveFile(from: string, to: string): Promise<void> {
|
||||
assertSafeRelativeKey(from);
|
||||
assertSafeRelativeKey(to);
|
||||
const client = await this.getClient();
|
||||
const { CopyObjectCommand, DeleteObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
const fromKey = this.config.prefix + from;
|
||||
const toKey = this.config.prefix + to;
|
||||
await client.send(new CopyObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
CopySource: `${this.config.bucket}/${fromKey}`,
|
||||
Key: toKey,
|
||||
}));
|
||||
await client.send(new DeleteObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
Key: fromKey,
|
||||
}));
|
||||
}
|
||||
|
||||
async getStorageInfo(): Promise<StorageInfo> {
|
||||
const client = await this.getClient();
|
||||
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
|
||||
const response = await client.send(new ListObjectsV2Command({
|
||||
Bucket: this.config.bucket,
|
||||
Prefix: this.config.prefix,
|
||||
}));
|
||||
let usedBytes = 0;
|
||||
let fileCount = 0;
|
||||
for (const obj of response.Contents ?? []) {
|
||||
usedBytes += obj.Size ?? 0;
|
||||
fileCount++;
|
||||
}
|
||||
return { usedBytes, fileCount, storageType: 'virtual' };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Factory ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Create the appropriate FileStore for a workspace.
|
||||
* If s3Config is provided, use S3FileStore (team/cloud deployment).
|
||||
* If the workspace has a linked directory, use LinkedDirStore.
|
||||
* Otherwise, use LocalFileStore (virtual managed storage).
|
||||
*/
|
||||
export function createFileStore(
|
||||
dataDir: string,
|
||||
workspaceId: string,
|
||||
linkedDirectory?: string,
|
||||
s3Config?: S3Config,
|
||||
): FileStore {
|
||||
if (s3Config) return new S3FileStore(s3Config);
|
||||
if (linkedDirectory && fs.existsSync(linkedDirectory)) {
|
||||
return new LinkedDirStore(linkedDirectory);
|
||||
}
|
||||
return new LocalFileStore(dataDir, workspaceId);
|
||||
}
|
||||
159
packages/core/src/index.ts
Normal file
159
packages/core/src/index.ts
Normal file
@@ -0,0 +1,159 @@
|
||||
// @waggle/core — Waggle-specific orchestration layer.
|
||||
//
|
||||
// As of CC Sesija B 2026-04-30 (PM Q3 Plan A ratification), the substrate
|
||||
// (mind/ + harvest/ + logger + injection-scanner) lives in @waggle/hive-mind-core
|
||||
// and is distributed as Apache 2.0 OSS via subtree-split. This barrel re-exports
|
||||
// substrate symbols for backward compatibility — existing consumers (packages/agent,
|
||||
// apps/web, packages/server, etc.) keep their `import { ... } from '@waggle/core'`
|
||||
// imports unchanged.
|
||||
//
|
||||
// Waggle-specific (non-extracted) modules below the substrate re-exports stay
|
||||
// in this package: config, multi-mind, workspace orchestration, vault, telemetry,
|
||||
// install-audit, cron-store, skill-hashes, team-sync, file-store, file-indexer,
|
||||
// memory-import, optimization-log, compliance/.
|
||||
|
||||
// ── Substrate re-exports from @waggle/hive-mind-core (Apache 2.0 OSS) ──
|
||||
export {
|
||||
// Logger + injection scanner
|
||||
createCoreLogger, type CoreLogger,
|
||||
scanForInjection, type ScanResult,
|
||||
// mind/ — memory substrate
|
||||
MindDB, EmbeddingDimMismatchError,
|
||||
type EmbeddingFingerprint, type FingerprintCheck,
|
||||
IdentityLayer, type Identity,
|
||||
AwarenessLayer, type AwarenessItem, type AwarenessCategory,
|
||||
FrameStore, stripHmPrefix, hashFrameContent,
|
||||
RawArchive, hashRaw, readArchiveUids, withArchiveUid, type RawArchiveRow, type ArchiveInput,
|
||||
MindErasure, type EraseResult,
|
||||
SuppressionStore, type SuppressedSubject,
|
||||
type MemoryFrame, type FrameType, type Importance, type FrameSource,
|
||||
SessionStore, type Session,
|
||||
HybridSearch, type SearchResult,
|
||||
// D1 (oss-drift triage, 2026-06-11) — chunk-level retrieval (flag-gated, default OFF)
|
||||
chunkRetrievalEnabled, rechunkAllFrames, type RechunkResult,
|
||||
chunkText, type ChunkOptions, type FrameChunk,
|
||||
KnowledgeGraph, type Entity, type Relation, type ValidationSchema,
|
||||
SCHEMA_SQL, VEC_TABLE_SQL, CHUNKS_VEC_TABLE_SQL, SCHEMA_VERSION,
|
||||
vecTableSqlForDim, chunksVecTableSqlForDim,
|
||||
computeRelevance,
|
||||
computeTemporalScore,
|
||||
computePopularityScore,
|
||||
computeContextualScore,
|
||||
computeImportanceScore,
|
||||
SCORING_PROFILES,
|
||||
type ScoringProfile,
|
||||
type ScoringWeights,
|
||||
type Embedder,
|
||||
createLiteLLMEmbedder, type LiteLLMEmbedderConfig,
|
||||
createInProcessEmbedder, normalizeDimensions, type InProcessEmbedderConfig,
|
||||
createOllamaEmbedder, type OllamaEmbedderConfig,
|
||||
createApiEmbedder, type ApiEmbedderConfig,
|
||||
createEmbeddingProvider, EmbeddingQuotaExceededError, getMinimumTierForProvider,
|
||||
type EmbeddingProviderConfig, type EmbeddingProviderStatus, type EmbeddingProviderType,
|
||||
type EmbeddingProviderInstance, type EmbeddingQuotaStatus,
|
||||
normalizeEntityName, findDuplicates, isNoiseName, isLikelyAcronym,
|
||||
Ontology, validateEntity, type EntitySchema, type ValidationResult,
|
||||
ImprovementSignalStore,
|
||||
type ImprovementSignal, type ActionableSignal, type SignalCategory, type ActionableThresholds,
|
||||
ExecutionTraceStore, EXECUTION_TRACES_TABLE_SQL,
|
||||
type ExecutionTrace, type ParsedExecutionTrace, type TraceOutcome,
|
||||
type TracePayload, type TraceToolCall, type TraceReasoningStep,
|
||||
type StartTraceInput, type FinalizeTraceInput, type TraceQueryFilter,
|
||||
EvolutionRunStore, EVOLUTION_RUNS_TABLE_SQL,
|
||||
type EvolutionRun, type EvolutionRunStatus, type EvolutionRunTarget,
|
||||
type CreateEvolutionRunInput, type EvolutionRunFilter,
|
||||
reconcileIndexes, reconcileFtsIndex, reconcileVecIndex, cleanOrphanVectors, cleanOrphanFts,
|
||||
type ReconcileResult,
|
||||
ConceptTracker, CONCEPT_MASTERY_TABLE_SQL,
|
||||
type ConceptEntry, type ConceptUpdate,
|
||||
// recall-context — temporal rendering helpers (W4.1 production port)
|
||||
TEMPORAL_GUIDANCE, toDatePrefix, renderDatedSnippet, referenceDate, renderReferenceDateLine,
|
||||
parseDateWindow, type DateWindow,
|
||||
resolveRelativeDate, type ResolvedDate,
|
||||
createInProcessReranker, type Reranker, type InProcessRerankerConfig,
|
||||
// harvest/ — universal memory ingestion pipeline
|
||||
HarvestSourceStore,
|
||||
HarvestRunStore, type HarvestRun, type HarvestRunStatus,
|
||||
ChatGPTAdapter,
|
||||
ClaudeAdapter,
|
||||
ClaudeCodeAdapter,
|
||||
GeminiAdapter,
|
||||
UniversalAdapter,
|
||||
MarkdownAdapter,
|
||||
PlaintextAdapter,
|
||||
UrlAdapter,
|
||||
PdfAdapter,
|
||||
HarvestPipeline, type LLMCallFn, type PipelineOptions,
|
||||
extractMemoryLanes, writeMemoryLaneFrames,
|
||||
MIND_FACT_PREFIX, MIND_EVENT_PREFIX, MIND_PROFILE_PREFIX,
|
||||
type MemoryLaneExtraction, type ExtractedEvent, type ExtractedFact, type ExtractedProfile,
|
||||
type WriteLaneFramesResult,
|
||||
// D2 — LLM-based KG entity extraction (oss-drift triage, 2026-06-11)
|
||||
extractKgEntities, writeKgEntities, KG_ENTITY_TYPES,
|
||||
type KgEntity, type KgEntityType, type KgEntityExtraction, type WriteKgEntitiesResult,
|
||||
// W4.6 — raw-turn storage + RAWDETAIL recall lane
|
||||
writeRawTurnFrames, rawTurnHeader, parseRawTurnHeader, rawTurnConvKey,
|
||||
MIND_RAWTURN_PREFIX, MAX_TURNS_PER_ITEM, RAWDETAIL_KILL_SWITCH,
|
||||
type WriteRawTurnsResult, type ParsedRawTurnHeader,
|
||||
fetchRawDetailLane, rawTurnBody, RAW_DETAIL_K,
|
||||
type RawTurnHit, type RawDetailLaneOptions,
|
||||
dedup, harvestSetHash,
|
||||
HARVEST_FRAME_CONTENT_CAP,
|
||||
type ImportSourceType, type ImportItemType, type UniversalImportItem, type DistilledKnowledge,
|
||||
type HarvestPipelineResult, type HarvestSource, type SourceAdapter, type FilesystemAdapter,
|
||||
type ClassifiedItem, type ExtractedContent, type KnowledgeProvenance,
|
||||
// Plan A AMENDMENT 2026-04-30 — multi-workspace orchestration
|
||||
MultiMind, type MultiMindSearchResult, type MindSource, type SearchScope,
|
||||
MultiMindCache, type MultiMindCacheConfig,
|
||||
WorkspaceManager, type WorkspaceConfig, type CreateWorkspaceOptions,
|
||||
// Supersession (P) + bridge (B) frame producer (mono-parity 2026-07-05)
|
||||
detectSupersessionChains, detectEntityGroups, applyConsolidation,
|
||||
collectObservations, getCurrentValues,
|
||||
type ConsolidationLlm, type Observation, type SupersessionChain,
|
||||
type EntityGroup, type ConsolidationResult, type CollectObservationsOptions,
|
||||
} from '@waggle/hive-mind-core';
|
||||
|
||||
// ── Waggle-specific orchestration (stays in @waggle/core) ──
|
||||
export { WaggleConfig, type ProviderEntry, type TeamServerConfig } from './config.js';
|
||||
export { needsMigration, migrateToMultiMind } from './migration.js';
|
||||
export { TeamSync, frameToEntity, entityToSyncedFrame, type TeamSyncConfig, type SyncedFrame } from './team-sync.js';
|
||||
export {
|
||||
InstallAuditStore, INSTALL_AUDIT_TABLE_SQL,
|
||||
type InstallAuditEntry, type RecordAuditInput,
|
||||
type AuditAction, type AuditRiskLevel, type AuditTrustSource,
|
||||
type AuditApprovalClass, type AuditInitiator, type AuditCapabilityType,
|
||||
} from './install-audit.js';
|
||||
export {
|
||||
CronStore, CRON_SCHEDULES_TABLE_SQL, CRON_RUN_LEASES_TABLE_SQL, VALID_JOB_TYPES, cronExprError,
|
||||
type CronSchedule, type CreateScheduleInput, type CronJobType,
|
||||
type CronExecutionRow, type CronRunLeaseRow,
|
||||
type PendingActionRow, type PendingActionStatus, type SavePendingActionInput,
|
||||
} from './cron-store.js';
|
||||
export { VaultStore, type VaultEntry } from './vault.js';
|
||||
export { TelemetryStore, TelemetryCollector, TELEMETRY_EVENTS, type TelemetryEvent, type TelemetrySummary } from './telemetry.js';
|
||||
export {
|
||||
SkillHashStore, computeSkillHash, SKILL_HASHES_TABLE_SQL,
|
||||
type SkillHash,
|
||||
} from './skill-hashes.js';
|
||||
export { processImport, parseChatGPTExport, parseClaudeExport, extractKnowledge } from './memory-import.js';
|
||||
export { createFileStore, LocalFileStore, LinkedDirStore, S3FileStore, isSensitiveFilePath, type FileStore, type FileEntry, type StorageInfo, type S3Config } from './file-store.js';
|
||||
export { FileIndexer, MAX_CONTENT_BYTES, type FileIndexRow, type FileIndexResult } from './file-indexer.js';
|
||||
export type { ImportSource, ImportResult, ExtractedKnowledge, ParsedConversation, ConversationMessage } from './memory-import.js';
|
||||
export {
|
||||
OptimizationLogStore, OPTIMIZATION_LOG_TABLE_SQL,
|
||||
type OptimizationLogEntry, type CreateOptimizationLogInput,
|
||||
} from './optimization-log.js';
|
||||
|
||||
// ── Compliance (AI Act) — stays in @waggle/core (NOT extracted per .github/sync.md) ──
|
||||
export { InteractionStore } from './compliance/interaction-store.js';
|
||||
export { ComplianceStatusChecker } from './compliance/status-checker.js';
|
||||
export { ReportGenerator, type ReportGeneratorDeps } from './compliance/report-generator.js';
|
||||
export { ComplianceTemplateStore, KVARK_TEMPLATE_NAME } from './compliance/template-store.js';
|
||||
export { TEMPLATE_RISK_MAP } from './compliance/types.js';
|
||||
export type {
|
||||
AIActRiskLevel, HumanAction, AIInteraction, RecordInteractionInput,
|
||||
ComplianceStatus, ArticleStatus, AuditReport, AuditReportRequest,
|
||||
ModelInventoryEntry, OversightLogEntry, HarvestProvenanceEntry,
|
||||
ComplianceTemplate, ComplianceTemplateSections,
|
||||
CreateComplianceTemplateInput, UpdateComplianceTemplateInput,
|
||||
} from './compliance/types.js';
|
||||
221
packages/core/src/install-audit.ts
Normal file
221
packages/core/src/install-audit.ts
Normal file
@@ -0,0 +1,221 @@
|
||||
/**
|
||||
* Install Audit Store — persistent audit trail for capability install events.
|
||||
*
|
||||
* Records every install-relevant action (proposed, approved, installed, rejected,
|
||||
* failed) so there is a verifiable history of what was installed, when, why,
|
||||
* and by whom.
|
||||
*
|
||||
* Follows the same pattern as ImprovementSignalStore — operates on the .mind DB.
|
||||
*/
|
||||
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
import {
|
||||
sqlInList, RISK_LEVELS, APPROVAL_CLASSES, AUDIT_ACTIONS,
|
||||
AUDIT_CAPABILITY_TYPES, AUDIT_INITIATORS, TRUST_SOURCES,
|
||||
} from '@waggle/shared';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
// P7/D15 A2b: the audit vocabulary is now canonical in @waggle/shared. The
|
||||
// Audit*-prefixed names are kept as aliases (re-exported) so every downstream
|
||||
// `import { AuditAction, ... } from '@waggle/core'` keeps working unchanged.
|
||||
// All six sets were already byte-identical to the shared ones (incl. P5/D4's
|
||||
// 'uninstalled'), so this is a pure structural re-point — no value change.
|
||||
export type {
|
||||
AuditAction, AuditCapabilityType, AuditInitiator,
|
||||
RiskLevel as AuditRiskLevel, ApprovalClass as AuditApprovalClass, TrustSource as AuditTrustSource,
|
||||
} from '@waggle/shared';
|
||||
import type {
|
||||
AuditAction, AuditCapabilityType, AuditInitiator,
|
||||
RiskLevel as AuditRiskLevel, ApprovalClass as AuditApprovalClass, TrustSource as AuditTrustSource,
|
||||
} from '@waggle/shared';
|
||||
|
||||
export interface InstallAuditEntry {
|
||||
id: number;
|
||||
timestamp: string;
|
||||
capability_name: string;
|
||||
capability_type: AuditCapabilityType;
|
||||
source: string;
|
||||
version: string | null;
|
||||
risk_level: AuditRiskLevel;
|
||||
trust_source: AuditTrustSource;
|
||||
approval_class: AuditApprovalClass;
|
||||
action: AuditAction;
|
||||
initiator: AuditInitiator;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
export interface RecordAuditInput {
|
||||
capabilityName: string;
|
||||
capabilityType: AuditCapabilityType;
|
||||
source: string;
|
||||
version?: string | null;
|
||||
riskLevel: AuditRiskLevel;
|
||||
trustSource: AuditTrustSource;
|
||||
approvalClass: AuditApprovalClass;
|
||||
action: AuditAction;
|
||||
initiator: AuditInitiator;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
// ── Table DDL ──────────────────────────────────────────────────────────
|
||||
|
||||
// P7/D15 A3: the CHECK lists are generated from the canonical @waggle/shared
|
||||
// arrays via sqlInList, so the SQLite constraint and the TS union can no longer
|
||||
// drift (divergence #14 — the old comment admitted "drift silently crashes
|
||||
// record()"). The mirror DDL in hive-mind-core/src/mind/schema.ts stays a
|
||||
// standalone literal (it's the OSS substrate, §7.5) but is locked to these same
|
||||
// canonical lists by the parity test in install-audit-check-parity.test.ts.
|
||||
// P7/D15 #15: trust_source now also carries a CHECK (was unconstrained at the DB
|
||||
// while the TS type claimed a closed set). Every historical value came from the
|
||||
// typed AuditTrustSource (the pre-security-gate 6-set ⊂ the current 7-set), so
|
||||
// the rebuild migration's row copy can never violate it.
|
||||
export const INSTALL_AUDIT_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS install_audit (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
capability_name TEXT NOT NULL,
|
||||
capability_type TEXT NOT NULL CHECK (capability_type IN (${sqlInList(AUDIT_CAPABILITY_TYPES)})),
|
||||
source TEXT NOT NULL,
|
||||
version TEXT,
|
||||
risk_level TEXT NOT NULL CHECK (risk_level IN (${sqlInList(RISK_LEVELS)})),
|
||||
trust_source TEXT NOT NULL CHECK (trust_source IN (${sqlInList(TRUST_SOURCES)})),
|
||||
approval_class TEXT NOT NULL CHECK (approval_class IN (${sqlInList(APPROVAL_CLASSES)})),
|
||||
action TEXT NOT NULL CHECK (action IN (${sqlInList(AUDIT_ACTIONS)})),
|
||||
initiator TEXT NOT NULL CHECK (initiator IN (${sqlInList(AUDIT_INITIATORS)})),
|
||||
detail TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_capability ON install_audit (capability_name, action);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_timestamp ON install_audit (timestamp DESC);
|
||||
`;
|
||||
|
||||
// ── Store ──────────────────────────────────────────────────────────────
|
||||
|
||||
export class InstallAuditStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
const existing = raw.prepare(
|
||||
"SELECT sql FROM sqlite_master WHERE type='table' AND name='install_audit'",
|
||||
).get() as { sql: string } | undefined;
|
||||
if (!existing) {
|
||||
raw.exec(INSTALL_AUDIT_TABLE_SQL);
|
||||
return;
|
||||
}
|
||||
// P5/D4 migration: pre-'uninstalled' installs carry a narrower action CHECK
|
||||
// baked into the table DDL. SQLite can't ALTER a CHECK, so rebuild the table
|
||||
// when the stored DDL lacks the new value. Idempotent — a no-op once migrated.
|
||||
// #15: also rebuild when the stored DDL has no trust_source CHECK (the column
|
||||
// was previously unconstrained). "CHECK (trust_source IN" is a safe sentinel —
|
||||
// it appears nowhere else in this DDL.
|
||||
const needsActionWiden = !existing.sql.includes("'uninstalled'");
|
||||
const needsTrustSourceCheck = !existing.sql.includes('CHECK (trust_source IN');
|
||||
if (needsActionWiden || needsTrustSourceCheck) {
|
||||
this.rebuildForWidenedActionCheck(raw);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild install_audit with the widened `action` CHECK, preserving all rows.
|
||||
* Classic SQLite 12-step table redefinition, wrapped in a transaction so a
|
||||
* crash mid-rebuild leaves the original table intact.
|
||||
*/
|
||||
private rebuildForWidenedActionCheck(raw: ReturnType<MindDB['getDatabase']>): void {
|
||||
const migrate = raw.transaction(() => {
|
||||
raw.exec('ALTER TABLE install_audit RENAME TO install_audit_legacy');
|
||||
// SQLite carries indexes along with RENAME (still named idx_audit_*), so
|
||||
// INSTALL_AUDIT_TABLE_SQL's CREATE INDEX IF NOT EXISTS would no-op and the
|
||||
// DROP below would take the indexes with the legacy table. Drop them first
|
||||
// (mirrors hive-mind-core db.ts) so they get recreated on the new table.
|
||||
raw.exec('DROP INDEX IF EXISTS idx_audit_capability');
|
||||
raw.exec('DROP INDEX IF EXISTS idx_audit_timestamp');
|
||||
raw.exec(INSTALL_AUDIT_TABLE_SQL);
|
||||
raw.exec(`
|
||||
INSERT INTO install_audit (
|
||||
id, timestamp, capability_name, capability_type, source, version,
|
||||
risk_level, trust_source, approval_class, action, initiator, detail
|
||||
)
|
||||
SELECT id, timestamp, capability_name, capability_type, source, version,
|
||||
risk_level, trust_source, approval_class, action, initiator, detail
|
||||
FROM install_audit_legacy
|
||||
`);
|
||||
raw.exec('DROP TABLE install_audit_legacy');
|
||||
});
|
||||
migrate();
|
||||
}
|
||||
|
||||
/** Record an install audit event. */
|
||||
record(input: RecordAuditInput): InstallAuditEntry {
|
||||
const raw = this.db.getDatabase();
|
||||
|
||||
raw.prepare(`
|
||||
INSERT INTO install_audit (
|
||||
capability_name, capability_type, source, version,
|
||||
risk_level, trust_source, approval_class, action, initiator, detail
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
input.capabilityName,
|
||||
input.capabilityType,
|
||||
input.source,
|
||||
input.version ?? null,
|
||||
input.riskLevel,
|
||||
input.trustSource,
|
||||
input.approvalClass,
|
||||
input.action,
|
||||
input.initiator,
|
||||
input.detail ?? '',
|
||||
);
|
||||
|
||||
// Return the inserted row
|
||||
return raw.prepare(
|
||||
'SELECT * FROM install_audit ORDER BY id DESC LIMIT 1',
|
||||
).get() as InstallAuditEntry;
|
||||
}
|
||||
|
||||
/** Get audit history for a specific capability. */
|
||||
getByCapability(name: string): InstallAuditEntry[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM install_audit WHERE capability_name = ? ORDER BY id DESC',
|
||||
).all(name) as InstallAuditEntry[];
|
||||
}
|
||||
|
||||
/** Get audit history filtered by action type. */
|
||||
getByAction(action: AuditAction): InstallAuditEntry[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM install_audit WHERE action = ? ORDER BY id DESC',
|
||||
).all(action) as InstallAuditEntry[];
|
||||
}
|
||||
|
||||
/** Get recent audit entries (most recent first). */
|
||||
getRecent(limit: number = 20): InstallAuditEntry[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM install_audit ORDER BY id DESC LIMIT ?',
|
||||
).all(limit) as InstallAuditEntry[];
|
||||
}
|
||||
|
||||
/** Get recent entries for one capability type (most recent first) — backs
|
||||
* the shared Extend-layer audit read (GET /api/extend/audit?type=, C18). */
|
||||
getRecentByType(type: AuditCapabilityType, limit: number = 20): InstallAuditEntry[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM install_audit WHERE capability_type = ? ORDER BY id DESC LIMIT ?',
|
||||
).all(type, limit) as InstallAuditEntry[];
|
||||
}
|
||||
|
||||
/** Get all entries (for testing). */
|
||||
getAll(): InstallAuditEntry[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM install_audit ORDER BY id ASC',
|
||||
).all() as InstallAuditEntry[];
|
||||
}
|
||||
|
||||
/** Clear all entries (for testing). */
|
||||
clear(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM install_audit').run();
|
||||
}
|
||||
}
|
||||
292
packages/core/src/memory-import.ts
Normal file
292
packages/core/src/memory-import.ts
Normal file
@@ -0,0 +1,292 @@
|
||||
/**
|
||||
* Memory Import — parse ChatGPT and Claude exports, extract knowledge.
|
||||
*
|
||||
* Flow: raw JSON → parse conversations → extract knowledge items → ImportResult
|
||||
*/
|
||||
|
||||
export type ImportSource = 'chatgpt' | 'claude';
|
||||
|
||||
export interface ConversationMessage {
|
||||
role: 'user' | 'assistant';
|
||||
text: string;
|
||||
timestamp?: string;
|
||||
}
|
||||
|
||||
export interface ParsedConversation {
|
||||
title: string;
|
||||
messages: ConversationMessage[];
|
||||
createdAt?: string;
|
||||
source: ImportSource;
|
||||
}
|
||||
|
||||
export interface ExtractedKnowledge {
|
||||
type: 'decision' | 'fact' | 'preference' | 'topic';
|
||||
content: string;
|
||||
source: ImportSource;
|
||||
conversationTitle: string;
|
||||
importance: 'important' | 'normal';
|
||||
}
|
||||
|
||||
export interface ImportResult {
|
||||
source: ImportSource;
|
||||
conversationsFound: number;
|
||||
conversationsParsed: number;
|
||||
knowledgeExtracted: ExtractedKnowledge[];
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
// ── Raw export shapes (untrusted external JSON; all fields optional) ──
|
||||
|
||||
/** A ChatGPT export node carries an optional message in its `mapping` entry. */
|
||||
interface ChatGPTMessage {
|
||||
author?: { role?: string };
|
||||
content?: { parts?: unknown[] };
|
||||
create_time?: number;
|
||||
}
|
||||
|
||||
interface ChatGPTNode {
|
||||
message?: ChatGPTMessage;
|
||||
}
|
||||
|
||||
interface ChatGPTConversation {
|
||||
title?: string;
|
||||
mapping?: Record<string, ChatGPTNode>;
|
||||
create_time?: number;
|
||||
}
|
||||
|
||||
interface ClaudeMessage {
|
||||
sender?: string;
|
||||
role?: string;
|
||||
text?: string;
|
||||
content?: string;
|
||||
created_at?: string;
|
||||
timestamp?: string;
|
||||
}
|
||||
|
||||
interface ClaudeConversation {
|
||||
name?: string;
|
||||
title?: string;
|
||||
chat_messages?: ClaudeMessage[];
|
||||
messages?: ClaudeMessage[];
|
||||
created_at?: string;
|
||||
create_time?: string;
|
||||
}
|
||||
|
||||
/** Pull the `conversations` array out of an export that may be the array itself. */
|
||||
function extractConversations(json: unknown): unknown[] {
|
||||
if (Array.isArray(json)) return json;
|
||||
if (json && typeof json === 'object' && 'conversations' in json) {
|
||||
const c = (json as { conversations?: unknown }).conversations;
|
||||
if (Array.isArray(c)) return c;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
// ── Parsers ───────────────────────────────────────
|
||||
|
||||
export function parseChatGPTExport(json: unknown): ParsedConversation[] {
|
||||
const conversations = extractConversations(json);
|
||||
|
||||
return conversations.map((raw) => {
|
||||
const conv = raw as ChatGPTConversation;
|
||||
const title = conv.title || 'Untitled';
|
||||
const messages: ConversationMessage[] = [];
|
||||
|
||||
// ChatGPT uses a mapping object with node IDs
|
||||
if (conv.mapping && typeof conv.mapping === 'object') {
|
||||
const nodes = Object.values(conv.mapping);
|
||||
// Sort by create_time for chronological order
|
||||
const sorted = nodes
|
||||
.filter((n) => (n?.message?.content?.parts?.length ?? 0) > 0)
|
||||
.sort((a, b) => (a.message?.create_time ?? 0) - (b.message?.create_time ?? 0));
|
||||
|
||||
for (const node of sorted) {
|
||||
const msg = node.message;
|
||||
if (!msg || !msg.author?.role) continue;
|
||||
const role = msg.author.role === 'user' ? 'user' : 'assistant';
|
||||
if (msg.author.role === 'system') continue; // Skip system messages
|
||||
|
||||
const text = (msg.content?.parts ?? [])
|
||||
.filter((p): p is string => typeof p === 'string')
|
||||
.join('\n')
|
||||
.trim();
|
||||
if (!text) continue;
|
||||
|
||||
messages.push({
|
||||
role,
|
||||
text,
|
||||
timestamp: msg.create_time ? new Date(msg.create_time * 1000).toISOString() : undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
title,
|
||||
messages,
|
||||
createdAt: conv.create_time ? new Date(conv.create_time * 1000).toISOString() : undefined,
|
||||
source: 'chatgpt' as ImportSource,
|
||||
};
|
||||
}).filter(c => c.messages.length > 0);
|
||||
}
|
||||
|
||||
export function parseClaudeExport(json: unknown): ParsedConversation[] {
|
||||
const conversations = extractConversations(json);
|
||||
|
||||
return conversations.map((raw) => {
|
||||
const conv = raw as ClaudeConversation;
|
||||
const title = conv.name || conv.title || 'Untitled';
|
||||
const messages: ConversationMessage[] = [];
|
||||
|
||||
const chatMessages = conv.chat_messages ?? conv.messages ?? [];
|
||||
for (const msg of chatMessages) {
|
||||
const role = (msg.sender === 'human' || msg.role === 'user') ? 'user' : 'assistant';
|
||||
const text = (msg.text ?? msg.content ?? '').trim();
|
||||
if (!text) continue;
|
||||
|
||||
messages.push({
|
||||
role,
|
||||
text,
|
||||
timestamp: msg.created_at ?? msg.timestamp,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
title,
|
||||
messages,
|
||||
createdAt: conv.created_at ?? conv.create_time,
|
||||
source: 'claude' as ImportSource,
|
||||
};
|
||||
}).filter(c => c.messages.length > 0);
|
||||
}
|
||||
|
||||
// ── Knowledge Extraction ──────────────────────────
|
||||
|
||||
const DECISION_PATTERNS = [
|
||||
/\b(?:decided|chose|going with|we'll use|confirmed|agreed|settled on|picked|selected)\b/i,
|
||||
/\b(?:decision|choice):\s/i,
|
||||
/\blet's go with\b/i,
|
||||
];
|
||||
|
||||
const PREFERENCE_PATTERNS = [
|
||||
/\b(?:I prefer|I like|I always|I never|I don't like|don't use|use .+ instead)\b/i,
|
||||
/\b(?:my preference|I'd rather|please always|please never|from now on)\b/i,
|
||||
/\b(?:my style is|I want you to|format it as)\b/i,
|
||||
];
|
||||
|
||||
const FACT_PATTERNS = [
|
||||
/\b(?:my name is|I work at|I'm a|I am a|my company|my team|my role|our product|our stack)\b/i,
|
||||
/\b(?:we use|our tech|we're building|the project is|the codebase)\b/i,
|
||||
];
|
||||
|
||||
export function extractKnowledge(conversations: ParsedConversation[]): ExtractedKnowledge[] {
|
||||
const knowledge: ExtractedKnowledge[] = [];
|
||||
const seen = new Set<string>(); // Deduplicate
|
||||
|
||||
for (const conv of conversations) {
|
||||
// Extract conversation topic
|
||||
if (conv.title && conv.title !== 'Untitled' && conv.title.length > 5) {
|
||||
const topicKey = `topic:${conv.title.toLowerCase().trim()}`;
|
||||
if (!seen.has(topicKey)) {
|
||||
seen.add(topicKey);
|
||||
knowledge.push({
|
||||
type: 'topic',
|
||||
content: `Conversation topic: ${conv.title}`,
|
||||
source: conv.source,
|
||||
conversationTitle: conv.title,
|
||||
importance: 'normal',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Scan user messages for decisions, preferences, facts
|
||||
for (const msg of conv.messages) {
|
||||
if (msg.role !== 'user') continue;
|
||||
if (msg.text.length < 10 || msg.text.length > 500) continue; // Skip too short/long
|
||||
|
||||
const sentences = msg.text.split(/[.!?\n]+/).map(s => s.trim()).filter(s => s.length > 10);
|
||||
|
||||
for (const sentence of sentences) {
|
||||
const contentKey = sentence.toLowerCase().slice(0, 80);
|
||||
if (seen.has(contentKey)) continue;
|
||||
|
||||
// Check for decisions
|
||||
if (DECISION_PATTERNS.some(p => p.test(sentence))) {
|
||||
seen.add(contentKey);
|
||||
knowledge.push({
|
||||
type: 'decision',
|
||||
content: `Decision: ${sentence}`,
|
||||
source: conv.source,
|
||||
conversationTitle: conv.title,
|
||||
importance: 'important',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check for preferences
|
||||
if (PREFERENCE_PATTERNS.some(p => p.test(sentence))) {
|
||||
seen.add(contentKey);
|
||||
knowledge.push({
|
||||
type: 'preference',
|
||||
content: `Preference: ${sentence}`,
|
||||
source: conv.source,
|
||||
conversationTitle: conv.title,
|
||||
importance: 'important',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check for facts
|
||||
if (FACT_PATTERNS.some(p => p.test(sentence))) {
|
||||
seen.add(contentKey);
|
||||
knowledge.push({
|
||||
type: 'fact',
|
||||
content: `${sentence}`,
|
||||
source: conv.source,
|
||||
conversationTitle: conv.title,
|
||||
importance: 'normal',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cap at 100 items to avoid polluting memory
|
||||
return knowledge.slice(0, 100);
|
||||
}
|
||||
|
||||
// ── Main Import Function ──────────────────────────
|
||||
|
||||
export function processImport(jsonData: unknown, source: ImportSource): ImportResult {
|
||||
const errors: string[] = [];
|
||||
|
||||
let conversations: ParsedConversation[];
|
||||
try {
|
||||
conversations = source === 'chatgpt'
|
||||
? parseChatGPTExport(jsonData)
|
||||
: parseClaudeExport(jsonData);
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
return {
|
||||
source,
|
||||
conversationsFound: 0,
|
||||
conversationsParsed: 0,
|
||||
knowledgeExtracted: [],
|
||||
errors: [`Parse error: ${message}`],
|
||||
};
|
||||
}
|
||||
|
||||
if (conversations.length === 0) {
|
||||
errors.push('No conversations found in export');
|
||||
}
|
||||
|
||||
const knowledge = extractKnowledge(conversations);
|
||||
|
||||
return {
|
||||
source,
|
||||
conversationsFound: conversations.length,
|
||||
conversationsParsed: conversations.filter(c => c.messages.length > 0).length,
|
||||
knowledgeExtracted: knowledge,
|
||||
errors,
|
||||
};
|
||||
}
|
||||
45
packages/core/src/migration.ts
Normal file
45
packages/core/src/migration.ts
Normal file
@@ -0,0 +1,45 @@
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
/**
|
||||
* Check if a migration from default.mind to personal.mind is needed.
|
||||
* Returns true if default.mind exists AND personal.mind does NOT exist.
|
||||
*/
|
||||
export function needsMigration(waggleDir: string): boolean {
|
||||
const defaultMind = path.join(waggleDir, 'default.mind');
|
||||
const personalMind = path.join(waggleDir, 'personal.mind');
|
||||
return fs.existsSync(defaultMind) && !fs.existsSync(personalMind);
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrate from the old single-mind layout (default.mind) to the
|
||||
* M4 multi-mind layout (personal.mind + workspaces/).
|
||||
*
|
||||
* Steps when migration is needed:
|
||||
* 1. Copy default.mind -> personal.mind
|
||||
* 2. Rename default.mind -> default.mind.bak
|
||||
* 3. Create workspaces/ directory
|
||||
*
|
||||
* This is a pure file operation -- no data transformation is performed.
|
||||
*/
|
||||
export function migrateToMultiMind(waggleDir: string): { migrated: boolean; message: string } {
|
||||
if (!needsMigration(waggleDir)) {
|
||||
return { migrated: false, message: 'No migration needed' };
|
||||
}
|
||||
|
||||
const defaultMind = path.join(waggleDir, 'default.mind');
|
||||
const personalMind = path.join(waggleDir, 'personal.mind');
|
||||
const backupMind = path.join(waggleDir, 'default.mind.bak');
|
||||
const workspacesDir = path.join(waggleDir, 'workspaces');
|
||||
|
||||
// 1. Copy default.mind -> personal.mind
|
||||
fs.copyFileSync(defaultMind, personalMind);
|
||||
|
||||
// 2. Rename default.mind -> default.mind.bak
|
||||
fs.renameSync(defaultMind, backupMind);
|
||||
|
||||
// 3. Create workspaces/ directory (idempotent)
|
||||
fs.mkdirSync(workspacesDir, { recursive: true });
|
||||
|
||||
return { migrated: true, message: 'Migrated default.mind to personal.mind' };
|
||||
}
|
||||
154
packages/core/src/optimization-log.ts
Normal file
154
packages/core/src/optimization-log.ts
Normal file
@@ -0,0 +1,154 @@
|
||||
/**
|
||||
* Optimization Log Store — SQLite persistence for GEPA prompt optimization data.
|
||||
*
|
||||
* Captures agent interaction metadata (system prompts, tools used, turn counts,
|
||||
* correction signals) for background analysis by the PromptOptimizer.
|
||||
*
|
||||
* Follows the same lazy-table pattern as CronStore and InstallAuditStore —
|
||||
* operates on the .mind DB with auto-table creation.
|
||||
*/
|
||||
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
export interface OptimizationLogEntry {
|
||||
id: number;
|
||||
session_id: string;
|
||||
workspace_id: string;
|
||||
system_prompt: string;
|
||||
tools_used: string; // JSON array
|
||||
turn_count: number;
|
||||
was_correction: number; // SQLite integer boolean
|
||||
input_tokens: number;
|
||||
output_tokens: number;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
export interface CreateOptimizationLogInput {
|
||||
sessionId: string;
|
||||
workspaceId: string;
|
||||
systemPrompt: string;
|
||||
toolsUsed: string[];
|
||||
turnCount: number;
|
||||
wasCorrection: boolean;
|
||||
inputTokens?: number;
|
||||
outputTokens?: number;
|
||||
}
|
||||
|
||||
// ── Table DDL ──────────────────────────────────────────────────────────
|
||||
|
||||
export const OPTIMIZATION_LOG_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS optimization_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT NOT NULL,
|
||||
workspace_id TEXT NOT NULL,
|
||||
system_prompt TEXT NOT NULL,
|
||||
tools_used TEXT NOT NULL DEFAULT '[]',
|
||||
turn_count INTEGER NOT NULL DEFAULT 0,
|
||||
was_correction INTEGER NOT NULL DEFAULT 0,
|
||||
input_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
output_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
timestamp TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_optlog_workspace ON optimization_log (workspace_id, timestamp DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_optlog_timestamp ON optimization_log (timestamp DESC);
|
||||
`;
|
||||
|
||||
// ── Store ──────────────────────────────────────────────────────────────
|
||||
|
||||
export class OptimizationLogStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
const exists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='optimization_log'",
|
||||
).get();
|
||||
if (!exists) {
|
||||
raw.exec(OPTIMIZATION_LOG_TABLE_SQL);
|
||||
}
|
||||
}
|
||||
|
||||
/** Insert a new optimization log entry capturing an agent interaction. */
|
||||
insert(input: CreateOptimizationLogInput): OptimizationLogEntry {
|
||||
const raw = this.db.getDatabase();
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO optimization_log (session_id, workspace_id, system_prompt, tools_used, turn_count, was_correction, input_tokens, output_tokens)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
input.sessionId,
|
||||
input.workspaceId,
|
||||
input.systemPrompt,
|
||||
JSON.stringify(input.toolsUsed),
|
||||
input.turnCount,
|
||||
input.wasCorrection ? 1 : 0,
|
||||
input.inputTokens ?? 0,
|
||||
input.outputTokens ?? 0,
|
||||
);
|
||||
|
||||
return raw.prepare(
|
||||
'SELECT * FROM optimization_log WHERE id = ?',
|
||||
).get(result.lastInsertRowid) as OptimizationLogEntry;
|
||||
}
|
||||
|
||||
/** Get recent optimization log entries, ordered by most recent first. */
|
||||
getRecent(limit: number = 50): OptimizationLogEntry[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM optimization_log ORDER BY timestamp DESC, id DESC LIMIT ?',
|
||||
).all(limit) as OptimizationLogEntry[];
|
||||
}
|
||||
|
||||
/** Get log entries for a specific workspace. */
|
||||
getByWorkspace(workspaceId: string, limit: number = 50): OptimizationLogEntry[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM optimization_log WHERE workspace_id = ? ORDER BY timestamp DESC, id DESC LIMIT ?',
|
||||
).all(workspaceId, limit) as OptimizationLogEntry[];
|
||||
}
|
||||
|
||||
/** Get aggregate stats: total entries, correction rate, avg turn count, total tokens. */
|
||||
getStats(): { total: number; correctionRate: number; avgTurnCount: number; totalInputTokens: number; totalOutputTokens: number } {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare(`
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
COALESCE(AVG(was_correction * 1.0), 0) as correction_rate,
|
||||
COALESCE(AVG(turn_count), 0) as avg_turn_count,
|
||||
COALESCE(SUM(input_tokens), 0) as total_input_tokens,
|
||||
COALESCE(SUM(output_tokens), 0) as total_output_tokens
|
||||
FROM optimization_log
|
||||
`).get() as {
|
||||
total: number;
|
||||
correction_rate: number;
|
||||
avg_turn_count: number;
|
||||
total_input_tokens: number;
|
||||
total_output_tokens: number;
|
||||
};
|
||||
|
||||
return {
|
||||
total: row.total,
|
||||
correctionRate: row.correction_rate,
|
||||
avgTurnCount: row.avg_turn_count,
|
||||
totalInputTokens: row.total_input_tokens,
|
||||
totalOutputTokens: row.total_output_tokens,
|
||||
};
|
||||
}
|
||||
|
||||
/** Delete entries older than the given number of days. */
|
||||
pruneOlderThan(days: number): number {
|
||||
const result = this.db.getDatabase().prepare(
|
||||
`DELETE FROM optimization_log WHERE timestamp < datetime('now', '-' || ? || ' days')`,
|
||||
).run(days);
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
/** Clear all entries (for testing). */
|
||||
clear(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM optimization_log').run();
|
||||
}
|
||||
}
|
||||
122
packages/core/src/skill-hashes.ts
Normal file
122
packages/core/src/skill-hashes.ts
Normal file
@@ -0,0 +1,122 @@
|
||||
/**
|
||||
* Skill Hash Store — SHA-256 content hashing for skill update detection.
|
||||
*
|
||||
* Stores hashes of skill file content in the .mind DB. On server startup,
|
||||
* compares current file hashes to stored hashes to detect changes.
|
||||
* Follows the same ensureTable pattern as CronStore / InstallAuditStore.
|
||||
*/
|
||||
|
||||
import * as crypto from 'node:crypto';
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
export interface SkillHash {
|
||||
name: string;
|
||||
hash: string;
|
||||
verified_at: string;
|
||||
}
|
||||
|
||||
// ── Table DDL ──────────────────────────────────────────────────────────
|
||||
|
||||
export const SKILL_HASHES_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS skill_hashes (
|
||||
name TEXT PRIMARY KEY,
|
||||
hash TEXT NOT NULL,
|
||||
verified_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
`;
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
/** Compute SHA-256 hex digest of skill content. */
|
||||
export function computeSkillHash(content: string): string {
|
||||
return crypto.createHash('sha256').update(content).digest('hex');
|
||||
}
|
||||
|
||||
// ── Store ──────────────────────────────────────────────────────────────
|
||||
|
||||
export class SkillHashStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
const exists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='skill_hashes'",
|
||||
).get();
|
||||
if (!exists) {
|
||||
raw.exec(SKILL_HASHES_TABLE_SQL);
|
||||
}
|
||||
}
|
||||
|
||||
/** Store or update hash for a skill. */
|
||||
setHash(name: string, hash: string): void {
|
||||
this.db.getDatabase().prepare(`
|
||||
INSERT INTO skill_hashes (name, hash) VALUES (?, ?)
|
||||
ON CONFLICT (name) DO UPDATE SET hash = ?, verified_at = datetime('now')
|
||||
`).run(name, hash, hash);
|
||||
}
|
||||
|
||||
/** Get stored hash for a skill. */
|
||||
getHash(name: string): SkillHash | undefined {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM skill_hashes WHERE name = ?',
|
||||
).get(name) as SkillHash | undefined;
|
||||
}
|
||||
|
||||
/** Remove hash (when skill is deleted). */
|
||||
removeHash(name: string): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM skill_hashes WHERE name = ?').run(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check all skills against stored hashes.
|
||||
* Returns lists of changed, added, and removed skill names.
|
||||
*/
|
||||
checkAll(currentSkills: Array<{ name: string; content: string }>): {
|
||||
changed: string[];
|
||||
added: string[];
|
||||
removed: string[];
|
||||
} {
|
||||
const storedHashes = this.db.getDatabase().prepare(
|
||||
'SELECT name, hash FROM skill_hashes',
|
||||
).all() as SkillHash[];
|
||||
const storedMap = new Map(storedHashes.map(h => [h.name, h.hash]));
|
||||
const currentNames = new Set(currentSkills.map(s => s.name));
|
||||
|
||||
const changed: string[] = [];
|
||||
const added: string[] = [];
|
||||
|
||||
for (const skill of currentSkills) {
|
||||
const currentHash = computeSkillHash(skill.content);
|
||||
const storedHash = storedMap.get(skill.name);
|
||||
|
||||
if (!storedHash) {
|
||||
added.push(skill.name);
|
||||
} else if (storedHash !== currentHash) {
|
||||
changed.push(skill.name);
|
||||
}
|
||||
}
|
||||
|
||||
const removed = storedHashes
|
||||
.filter(h => !currentNames.has(h.name))
|
||||
.map(h => h.name);
|
||||
|
||||
return { changed, added, removed };
|
||||
}
|
||||
|
||||
/** Mark a skill as verified (update hash to current content). */
|
||||
verify(name: string, content: string): void {
|
||||
this.setHash(name, computeSkillHash(content));
|
||||
}
|
||||
|
||||
/** Clear all hashes (for testing). */
|
||||
clear(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM skill_hashes').run();
|
||||
}
|
||||
}
|
||||
186
packages/core/src/team-sync.ts
Normal file
186
packages/core/src/team-sync.ts
Normal file
@@ -0,0 +1,186 @@
|
||||
/**
|
||||
* TeamSync — Syncs memory frames between local .mind and team server.
|
||||
*
|
||||
* Uses the team server's entities API (entityType='memory_frame') for storage.
|
||||
* Push-on-write: new frames are pushed to team server after local write.
|
||||
* Pull-on-activate: frames are pulled from team server when workspace is activated.
|
||||
* Attribution: each frame carries the author's userId and displayName.
|
||||
*
|
||||
* Sync protocol:
|
||||
* - Frames are identified by a composite key: gopId + t (group-of-pictures ID + temporal index)
|
||||
* - Pull uses ?type=memory_frame&since=<timestamp> to get only new frames
|
||||
* - Push sends each frame as a teamEntity with entityType='memory_frame'
|
||||
* - Last-write-wins at frame level (frames are append-mostly, conflicts are rare)
|
||||
*/
|
||||
|
||||
import type { MemoryFrame, FrameType, Importance } from '@waggle/hive-mind-core';
|
||||
import { createCoreLogger } from '@waggle/hive-mind-core';
|
||||
const log = createCoreLogger('team-sync');
|
||||
|
||||
export interface TeamSyncConfig {
|
||||
teamServerUrl: string;
|
||||
teamSlug: string;
|
||||
authToken: string;
|
||||
userId: string;
|
||||
displayName: string;
|
||||
}
|
||||
|
||||
export interface SyncedFrame {
|
||||
/** Server-side entity ID (UUID). */
|
||||
remoteId: string;
|
||||
gopId: string;
|
||||
t: number;
|
||||
frameType: FrameType;
|
||||
content: string;
|
||||
importance: Importance;
|
||||
authorId: string;
|
||||
authorName: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a local MemoryFrame to the team server entity format.
|
||||
*/
|
||||
export function frameToEntity(frame: MemoryFrame, authorId: string, authorName: string) {
|
||||
return {
|
||||
entityType: 'memory_frame',
|
||||
name: frame.gop_id,
|
||||
properties: {
|
||||
frameType: frame.frame_type,
|
||||
t: frame.t,
|
||||
baseFrameId: frame.base_frame_id,
|
||||
content: frame.content,
|
||||
importance: frame.importance,
|
||||
authorId,
|
||||
authorName,
|
||||
localId: frame.id,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a team server entity back to a SyncedFrame.
|
||||
*/
|
||||
export function entityToSyncedFrame(entity: {
|
||||
id: string;
|
||||
name: string;
|
||||
properties: Record<string, unknown>;
|
||||
createdAt: string;
|
||||
}): SyncedFrame {
|
||||
const props = entity.properties;
|
||||
return {
|
||||
remoteId: entity.id,
|
||||
gopId: entity.name,
|
||||
t: (props.t as number) ?? 0,
|
||||
frameType: (props.frameType as FrameType) ?? 'I',
|
||||
content: (props.content as string) ?? '',
|
||||
importance: (props.importance as Importance) ?? 'normal',
|
||||
authorId: (props.authorId as string) ?? '',
|
||||
authorName: (props.authorName as string) ?? '',
|
||||
createdAt: entity.createdAt,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* TeamSync handles push/pull of memory frames to/from the team server.
|
||||
* This is a stateless client — call pushFrame() after writing locally,
|
||||
* call pullFrames() on workspace activation.
|
||||
*/
|
||||
export class TeamSync {
|
||||
private config: TeamSyncConfig;
|
||||
private lastSyncTimestamp: string | null = null;
|
||||
|
||||
constructor(config: TeamSyncConfig) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Push a single frame to the team server.
|
||||
* Call this after writing a frame to the local workspace .mind.
|
||||
*/
|
||||
async pushFrame(frame: MemoryFrame): Promise<{ remoteId: string } | null> {
|
||||
const entity = frameToEntity(frame, this.config.userId, this.config.displayName);
|
||||
|
||||
try {
|
||||
const response = await fetch(
|
||||
`${this.config.teamServerUrl}/api/teams/${this.config.teamSlug}/entities`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${this.config.authToken}`,
|
||||
},
|
||||
body: JSON.stringify(entity),
|
||||
}
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
log.error(`Push failed: ${response.status} ${response.statusText}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const created = await response.json() as { id: string };
|
||||
return { remoteId: created.id };
|
||||
} catch (err) {
|
||||
log.error('Push error:', err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull all memory frames from the team server.
|
||||
* Returns frames sorted by creation time (newest first).
|
||||
* Optionally pass `since` to get only frames created after that timestamp.
|
||||
*/
|
||||
async pullFrames(since?: string): Promise<SyncedFrame[]> {
|
||||
try {
|
||||
const url = `${this.config.teamServerUrl}/api/teams/${this.config.teamSlug}/entities?type=memory_frame`;
|
||||
// Note: `since` filtering would require server-side support. For now, pull all and filter client-side.
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
'Authorization': `Bearer ${this.config.authToken}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
log.error(`Pull failed: ${response.status} ${response.statusText}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const entities = await response.json() as Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
properties: Record<string, unknown>;
|
||||
createdAt: string;
|
||||
}>;
|
||||
|
||||
const frames = entities.map(entityToSyncedFrame);
|
||||
|
||||
// Client-side filter by timestamp if requested
|
||||
if (since) {
|
||||
return frames.filter(f => f.createdAt > since);
|
||||
}
|
||||
|
||||
this.lastSyncTimestamp = new Date().toISOString();
|
||||
return frames;
|
||||
} catch (err) {
|
||||
log.error('Pull error:', err);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the last sync timestamp (for incremental sync).
|
||||
*/
|
||||
getLastSyncTimestamp(): string | null {
|
||||
return this.lastSyncTimestamp;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the last sync timestamp (e.g. loaded from persistent storage).
|
||||
*/
|
||||
setLastSyncTimestamp(ts: string): void {
|
||||
this.lastSyncTimestamp = ts;
|
||||
}
|
||||
}
|
||||
324
packages/core/src/telemetry.ts
Normal file
324
packages/core/src/telemetry.ts
Normal file
@@ -0,0 +1,324 @@
|
||||
/**
|
||||
* TelemetryStore — local, privacy-first event tracking.
|
||||
*
|
||||
* All data stays in ~/.waggle/telemetry.db (SQLite).
|
||||
* Default: OFF. User opts in via Settings toggle.
|
||||
* No cloud reporting in M2 — data is queryable via local API only.
|
||||
*
|
||||
* NEVER tracked: message content, memory content, file paths,
|
||||
* API keys, personal info, IP addresses, device IDs.
|
||||
*/
|
||||
|
||||
import Database from 'better-sqlite3';
|
||||
import type { Database as DatabaseType } from 'better-sqlite3';
|
||||
import path from 'node:path';
|
||||
import fs from 'node:fs';
|
||||
|
||||
/* ── Schema ── */
|
||||
const TELEMETRY_SCHEMA = `
|
||||
CREATE TABLE IF NOT EXISTS telemetry_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
event TEXT NOT NULL,
|
||||
properties TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_telemetry_event ON telemetry_events (event, created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_telemetry_date ON telemetry_events (created_at);
|
||||
`;
|
||||
|
||||
/* ── Event constants ── */
|
||||
export const TELEMETRY_EVENTS = {
|
||||
// Onboarding funnel
|
||||
ONBOARDING_START: 'onboarding_start',
|
||||
ONBOARDING_STEP: 'onboarding_step',
|
||||
ONBOARDING_COMPLETE: 'onboarding_complete',
|
||||
ONBOARDING_SKIP: 'onboarding_skip',
|
||||
|
||||
// Session engagement
|
||||
SESSION_START: 'session_start',
|
||||
SESSION_END: 'session_end',
|
||||
|
||||
// Infrastructure
|
||||
EMBEDDING_PROVIDER: 'embedding_provider',
|
||||
LLM_PROVIDER: 'llm_provider',
|
||||
|
||||
// Feature usage
|
||||
TEMPLATE_SELECTED: 'template_selected',
|
||||
WORKSPACE_CREATED: 'workspace_created',
|
||||
FIRST_AGENT_RESPONSE: 'first_agent_response',
|
||||
SLASH_COMMAND_USED: 'slash_command_used',
|
||||
|
||||
// App lifecycle
|
||||
APP_START: 'app_start',
|
||||
APP_ERROR: 'app_error',
|
||||
} as const;
|
||||
|
||||
/* ── Types ── */
|
||||
export interface TelemetryEvent {
|
||||
id: number;
|
||||
event: string;
|
||||
properties: Record<string, unknown>;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export interface TelemetrySummary {
|
||||
enabled: boolean;
|
||||
totalEvents: number;
|
||||
firstEvent: string | null;
|
||||
lastEvent: string | null;
|
||||
onboardingCompleted: boolean;
|
||||
totalSessions: number;
|
||||
embeddingProvider: string | null;
|
||||
templatesUsed: string[];
|
||||
eventBreakdown: Record<string, number>;
|
||||
}
|
||||
|
||||
/* ── Store ── */
|
||||
export class TelemetryStore {
|
||||
private db: DatabaseType;
|
||||
private enabled: boolean;
|
||||
|
||||
constructor(dataDir: string, enabled = false) {
|
||||
this.enabled = enabled;
|
||||
const dbPath = path.join(dataDir, 'telemetry.db');
|
||||
this.db = new Database(dbPath);
|
||||
this.db.pragma('journal_mode = WAL');
|
||||
this.db.exec(TELEMETRY_SCHEMA);
|
||||
}
|
||||
|
||||
/** Track an event. No-op if telemetry is disabled. */
|
||||
track(event: string, properties?: Record<string, unknown>): void {
|
||||
if (!this.enabled) return;
|
||||
this.db.prepare(
|
||||
'INSERT INTO telemetry_events (event, properties) VALUES (?, ?)'
|
||||
).run(event, JSON.stringify(properties ?? {}));
|
||||
}
|
||||
|
||||
/** Enable/disable telemetry at runtime. */
|
||||
setEnabled(enabled: boolean): void {
|
||||
this.enabled = enabled;
|
||||
}
|
||||
|
||||
/** Whether telemetry is currently enabled. */
|
||||
isEnabled(): boolean {
|
||||
return this.enabled;
|
||||
}
|
||||
|
||||
/** Get aggregated summary. */
|
||||
getSummary(): TelemetrySummary {
|
||||
const total = (this.db.prepare(
|
||||
'SELECT COUNT(*) as cnt FROM telemetry_events'
|
||||
).get() as { cnt: number }).cnt;
|
||||
|
||||
const first = this.db.prepare(
|
||||
'SELECT MIN(created_at) as d FROM telemetry_events'
|
||||
).get() as { d: string | null };
|
||||
|
||||
const last = this.db.prepare(
|
||||
'SELECT MAX(created_at) as d FROM telemetry_events'
|
||||
).get() as { d: string | null };
|
||||
|
||||
const onboarded = (this.db.prepare(
|
||||
"SELECT COUNT(*) as cnt FROM telemetry_events WHERE event = 'onboarding_complete'"
|
||||
).get() as { cnt: number }).cnt > 0;
|
||||
|
||||
const sessions = (this.db.prepare(
|
||||
"SELECT COUNT(*) as cnt FROM telemetry_events WHERE event = 'session_start'"
|
||||
).get() as { cnt: number }).cnt;
|
||||
|
||||
const embRow = this.db.prepare(
|
||||
"SELECT properties FROM telemetry_events WHERE event = 'embedding_provider' ORDER BY created_at DESC LIMIT 1"
|
||||
).get() as { properties: string } | undefined;
|
||||
const embProvider = embRow
|
||||
? (JSON.parse(embRow.properties) as Record<string, unknown>).provider as string ?? null
|
||||
: null;
|
||||
|
||||
const tplRows = this.db.prepare(
|
||||
"SELECT DISTINCT json_extract(properties, '$.templateId') as tid FROM telemetry_events WHERE event IN ('template_selected', 'workspace_created') AND json_extract(properties, '$.templateId') IS NOT NULL"
|
||||
).all() as Array<{ tid: string }>;
|
||||
|
||||
const breakdownRows = this.db.prepare(
|
||||
'SELECT event, COUNT(*) as cnt FROM telemetry_events GROUP BY event ORDER BY cnt DESC'
|
||||
).all() as Array<{ event: string; cnt: number }>;
|
||||
const breakdown: Record<string, number> = {};
|
||||
for (const row of breakdownRows) {
|
||||
breakdown[row.event] = row.cnt;
|
||||
}
|
||||
|
||||
return {
|
||||
enabled: this.enabled,
|
||||
totalEvents: total,
|
||||
firstEvent: first.d,
|
||||
lastEvent: last.d,
|
||||
onboardingCompleted: onboarded,
|
||||
totalSessions: sessions,
|
||||
embeddingProvider: embProvider,
|
||||
templatesUsed: tplRows.map(r => r.tid),
|
||||
eventBreakdown: breakdown,
|
||||
};
|
||||
}
|
||||
|
||||
/** Get raw events with optional filters. */
|
||||
getEvents(options?: { event?: string; since?: string; until?: string; limit?: number }): TelemetryEvent[] {
|
||||
const conditions: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
|
||||
if (options?.event) {
|
||||
conditions.push('event = ?');
|
||||
params.push(options.event);
|
||||
}
|
||||
if (options?.since) {
|
||||
conditions.push('created_at >= ?');
|
||||
params.push(options.since);
|
||||
}
|
||||
if (options?.until) {
|
||||
conditions.push('created_at <= ?');
|
||||
params.push(options.until);
|
||||
}
|
||||
|
||||
const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
|
||||
const limit = options?.limit ?? 100;
|
||||
|
||||
const rows = this.db.prepare(
|
||||
`SELECT id, event, properties, created_at FROM telemetry_events ${where} ORDER BY created_at DESC LIMIT ?`
|
||||
).all(...params, limit) as Array<{ id: number; event: string; properties: string; created_at: string }>;
|
||||
|
||||
return rows.map(r => ({
|
||||
id: r.id,
|
||||
event: r.event,
|
||||
properties: JSON.parse(r.properties) as Record<string, unknown>,
|
||||
created_at: r.created_at,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Delete all telemetry data (user right-to-delete). */
|
||||
clear(): { deleted: number } {
|
||||
const result = this.db.prepare('DELETE FROM telemetry_events').run();
|
||||
return { deleted: result.changes };
|
||||
}
|
||||
|
||||
/** Close the database connection. */
|
||||
close(): void {
|
||||
this.db.close();
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Collector ── */
|
||||
|
||||
const COLLECTOR_SCHEMA = `
|
||||
CREATE TABLE IF NOT EXISTS collector_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
category TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
count INTEGER NOT NULL DEFAULT 1,
|
||||
date TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_collector_date ON collector_events (date);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_collector_name_date ON collector_events (name, date);
|
||||
`;
|
||||
|
||||
/**
|
||||
* TelemetryCollector — higher-level daily-aggregated event tracker.
|
||||
*
|
||||
* Records tool usage, commands, errors, and capability gaps by day.
|
||||
* Privacy-safe: only category + name + count, no content or PII.
|
||||
*/
|
||||
export class TelemetryCollector {
|
||||
private db: DatabaseType;
|
||||
private enabled: boolean;
|
||||
private dataDir: string;
|
||||
|
||||
constructor(dataDir: string, enabled = false) {
|
||||
this.dataDir = dataDir;
|
||||
this.enabled = enabled;
|
||||
const dbPath = path.join(dataDir, 'telemetry.db');
|
||||
this.db = new Database(dbPath);
|
||||
this.db.pragma('journal_mode = WAL');
|
||||
this.db.exec(COLLECTOR_SCHEMA);
|
||||
}
|
||||
|
||||
recordToolUse(toolName: string): void {
|
||||
if (!this.enabled) return;
|
||||
this.upsertEvent('tool', toolName);
|
||||
}
|
||||
|
||||
recordCommand(command: string): void {
|
||||
if (!this.enabled) return;
|
||||
this.upsertEvent('command', command);
|
||||
}
|
||||
|
||||
recordError(errorName: string): void {
|
||||
if (!this.enabled) return;
|
||||
this.upsertEvent('error', errorName);
|
||||
}
|
||||
|
||||
recordCapabilityGap(capability: string): void {
|
||||
if (!this.enabled) return;
|
||||
this.upsertEvent('capability_gap', capability);
|
||||
}
|
||||
|
||||
recordSession(durationMs: number, interactionCount: number): void {
|
||||
if (!this.enabled) return;
|
||||
const today = new Date().toISOString().split('T')[0];
|
||||
this.db.prepare(
|
||||
`INSERT INTO collector_events (category, name, count, date) VALUES ('session', 'duration_total_ms', ?, ?)
|
||||
ON CONFLICT(name, date) DO UPDATE SET count = count + excluded.count`
|
||||
).run(durationMs, today);
|
||||
this.db.prepare(
|
||||
`INSERT INTO collector_events (category, name, count, date) VALUES ('session', 'interaction_count', ?, ?)
|
||||
ON CONFLICT(name, date) DO UPDATE SET count = count + excluded.count`
|
||||
).run(interactionCount, today);
|
||||
this.db.prepare(
|
||||
`INSERT INTO collector_events (category, name, count, date) VALUES ('session', 'session_count', 1, ?)
|
||||
ON CONFLICT(name, date) DO UPDATE SET count = count + 1`
|
||||
).run(today);
|
||||
}
|
||||
|
||||
private upsertEvent(category: string, name: string): void {
|
||||
const today = new Date().toISOString().split('T')[0];
|
||||
this.db.prepare(
|
||||
`INSERT INTO collector_events (category, name, count, date) VALUES (?, ?, 1, ?)
|
||||
ON CONFLICT(name, date) DO UPDATE SET count = count + 1`
|
||||
).run(category, name, today);
|
||||
}
|
||||
|
||||
getReport(days = 7): {
|
||||
totalEvents: number;
|
||||
events: Array<{ name: string; count: number; category: string; date: string }>;
|
||||
dateRange: { from: string | null };
|
||||
} {
|
||||
const since = new Date();
|
||||
since.setDate(since.getDate() - days);
|
||||
const sinceStr = since.toISOString().split('T')[0];
|
||||
|
||||
const rows = this.db.prepare(
|
||||
`SELECT name, SUM(count) as total_count, category, date
|
||||
FROM collector_events WHERE date >= ?
|
||||
GROUP BY name, category, date ORDER BY date DESC, name`
|
||||
).all(sinceStr) as Array<{ name: string; total_count: number; category: string; date: string }>;
|
||||
|
||||
const totalEvents = rows.reduce((sum, r) => sum + r.total_count, 0);
|
||||
const events = rows.map(r => ({ name: r.name, count: r.total_count, category: r.category, date: r.date }));
|
||||
const firstRow = this.db.prepare(
|
||||
'SELECT MIN(date) as first_date FROM collector_events'
|
||||
).get() as { first_date: string | null };
|
||||
|
||||
return { totalEvents, events, dateRange: { from: firstRow.first_date } };
|
||||
}
|
||||
|
||||
/** Write events to telemetry.json for external consumption. */
|
||||
flush(): void {
|
||||
const rows = this.db.prepare(
|
||||
'SELECT name, count, category, date FROM collector_events ORDER BY date DESC, name'
|
||||
).all() as Array<{ name: string; count: number; category: string; date: string }>;
|
||||
fs.writeFileSync(
|
||||
path.join(this.dataDir, 'telemetry.json'),
|
||||
JSON.stringify(rows, null, 2),
|
||||
);
|
||||
}
|
||||
|
||||
setEnabled(enabled: boolean): void { this.enabled = enabled; }
|
||||
isEnabled(): boolean { return this.enabled; }
|
||||
close(): void { this.db.close(); }
|
||||
}
|
||||
299
packages/core/src/vault.ts
Normal file
299
packages/core/src/vault.ts
Normal file
@@ -0,0 +1,299 @@
|
||||
/**
|
||||
* VaultStore — encrypted local secrets store.
|
||||
*
|
||||
* Encrypts secrets using AES-256-GCM (Node.js built-in crypto).
|
||||
* Stores encrypted data in vault.json with a machine-local key file.
|
||||
* Each entry is independently encrypted so individual secrets can be
|
||||
* updated without re-encrypting everything.
|
||||
*/
|
||||
|
||||
import * as crypto from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { createCoreLogger } from '@waggle/hive-mind-core';
|
||||
|
||||
const log = createCoreLogger('vault');
|
||||
const ALGORITHM = 'aes-256-gcm';
|
||||
const KEY_LENGTH = 32;
|
||||
const IV_LENGTH = 16;
|
||||
|
||||
export interface VaultEntry {
|
||||
name: string;
|
||||
value: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
interface VaultRecord {
|
||||
encrypted: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export class VaultStore {
|
||||
private dataDir: string;
|
||||
private vaultPath: string;
|
||||
private keyPath: string;
|
||||
private encryptionKey: Buffer;
|
||||
|
||||
constructor(dataDir: string) {
|
||||
this.dataDir = dataDir;
|
||||
this.vaultPath = path.join(dataDir, 'vault.json');
|
||||
this.keyPath = path.join(dataDir, '.vault-key');
|
||||
|
||||
// Ensure data directory exists
|
||||
if (!fs.existsSync(this.dataDir)) {
|
||||
fs.mkdirSync(this.dataDir, { recursive: true });
|
||||
}
|
||||
|
||||
this.encryptionKey = this.ensureKey();
|
||||
}
|
||||
|
||||
/** Ensure the encryption key exists. Generate if missing. */
|
||||
private ensureKey(): Buffer {
|
||||
if (fs.existsSync(this.keyPath)) {
|
||||
const key = Buffer.from(fs.readFileSync(this.keyPath, 'utf-8').trim(), 'hex');
|
||||
if (key.length !== KEY_LENGTH) {
|
||||
throw new Error(
|
||||
`Vault key file is corrupted — expected ${KEY_LENGTH} bytes, got ${key.length}. Delete ${this.keyPath} to regenerate.`
|
||||
);
|
||||
}
|
||||
return key;
|
||||
}
|
||||
const key = crypto.randomBytes(KEY_LENGTH);
|
||||
fs.writeFileSync(this.keyPath, key.toString('hex'), { mode: 0o600 });
|
||||
// Review Critical #1: On Windows, restrict key file access to current user only.
|
||||
// Previously used `require('node:child_process')` inline which fails silently under
|
||||
// ESM (`type: module` in the sidecar) — the try/catch swallowed the ReferenceError
|
||||
// and every Windows install left the vault key with no ACL restriction.
|
||||
// Now imported statically at the top of the file; the try/catch only covers actual
|
||||
// icacls failures (e.g. icacls.exe not on PATH in a minimal Windows image).
|
||||
if (process.platform === 'win32') {
|
||||
try {
|
||||
// Resolve the current user via whoami (safer than process.env.USERNAME
|
||||
// which can be absent or spoofed in containerized/scripted setups)
|
||||
const currentUser = execFileSync('whoami', { encoding: 'utf-8' }).trim();
|
||||
execFileSync('icacls', [
|
||||
this.keyPath,
|
||||
'/inheritance:r',
|
||||
'/grant:r',
|
||||
`${currentUser}:F`,
|
||||
], { stdio: 'ignore' });
|
||||
} catch (err) {
|
||||
log.warn('Could not restrict key file permissions via icacls — vault key may be readable by other users', err);
|
||||
}
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
/** Encrypt a plaintext string. Returns iv:authTag:ciphertext (all hex). */
|
||||
private encrypt(plaintext: string): string {
|
||||
const iv = crypto.randomBytes(IV_LENGTH);
|
||||
const cipher = crypto.createCipheriv(ALGORITHM, this.encryptionKey, iv);
|
||||
const encrypted = Buffer.concat([cipher.update(plaintext, 'utf-8'), cipher.final()]);
|
||||
const authTag = cipher.getAuthTag();
|
||||
return `${iv.toString('hex')}:${authTag.toString('hex')}:${encrypted.toString('hex')}`;
|
||||
}
|
||||
|
||||
/** Decrypt an encoded string (iv:authTag:ciphertext, all hex). */
|
||||
private decrypt(encoded: string): string {
|
||||
const parts = encoded.split(':');
|
||||
if (parts.length !== 3) {
|
||||
throw new Error('Vault entry format invalid — expected iv:authTag:ciphertext');
|
||||
}
|
||||
const [ivHex, authTagHex, ciphertextHex] = parts;
|
||||
const iv = Buffer.from(ivHex, 'hex');
|
||||
const authTag = Buffer.from(authTagHex, 'hex');
|
||||
const ciphertext = Buffer.from(ciphertextHex, 'hex');
|
||||
const decipher = crypto.createDecipheriv(ALGORITHM, this.encryptionKey, iv);
|
||||
decipher.setAuthTag(authTag);
|
||||
return decipher.update(ciphertext) + decipher.final('utf-8');
|
||||
}
|
||||
|
||||
/** Read the vault file (encrypted entries). */
|
||||
private readVault(): Record<string, VaultRecord> {
|
||||
if (!fs.existsSync(this.vaultPath)) return {};
|
||||
try {
|
||||
// Review Critical #3: use a null-prototype object so a malicious `__proto__`
|
||||
// key in vault.json cannot pollute Object.prototype when accessed. Matches
|
||||
// the `name` validation at the route layer.
|
||||
const parsed = JSON.parse(fs.readFileSync(this.vaultPath, 'utf-8'));
|
||||
return Object.assign(Object.create(null), parsed);
|
||||
} catch (err) {
|
||||
// M5: corrupt vault.json — back up the corrupt file so data isn't silently lost
|
||||
log.error('Failed to parse vault.json — backing up corrupt file', err);
|
||||
const bakPath = this.vaultPath + '.bak';
|
||||
try { fs.copyFileSync(this.vaultPath, bakPath); } catch { /* best effort */ }
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/** Write the vault file atomically (write to .tmp, then replace). */
|
||||
private writeVault(vault: Record<string, VaultRecord>): void {
|
||||
const tmpPath = this.vaultPath + '.tmp';
|
||||
const data = JSON.stringify(vault, null, 2);
|
||||
fs.writeFileSync(tmpPath, data, { mode: 0o600 });
|
||||
try {
|
||||
try {
|
||||
fs.renameSync(tmpPath, this.vaultPath);
|
||||
} catch {
|
||||
// M6: On Windows, rename fails with EPERM when target exists.
|
||||
// Delete target first, then rename — preserves atomicity better than direct write.
|
||||
try { fs.unlinkSync(this.vaultPath); } catch { /* target may not exist */ }
|
||||
try {
|
||||
fs.renameSync(tmpPath, this.vaultPath);
|
||||
} catch (renameErr) {
|
||||
// Last resort: direct write (non-atomic) — log the degradation
|
||||
log.error('Atomic vault write failed, falling back to direct write', renameErr);
|
||||
fs.writeFileSync(this.vaultPath, data, { mode: 0o600 });
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
// M12: clean up .tmp if it still exists after any failure path
|
||||
try { if (fs.existsSync(tmpPath)) fs.unlinkSync(tmpPath); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
/** Set a secret (encrypts value, stores metadata alongside). Uses atomic write. */
|
||||
set(name: string, value: string, metadata?: Record<string, unknown>): void {
|
||||
const vault = this.readVault();
|
||||
vault[name] = {
|
||||
encrypted: this.encrypt(value),
|
||||
metadata,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
this.writeVault(vault);
|
||||
}
|
||||
|
||||
/**
|
||||
* Async set — delegates to the synchronous set().
|
||||
* Review Critical #2: the previous implementation used a promise chain
|
||||
* (.writeLock.then(...)) which deferred the actual write to a microtask.
|
||||
* A sync set() between the chain call and microtask execution would be
|
||||
* overwritten when the deferred write fired. Since all vault I/O is
|
||||
* synchronous (fs.readFileSync/writeFileSync), the lock is unnecessary —
|
||||
* each callback runs to completion before the next microtask. Delegating
|
||||
* to the sync method eliminates the interleaving window entirely.
|
||||
*/
|
||||
async setAsync(name: string, value: string, metadata?: Record<string, unknown>): Promise<void> {
|
||||
this.set(name, value, metadata);
|
||||
}
|
||||
|
||||
/** Get a decrypted secret by name. Returns null if not found or decryption fails. */
|
||||
get(name: string): VaultEntry | null {
|
||||
const vault = this.readVault();
|
||||
const entry = vault[name];
|
||||
if (!entry) return null;
|
||||
try {
|
||||
return {
|
||||
name,
|
||||
value: this.decrypt(entry.encrypted),
|
||||
metadata: entry.metadata,
|
||||
updatedAt: entry.updatedAt,
|
||||
};
|
||||
} catch {
|
||||
return null; // Decryption failed (key mismatch, corrupted)
|
||||
}
|
||||
}
|
||||
|
||||
/** Delete a secret. Returns true if it existed. Uses atomic write. */
|
||||
delete(name: string): boolean {
|
||||
const vault = this.readVault();
|
||||
if (!vault[name]) return false;
|
||||
delete vault[name];
|
||||
this.writeVault(vault);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Async delete — delegates to the synchronous delete().
|
||||
* Review Critical #2: same rationale as setAsync — sync I/O means no
|
||||
* interleaving risk. The previous promise-chain approach deferred the
|
||||
* actual delete, creating a window where a sync caller's write could
|
||||
* be overwritten.
|
||||
*/
|
||||
async deleteAsync(name: string): Promise<boolean> {
|
||||
return this.delete(name);
|
||||
}
|
||||
|
||||
/** List secret names (without values). */
|
||||
list(): Array<{ name: string; metadata?: Record<string, unknown>; updatedAt: string }> {
|
||||
const vault = this.readVault();
|
||||
return Object.entries(vault).map(([name, entry]) => ({
|
||||
name,
|
||||
metadata: entry.metadata,
|
||||
updatedAt: entry.updatedAt,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Check if a secret exists. */
|
||||
has(name: string): boolean {
|
||||
const vault = this.readVault();
|
||||
return name in vault;
|
||||
}
|
||||
|
||||
/** Set a connector credential with typed metadata */
|
||||
setConnectorCredential(connectorId: string, credential: {
|
||||
type: 'api_key' | 'oauth2' | 'bearer' | 'basic';
|
||||
value: string;
|
||||
refreshToken?: string;
|
||||
expiresAt?: string;
|
||||
scopes?: string[];
|
||||
}): void {
|
||||
this.set(`connector:${connectorId}`, credential.value, {
|
||||
credentialType: credential.type,
|
||||
expiresAt: credential.expiresAt,
|
||||
scopes: credential.scopes,
|
||||
});
|
||||
// Store refresh token as a separate encrypted entry (never in plaintext metadata)
|
||||
if (credential.refreshToken) {
|
||||
this.set(`connector:${connectorId}:refresh`, credential.refreshToken);
|
||||
} else {
|
||||
// Clear any previously stored refresh token if not provided
|
||||
this.delete(`connector:${connectorId}:refresh`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Get a connector credential with typed metadata */
|
||||
getConnectorCredential(connectorId: string): {
|
||||
value: string;
|
||||
type: string;
|
||||
refreshToken?: string;
|
||||
expiresAt?: string;
|
||||
scopes?: string[];
|
||||
isExpired: boolean;
|
||||
} | null {
|
||||
const entry = this.get(`connector:${connectorId}`);
|
||||
if (!entry) return null;
|
||||
const expiresAt = entry.metadata?.expiresAt as string | undefined;
|
||||
// Retrieve refresh token from its own encrypted entry
|
||||
const refreshEntry = this.get(`connector:${connectorId}:refresh`);
|
||||
return {
|
||||
value: entry.value,
|
||||
type: (entry.metadata?.credentialType as string) ?? 'api_key',
|
||||
// Review Major #7: removed dead plaintext metadata fallback that contradicted
|
||||
// the "never in plaintext metadata" security contract from setConnectorCredential.
|
||||
refreshToken: refreshEntry?.value,
|
||||
expiresAt,
|
||||
scopes: entry.metadata?.scopes as string[] | undefined,
|
||||
isExpired: expiresAt ? new Date(expiresAt) < new Date() : false,
|
||||
};
|
||||
}
|
||||
|
||||
/** Migrate plaintext providers from config.json to vault. Returns count migrated. */
|
||||
migrateFromConfig(config: { providers?: Record<string, { apiKey: string; models?: string[]; baseUrl?: string }> }): number {
|
||||
if (!config.providers) return 0;
|
||||
let migrated = 0;
|
||||
for (const [name, provider] of Object.entries(config.providers)) {
|
||||
if (provider.apiKey && !this.has(name)) {
|
||||
this.set(name, provider.apiKey, {
|
||||
models: provider.models,
|
||||
baseUrl: provider.baseUrl,
|
||||
});
|
||||
migrated++;
|
||||
}
|
||||
}
|
||||
return migrated;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user