This commit is contained in:
30
packages/core/package.json
Normal file
30
packages/core/package.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"name": "@waggle/core",
|
||||
"version": "0.1.0",
|
||||
"description": "Waggle-specific orchestration: config, multi-mind, workspace, vault, telemetry, install-audit, cron, skills, file-store, file-indexer, memory-import, optimization-log, compliance. Re-exports substrate from @waggle/hive-mind-core for backward compatibility.",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"import": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"test": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@waggle/hive-mind-core": "*",
|
||||
"@waggle/shared": "*",
|
||||
"better-sqlite3": "^12.6.2",
|
||||
"cron-parser": "^4.9.0",
|
||||
"glob": "^13.0.6"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@aws-sdk/client-s3": "^3.1019.0",
|
||||
"@types/better-sqlite3": "^7.6.13"
|
||||
},
|
||||
"license": "MIT"
|
||||
}
|
||||
5
packages/core/src/compliance/index.ts
Normal file
5
packages/core/src/compliance/index.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
export * from './types.js';
|
||||
export { InteractionStore } from './interaction-store.js';
|
||||
export { ComplianceStatusChecker } from './status-checker.js';
|
||||
export { ReportGenerator } from './report-generator.js';
|
||||
export { ComplianceTemplateStore } from './template-store.js';
|
||||
208
packages/core/src/compliance/interaction-store.ts
Normal file
208
packages/core/src/compliance/interaction-store.ts
Normal file
@@ -0,0 +1,208 @@
|
||||
/**
|
||||
* InteractionStore — CRUD for ai_interactions table (EU AI Act Art. 12).
|
||||
*
|
||||
* Records every AI interaction with model, tokens, cost, tools, and human oversight.
|
||||
* Operates on the .mind DB alongside FrameStore and InstallAuditStore.
|
||||
*/
|
||||
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
import type { AIInteraction, RecordInteractionInput, HumanAction, ModelInventoryEntry, OversightLogEntry } from './types.js';
|
||||
|
||||
export class InteractionStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
// Review Major #8: previously ensureTable() duplicated the DDL from schema.ts and
|
||||
// drifted whenever the canonical schema changed. Now schema.ts + MindDB.runMigrations()
|
||||
// own the table definition, including the input_text/output_text columns and the
|
||||
// append-only triggers (Critical #1, #3).
|
||||
}
|
||||
|
||||
/** Record an AI interaction event. */
|
||||
record(input: RecordInteractionInput): AIInteraction {
|
||||
const raw = this.db.getDatabase();
|
||||
|
||||
// Review Major #4: use lastInsertRowid from .run() instead of reading back
|
||||
// ORDER BY id DESC LIMIT 1 — under concurrent writes (WaggleDance multi-agent),
|
||||
// the LIMIT 1 row may belong to a different writer.
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO ai_interactions (
|
||||
workspace_id, session_id, model, provider,
|
||||
input_tokens, output_tokens, cost_usd,
|
||||
tools_called, human_action, risk_context, imported_from, persona,
|
||||
input_text, output_text
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
input.workspaceId ?? null,
|
||||
input.sessionId ?? null,
|
||||
input.model,
|
||||
input.provider,
|
||||
input.inputTokens,
|
||||
input.outputTokens,
|
||||
input.costUsd,
|
||||
JSON.stringify(input.toolsCalled ?? []),
|
||||
input.humanAction ?? 'none',
|
||||
input.riskContext ?? null,
|
||||
input.importedFrom ?? null,
|
||||
input.persona ?? null,
|
||||
input.inputText ?? null,
|
||||
input.outputText ?? null,
|
||||
);
|
||||
|
||||
const row = raw.prepare('SELECT * FROM ai_interactions WHERE id = ?').get(result.lastInsertRowid) as Record<string, unknown>;
|
||||
return this.rowToInteraction(row);
|
||||
}
|
||||
|
||||
/** Get interactions for a workspace within a date range. */
|
||||
getByWorkspace(workspaceId: string, from?: string, to?: string): AIInteraction[] {
|
||||
if (from && !/^\d{4}-\d{2}-\d{2}/.test(from)) throw new Error('Invalid date format');
|
||||
if (to && !/^\d{4}-\d{2}-\d{2}/.test(to)) throw new Error('Invalid date format');
|
||||
|
||||
const raw = this.db.getDatabase();
|
||||
let sql = 'SELECT * FROM ai_interactions WHERE workspace_id = ?';
|
||||
const params: unknown[] = [workspaceId];
|
||||
|
||||
if (from) { sql += ' AND timestamp >= ?'; params.push(from); }
|
||||
if (to) { sql += ' AND timestamp <= ?'; params.push(to); }
|
||||
sql += ' ORDER BY timestamp DESC LIMIT 1000';
|
||||
|
||||
return (raw.prepare(sql).all(...params) as Record<string, unknown>[]).map(r => this.rowToInteraction(r));
|
||||
}
|
||||
|
||||
/** Get all interactions within a date range. */
|
||||
getByDateRange(from: string, to: string, workspaceId?: string): AIInteraction[] {
|
||||
if (from && !/^\d{4}-\d{2}-\d{2}/.test(from)) throw new Error('Invalid date format');
|
||||
if (to && !/^\d{4}-\d{2}-\d{2}/.test(to)) throw new Error('Invalid date format');
|
||||
|
||||
const raw = this.db.getDatabase();
|
||||
let sql = 'SELECT * FROM ai_interactions WHERE timestamp >= ? AND timestamp <= ?';
|
||||
const params: unknown[] = [from, to];
|
||||
|
||||
if (workspaceId) { sql += ' AND workspace_id = ?'; params.push(workspaceId); }
|
||||
sql += ' ORDER BY timestamp DESC LIMIT 1000';
|
||||
|
||||
return (raw.prepare(sql).all(...params) as Record<string, unknown>[]).map(r => this.rowToInteraction(r));
|
||||
}
|
||||
|
||||
/** Get total interaction count. */
|
||||
count(workspaceId?: string): number {
|
||||
const raw = this.db.getDatabase();
|
||||
if (workspaceId) {
|
||||
const row = raw.prepare('SELECT COUNT(*) as cnt FROM ai_interactions WHERE workspace_id = ?').get(workspaceId) as { cnt: number };
|
||||
return row.cnt;
|
||||
}
|
||||
const row = raw.prepare('SELECT COUNT(*) as cnt FROM ai_interactions').get() as { cnt: number };
|
||||
return row.cnt;
|
||||
}
|
||||
|
||||
/** Get the oldest log timestamp (Art. 19 retention check). */
|
||||
getOldestTimestamp(): string | null {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT MIN(timestamp) as oldest FROM ai_interactions').get() as { oldest: string | null };
|
||||
return row.oldest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the `first_run_at` timestamp from the MindDB's `meta` table — set on
|
||||
* schema initialization and backfilled for pre-existing DBs. Used by the Art. 19
|
||||
* retention checker to distinguish 'new system' from 'pruned logs'.
|
||||
*/
|
||||
getFirstRunAt(): string | null {
|
||||
return this.db.getFirstRunAt();
|
||||
}
|
||||
|
||||
/** Get model inventory (aggregated usage per model). */
|
||||
getModelInventory(from?: string, to?: string, workspaceId?: string): ModelInventoryEntry[] {
|
||||
const raw = this.db.getDatabase();
|
||||
let sql = `
|
||||
SELECT model, provider,
|
||||
COUNT(*) as calls,
|
||||
SUM(input_tokens) as input_tokens,
|
||||
SUM(output_tokens) as output_tokens,
|
||||
SUM(cost_usd) as cost_usd
|
||||
FROM ai_interactions WHERE 1=1
|
||||
`;
|
||||
const params: unknown[] = [];
|
||||
|
||||
if (from) { sql += ' AND timestamp >= ?'; params.push(from); }
|
||||
if (to) { sql += ' AND timestamp <= ?'; params.push(to); }
|
||||
if (workspaceId) { sql += ' AND workspace_id = ?'; params.push(workspaceId); }
|
||||
sql += ' GROUP BY model, provider ORDER BY calls DESC';
|
||||
|
||||
return (raw.prepare(sql).all(...params) as Record<string, unknown>[]).map(r => ({
|
||||
model: r.model as string,
|
||||
provider: r.provider as string,
|
||||
calls: r.calls as number,
|
||||
inputTokens: r.input_tokens as number,
|
||||
outputTokens: r.output_tokens as number,
|
||||
costUsd: r.cost_usd as number,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Get human oversight actions (Art. 14). */
|
||||
getOversightLog(from?: string, to?: string, workspaceId?: string): OversightLogEntry[] {
|
||||
const raw = this.db.getDatabase();
|
||||
let sql = "SELECT * FROM ai_interactions WHERE human_action != 'none' AND human_action IS NOT NULL";
|
||||
const params: unknown[] = [];
|
||||
|
||||
if (from) { sql += ' AND timestamp >= ?'; params.push(from); }
|
||||
if (to) { sql += ' AND timestamp <= ?'; params.push(to); }
|
||||
if (workspaceId) { sql += ' AND workspace_id = ?'; params.push(workspaceId); }
|
||||
sql += ' ORDER BY timestamp DESC';
|
||||
|
||||
return (raw.prepare(sql).all(...params) as Record<string, unknown>[]).map(r => ({
|
||||
timestamp: r.timestamp as string,
|
||||
action: r.human_action as HumanAction,
|
||||
tool: (JSON.parse(r.tools_called as string) as string[])[0] ?? 'none',
|
||||
detail: r.persona ? `Persona: ${r.persona}` : '',
|
||||
}));
|
||||
}
|
||||
|
||||
/** Get human oversight counts for compliance check. */
|
||||
getOversightCounts(workspaceId?: string): { total: number; approved: number; denied: number; modified: number } {
|
||||
const raw = this.db.getDatabase();
|
||||
let sql = "SELECT human_action, COUNT(*) as cnt FROM ai_interactions WHERE human_action != 'none' AND human_action IS NOT NULL";
|
||||
const params: unknown[] = [];
|
||||
if (workspaceId) { sql += ' AND workspace_id = ?'; params.push(workspaceId); }
|
||||
sql += ' GROUP BY human_action';
|
||||
|
||||
const rows = raw.prepare(sql).all(...params) as { human_action: string; cnt: number }[];
|
||||
const counts = { total: 0, approved: 0, denied: 0, modified: 0 };
|
||||
for (const row of rows) {
|
||||
counts.total += row.cnt;
|
||||
if (row.human_action === 'approved') counts.approved = row.cnt;
|
||||
if (row.human_action === 'denied') counts.denied = row.cnt;
|
||||
if (row.human_action === 'modified') counts.modified = row.cnt;
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
/** Get recent interactions. */
|
||||
getRecent(limit: number = 20): AIInteraction[] {
|
||||
const raw = this.db.getDatabase();
|
||||
return (raw.prepare('SELECT * FROM ai_interactions ORDER BY id DESC LIMIT ?').all(limit) as Record<string, unknown>[])
|
||||
.map(r => this.rowToInteraction(r));
|
||||
}
|
||||
|
||||
private rowToInteraction(row: Record<string, unknown>): AIInteraction {
|
||||
return {
|
||||
id: row.id as number,
|
||||
timestamp: row.timestamp as string,
|
||||
workspaceId: row.workspace_id as string | null,
|
||||
sessionId: row.session_id as string | null,
|
||||
model: row.model as string,
|
||||
provider: row.provider as string,
|
||||
inputTokens: row.input_tokens as number,
|
||||
outputTokens: row.output_tokens as number,
|
||||
costUsd: row.cost_usd as number,
|
||||
toolsCalled: JSON.parse(row.tools_called as string) as string[],
|
||||
humanAction: (row.human_action as HumanAction) ?? 'none',
|
||||
riskContext: row.risk_context as string | null,
|
||||
importedFrom: row.imported_from as string | null,
|
||||
persona: row.persona as string | null,
|
||||
inputText: (row.input_text as string | null | undefined) ?? null,
|
||||
outputText: (row.output_text as string | null | undefined) ?? null,
|
||||
};
|
||||
}
|
||||
}
|
||||
104
packages/core/src/compliance/report-generator.ts
Normal file
104
packages/core/src/compliance/report-generator.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
/**
|
||||
* ReportGenerator — generates AI Act compliance audit reports.
|
||||
*
|
||||
* Produces JSON reports (PDF generation deferred to document tooling).
|
||||
* Reports cover: compliance status, model inventory, oversight log,
|
||||
* harvest provenance, and interaction counts.
|
||||
*/
|
||||
|
||||
import type { InteractionStore } from './interaction-store.js';
|
||||
import type { HarvestSourceStore } from '@waggle/hive-mind-core';
|
||||
import { ComplianceStatusChecker } from './status-checker.js';
|
||||
import type { AuditReport, AuditReportRequest, AIActRiskLevel } from './types.js';
|
||||
|
||||
const REPORT_VERSION = '1.0';
|
||||
|
||||
export interface ReportGeneratorDeps {
|
||||
interactionStore: InteractionStore;
|
||||
harvestStore: HarvestSourceStore;
|
||||
getWorkspaceRisk?: (workspaceId: string) => AIActRiskLevel;
|
||||
getWorkspaceName?: (workspaceId: string) => string;
|
||||
/**
|
||||
* Return the ISO timestamp of the last risk classification for this
|
||||
* workspace, or null if the workspace has no persisted classification
|
||||
* date (e.g., was created before the field existed).
|
||||
*/
|
||||
getWorkspaceRiskClassifiedAt?: (workspaceId: string) => string | null;
|
||||
}
|
||||
|
||||
export class ReportGenerator {
|
||||
private interactions: InteractionStore;
|
||||
private harvest: HarvestSourceStore;
|
||||
private getWorkspaceRisk: (id: string) => AIActRiskLevel;
|
||||
private getWorkspaceName: (id: string) => string;
|
||||
private getWorkspaceRiskClassifiedAt: (id: string) => string | null;
|
||||
|
||||
constructor(deps: ReportGeneratorDeps) {
|
||||
this.interactions = deps.interactionStore;
|
||||
this.harvest = deps.harvestStore;
|
||||
this.getWorkspaceRisk = deps.getWorkspaceRisk ?? (() => 'minimal');
|
||||
this.getWorkspaceName = deps.getWorkspaceName ?? ((id) => id);
|
||||
this.getWorkspaceRiskClassifiedAt = deps.getWorkspaceRiskClassifiedAt ?? (() => null);
|
||||
}
|
||||
|
||||
/** Generate a full audit report. */
|
||||
generate(request: AuditReportRequest): AuditReport {
|
||||
const checker = new ComplianceStatusChecker(this.interactions);
|
||||
const complianceStatus = checker.check(request.workspaceId);
|
||||
|
||||
const report: AuditReport = {
|
||||
report: {
|
||||
version: REPORT_VERSION,
|
||||
generatedAt: new Date().toISOString(),
|
||||
period: { from: request.from, to: request.to },
|
||||
generatedBy: 'Waggle OS',
|
||||
},
|
||||
workspace: request.workspaceId ? {
|
||||
id: request.workspaceId,
|
||||
name: this.getWorkspaceName(request.workspaceId),
|
||||
riskLevel: this.getWorkspaceRisk(request.workspaceId),
|
||||
riskClassifiedAt: this.getWorkspaceRiskClassifiedAt(request.workspaceId),
|
||||
} : null,
|
||||
complianceStatus,
|
||||
modelInventory: [],
|
||||
humanOversightLog: [],
|
||||
harvestProvenance: [],
|
||||
interactionCount: 0,
|
||||
};
|
||||
|
||||
// Model inventory
|
||||
if (request.include.models) {
|
||||
report.modelInventory = this.interactions.getModelInventory(
|
||||
request.from, request.to, request.workspaceId,
|
||||
);
|
||||
}
|
||||
|
||||
// Human oversight log
|
||||
if (request.include.oversight) {
|
||||
report.humanOversightLog = this.interactions.getOversightLog(
|
||||
request.from, request.to, request.workspaceId,
|
||||
);
|
||||
}
|
||||
|
||||
// Harvest provenance
|
||||
if (request.include.provenance) {
|
||||
const sources = this.harvest.getAll();
|
||||
report.harvestProvenance = sources.map(s => ({
|
||||
source: s.displayName,
|
||||
importedAt: s.lastSyncedAt ?? s.createdAt,
|
||||
itemsImported: s.itemsImported,
|
||||
framesCreated: s.framesCreated,
|
||||
}));
|
||||
}
|
||||
|
||||
// Interaction count
|
||||
if (request.include.interactions) {
|
||||
const interactions = this.interactions.getByDateRange(
|
||||
request.from, request.to, request.workspaceId,
|
||||
);
|
||||
report.interactionCount = interactions.length;
|
||||
}
|
||||
|
||||
return report;
|
||||
}
|
||||
}
|
||||
175
packages/core/src/compliance/status-checker.ts
Normal file
175
packages/core/src/compliance/status-checker.ts
Normal file
@@ -0,0 +1,175 @@
|
||||
/**
|
||||
* ComplianceStatusChecker — evaluates AI Act compliance per workspace.
|
||||
*
|
||||
* Checks:
|
||||
* - Art. 12: Automatic event logging (ai_interactions count)
|
||||
* - Art. 14: Human oversight (approval/denial actions recorded)
|
||||
* - Art. 19: Log retention (oldest log >= 6 months ago)
|
||||
* - Art. 26: Deployer monitoring (active monitors)
|
||||
* - Art. 50: Model transparency (models disclosed)
|
||||
*/
|
||||
|
||||
import type { InteractionStore } from './interaction-store.js';
|
||||
import type { ComplianceStatus, ArticleStatus } from './types.js';
|
||||
|
||||
const SIX_MONTHS_MS = 180 * 24 * 60 * 60 * 1000;
|
||||
|
||||
export class ComplianceStatusChecker {
|
||||
private store: InteractionStore;
|
||||
|
||||
constructor(store: InteractionStore) {
|
||||
this.store = store;
|
||||
}
|
||||
|
||||
/** Evaluate full compliance status for a workspace (or all workspaces if null). */
|
||||
check(workspaceId?: string): ComplianceStatus {
|
||||
const art12 = this.checkArt12(workspaceId);
|
||||
const art14 = this.checkArt14(workspaceId);
|
||||
const art19 = this.checkArt19();
|
||||
const art26 = this.checkArt26();
|
||||
const art50 = this.checkArt50(workspaceId);
|
||||
|
||||
const statuses = [art12.status, art14.status, art19.status, art26.status, art50.status];
|
||||
const overall = statuses.includes('non-compliant') ? 'non-compliant'
|
||||
: statuses.includes('warning') ? 'warning'
|
||||
: 'compliant';
|
||||
|
||||
return {
|
||||
overall,
|
||||
art12Logging: art12,
|
||||
art14Oversight: art14,
|
||||
art19Retention: art19,
|
||||
art26Monitoring: art26,
|
||||
art50Transparency: art50,
|
||||
};
|
||||
}
|
||||
|
||||
/** Art. 12: Automatic recording of events. */
|
||||
private checkArt12(workspaceId?: string): ArticleStatus & { totalInteractions: number } {
|
||||
const total = this.store.count(workspaceId);
|
||||
|
||||
if (total === 0) {
|
||||
// M11: if the DB has been active for >24h with zero interactions, escalate to non-compliant
|
||||
const firstRun = this.store.getFirstRunAt();
|
||||
const activeOver24h = firstRun
|
||||
? (Date.now() - new Date(firstRun).getTime()) > 24 * 60 * 60 * 1000
|
||||
: false;
|
||||
|
||||
return {
|
||||
status: activeOver24h ? 'non-compliant' : 'warning',
|
||||
detail: activeOver24h
|
||||
? 'No interactions logged despite system being active for over 24 hours. Verify logging pipeline.'
|
||||
: 'No interactions logged yet. Logging activates automatically on first AI interaction.',
|
||||
totalInteractions: 0,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
status: 'compliant',
|
||||
detail: `${total} interactions logged with full model, token, cost, and tool tracking.`,
|
||||
totalInteractions: total,
|
||||
};
|
||||
}
|
||||
|
||||
/** Art. 14: Human oversight capability. */
|
||||
private checkArt14(workspaceId?: string): ArticleStatus & { humanActions: number; approvalRate: number } {
|
||||
const counts = this.store.getOversightCounts(workspaceId);
|
||||
|
||||
if (counts.total === 0) {
|
||||
return {
|
||||
status: 'compliant',
|
||||
detail: 'Human oversight capabilities available (approval gates, tool deny lists). No oversight actions recorded yet.',
|
||||
humanActions: 0,
|
||||
approvalRate: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const approvalRate = counts.total > 0
|
||||
? Math.round((counts.approved / counts.total) * 100)
|
||||
: 0;
|
||||
|
||||
return {
|
||||
status: 'compliant',
|
||||
detail: `${counts.total} human oversight actions: ${counts.approved} approved, ${counts.denied} denied, ${counts.modified} modified.`,
|
||||
humanActions: counts.total,
|
||||
approvalRate,
|
||||
};
|
||||
}
|
||||
|
||||
/** Art. 19: Log retention (minimum 6 months). */
|
||||
private checkArt19(): ArticleStatus & { oldestLogDate: string | null; retentionDays: number } {
|
||||
const oldest = this.store.getOldestTimestamp();
|
||||
|
||||
if (!oldest) {
|
||||
return {
|
||||
status: 'compliant',
|
||||
detail: 'No logs to retain yet. Retention policy is permanent by default.',
|
||||
oldestLogDate: null,
|
||||
retentionDays: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const oldestDate = new Date(oldest);
|
||||
const now = new Date();
|
||||
const retentionMs = now.getTime() - oldestDate.getTime();
|
||||
const retentionDays = Math.floor(retentionMs / (24 * 60 * 60 * 1000));
|
||||
|
||||
// Review Critical #2: the previous expression was a tautology
|
||||
// (`retentionMs >= SIX_MONTHS_MS || retentionDays < 180`) that covered every
|
||||
// non-negative value of retentionDays. A deployment that pruned logs after 30
|
||||
// days still reported compliant.
|
||||
//
|
||||
// Proper fix requires distinguishing 'system is young' from 'logs were pruned'.
|
||||
// We track system age via MindDB's `meta.first_run_at` entry (set on schema init,
|
||||
// backfilled for pre-existing DBs). If the system has been running for 180+ days
|
||||
// but the oldest log is younger than that, something pruned the logs and we
|
||||
// correctly report warning.
|
||||
const firstRun = this.store.getFirstRunAt();
|
||||
const systemAgeMs = firstRun ? now.getTime() - new Date(firstRun).getTime() : retentionMs;
|
||||
const hasBeenRunning6Months = systemAgeMs >= SIX_MONTHS_MS;
|
||||
const logsOlderThan6Months = retentionMs >= SIX_MONTHS_MS;
|
||||
const meetsMinimum = !hasBeenRunning6Months || logsOlderThan6Months;
|
||||
|
||||
return {
|
||||
status: meetsMinimum ? 'compliant' : 'warning',
|
||||
detail: meetsMinimum
|
||||
? hasBeenRunning6Months
|
||||
? `Logs retained since ${oldest.split('T')[0]} (${retentionDays} days). Art. 19 minimum (180 days) met.`
|
||||
: `Logs retained since ${oldest.split('T')[0]} (${retentionDays} days). System is still within its first 180 days — retention compliance will be enforceable after 2026-${(new Date(firstRun ?? now).getMonth() + 7).toString().padStart(2, '0')}.`
|
||||
: `Oldest log: ${oldest.split('T')[0]} (${retentionDays} days) but system is ${Math.floor(systemAgeMs / (24 * 60 * 60 * 1000))} days old. Logs appear to have been pruned — EU AI Act Art. 19 requires 180-day minimum retention.`,
|
||||
oldestLogDate: oldest,
|
||||
retentionDays,
|
||||
};
|
||||
}
|
||||
|
||||
/** Art. 26: Deployer monitoring obligations. */
|
||||
private checkArt26(): ArticleStatus & { activeMonitors: string[] } {
|
||||
// Waggle always has these monitors active
|
||||
const monitors = [
|
||||
'cost_tracking', // CostTracker in packages/agent
|
||||
'tool_logging', // Tool calls logged per interaction
|
||||
'model_identification', // Model recorded per interaction
|
||||
'persona_tracking', // Persona recorded per interaction
|
||||
];
|
||||
|
||||
return {
|
||||
status: 'compliant',
|
||||
detail: `${monitors.length} active monitors: cost, tools, model ID, persona.`,
|
||||
activeMonitors: monitors,
|
||||
};
|
||||
}
|
||||
|
||||
/** Art. 50: Transparency — models disclosed. */
|
||||
private checkArt50(workspaceId?: string): ArticleStatus & { modelsDisclosed: boolean } {
|
||||
const inventory = this.store.getModelInventory(undefined, undefined, workspaceId);
|
||||
const modelsDisclosed = inventory.length > 0;
|
||||
|
||||
return {
|
||||
status: 'compliant',
|
||||
detail: modelsDisclosed
|
||||
? `${inventory.length} model(s) in use, all identified in StatusBar and interaction logs.`
|
||||
: 'Model identification active. Models will be disclosed on first interaction.',
|
||||
modelsDisclosed,
|
||||
};
|
||||
}
|
||||
}
|
||||
242
packages/core/src/compliance/template-store.ts
Normal file
242
packages/core/src/compliance/template-store.ts
Normal file
@@ -0,0 +1,242 @@
|
||||
/**
|
||||
* ComplianceTemplateStore — CRUD for user-editable compliance report templates.
|
||||
*
|
||||
* M-03: Templates let a user save a named "report shape" (which sections to include,
|
||||
* a risk-class override, and optional org/footer text) and re-apply it to subsequent
|
||||
* audit-report exports. Sections MERGE with the runtime selection (union semantics).
|
||||
*
|
||||
* Logo support is deferred to Bucket 2. See docs/plans/COMPLIANCE-AUDIT-2026-04-20.md.
|
||||
*
|
||||
* DDL is self-contained here (following HarvestRunStore pattern) rather than threaded
|
||||
* through mind/schema.ts — compliance_templates is not part of the Art. 12 audit
|
||||
* substrate and doesn't need append-only triggers or cross-layer migration tracking.
|
||||
*/
|
||||
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
import type {
|
||||
AIActRiskLevel,
|
||||
ComplianceTemplate,
|
||||
ComplianceTemplateSections,
|
||||
CreateComplianceTemplateInput,
|
||||
UpdateComplianceTemplateInput,
|
||||
} from './types.js';
|
||||
|
||||
const CREATE_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS compliance_templates (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
sections_json TEXT NOT NULL DEFAULT '{}',
|
||||
risk_classification TEXT CHECK (risk_classification IN ('minimal', 'limited', 'high-risk', 'unacceptable')),
|
||||
org_name TEXT,
|
||||
footer_text TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
)
|
||||
`;
|
||||
// TODO Bucket 2 (M-05 custom branding): add `logo_blob BLOB` + `logo_mime TEXT`
|
||||
// columns. The `Edit templates` modal will grow a file-picker; renderComplianceReportPdf
|
||||
// will render the blob into the PDF header when present.
|
||||
|
||||
const CREATE_INDEX_SQL = `
|
||||
CREATE INDEX IF NOT EXISTS idx_compliance_templates_name ON compliance_templates (name)
|
||||
`;
|
||||
|
||||
const DEFAULT_SECTIONS: ComplianceTemplateSections = {
|
||||
interactions: true,
|
||||
oversight: true,
|
||||
models: true,
|
||||
provenance: true,
|
||||
riskAssessment: true,
|
||||
fria: false,
|
||||
};
|
||||
|
||||
/**
|
||||
* Canonical name for the seeded KVARK template. Used for idempotency — the
|
||||
* seed only fires when no row with this exact name exists.
|
||||
*/
|
||||
export const KVARK_TEMPLATE_NAME = 'KVARK Enterprise Audit';
|
||||
|
||||
export class ComplianceTemplateStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
const existsRow = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='compliance_templates'",
|
||||
).get();
|
||||
if (existsRow) return;
|
||||
raw.prepare(CREATE_TABLE_SQL).run();
|
||||
raw.prepare(CREATE_INDEX_SQL).run();
|
||||
}
|
||||
|
||||
create(input: CreateComplianceTemplateInput): ComplianceTemplate {
|
||||
const name = input.name.trim();
|
||||
if (!name) throw new Error('Template name is required');
|
||||
|
||||
const raw = this.db.getDatabase();
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO compliance_templates (
|
||||
name, description, sections_json, risk_classification, org_name, footer_text
|
||||
) VALUES (?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
name,
|
||||
input.description ?? null,
|
||||
JSON.stringify(this.normalizeSections(input.sections)),
|
||||
input.riskClassification ?? null,
|
||||
input.orgName ?? null,
|
||||
input.footerText ?? null,
|
||||
);
|
||||
return this.getById(Number(result.lastInsertRowid))!;
|
||||
}
|
||||
|
||||
getById(id: number): ComplianceTemplate | null {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT * FROM compliance_templates WHERE id = ?').get(id) as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
return row ? this.rowToTemplate(row) : null;
|
||||
}
|
||||
|
||||
list(): ComplianceTemplate[] {
|
||||
const raw = this.db.getDatabase();
|
||||
return (
|
||||
raw
|
||||
.prepare('SELECT * FROM compliance_templates ORDER BY updated_at DESC')
|
||||
.all() as Record<string, unknown>[]
|
||||
).map(r => this.rowToTemplate(r));
|
||||
}
|
||||
|
||||
update(id: number, patch: UpdateComplianceTemplateInput): ComplianceTemplate | null {
|
||||
const existing = this.getById(id);
|
||||
if (!existing) return null;
|
||||
|
||||
const next: ComplianceTemplate = {
|
||||
...existing,
|
||||
name: patch.name !== undefined ? patch.name.trim() : existing.name,
|
||||
description: patch.description !== undefined ? patch.description : existing.description,
|
||||
sections: patch.sections !== undefined ? this.normalizeSections(patch.sections) : existing.sections,
|
||||
riskClassification:
|
||||
patch.riskClassification !== undefined ? patch.riskClassification : existing.riskClassification,
|
||||
orgName: patch.orgName !== undefined ? patch.orgName : existing.orgName,
|
||||
footerText: patch.footerText !== undefined ? patch.footerText : existing.footerText,
|
||||
};
|
||||
if (!next.name) throw new Error('Template name is required');
|
||||
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(`
|
||||
UPDATE compliance_templates SET
|
||||
name = ?,
|
||||
description = ?,
|
||||
sections_json = ?,
|
||||
risk_classification = ?,
|
||||
org_name = ?,
|
||||
footer_text = ?,
|
||||
updated_at = datetime('now')
|
||||
WHERE id = ?
|
||||
`).run(
|
||||
next.name,
|
||||
next.description,
|
||||
JSON.stringify(next.sections),
|
||||
next.riskClassification,
|
||||
next.orgName,
|
||||
next.footerText,
|
||||
id,
|
||||
);
|
||||
return this.getById(id);
|
||||
}
|
||||
|
||||
/** Returns true if the row existed and was removed, false otherwise. */
|
||||
delete(id: number): boolean {
|
||||
const raw = this.db.getDatabase();
|
||||
const info = raw.prepare('DELETE FROM compliance_templates WHERE id = ?').run(id);
|
||||
return info.changes > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge a template's section flags with runtime flags. Union semantics: a section
|
||||
* is included if EITHER the template OR the runtime toggle turns it on. This keeps
|
||||
* the template's role "additive" — it can never silently hide a section the user
|
||||
* asked for in the current export.
|
||||
*/
|
||||
static mergeSections(
|
||||
template: ComplianceTemplateSections,
|
||||
runtime: ComplianceTemplateSections,
|
||||
): ComplianceTemplateSections {
|
||||
return {
|
||||
interactions: template.interactions || runtime.interactions,
|
||||
oversight: template.oversight || runtime.oversight,
|
||||
models: template.models || runtime.models,
|
||||
provenance: template.provenance || runtime.provenance,
|
||||
riskAssessment: template.riskAssessment || runtime.riskAssessment,
|
||||
fria: template.fria || runtime.fria,
|
||||
};
|
||||
}
|
||||
|
||||
private normalizeSections(s: Partial<ComplianceTemplateSections> | undefined): ComplianceTemplateSections {
|
||||
return { ...DEFAULT_SECTIONS, ...(s ?? {}) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Idempotent: seed the built-in "KVARK Enterprise Audit" template if no
|
||||
* template with that exact name exists yet (M-06).
|
||||
*
|
||||
* The KVARK template is a sensible-default shape for sovereign enterprise
|
||||
* deployments — every section on (including FRIA, since KVARK serves
|
||||
* high-risk enterprise systems), risk pinned at `high-risk`, and org/footer
|
||||
* text that signals the sovereign-deployment narrative.
|
||||
*
|
||||
* Custom KVARK-specific sections (per-department risk breakdown, IAM
|
||||
* audit columns, data-residency attestation text) are deferred to
|
||||
* Bucket 2 along with the logo field — the current template schema
|
||||
* doesn't support custom sections yet.
|
||||
*
|
||||
* Returns the seeded template, or null if one already existed.
|
||||
*/
|
||||
seedKvarkTemplateIfMissing(): ComplianceTemplate | null {
|
||||
const raw = this.db.getDatabase();
|
||||
const existing = raw.prepare(
|
||||
'SELECT id FROM compliance_templates WHERE name = ? LIMIT 1',
|
||||
).get(KVARK_TEMPLATE_NAME) as { id: number } | undefined;
|
||||
if (existing) return null;
|
||||
return this.create({
|
||||
name: KVARK_TEMPLATE_NAME,
|
||||
description:
|
||||
'Sovereign AI Act audit shape for enterprise on-prem deployments. ' +
|
||||
'Includes all monitored articles (12/14/19/26/50) plus FRIA since KVARK ' +
|
||||
'customers typically operate high-risk systems. Data never leaves the ' +
|
||||
"customer perimeter; this template's org + footer text signal that " +
|
||||
'posture directly on every exported report.',
|
||||
sections: { ...DEFAULT_SECTIONS, fria: true },
|
||||
riskClassification: 'high-risk',
|
||||
orgName: 'KVARK Sovereign — Enterprise Deployment',
|
||||
footerText:
|
||||
'Confidential · EU AI Act attestation · KVARK sovereign deployment · Data remains within customer perimeter',
|
||||
});
|
||||
}
|
||||
|
||||
private rowToTemplate(row: Record<string, unknown>): ComplianceTemplate {
|
||||
let sections: ComplianceTemplateSections;
|
||||
try {
|
||||
sections = this.normalizeSections(JSON.parse((row.sections_json as string) ?? '{}'));
|
||||
} catch {
|
||||
sections = { ...DEFAULT_SECTIONS };
|
||||
}
|
||||
return {
|
||||
id: row.id as number,
|
||||
name: row.name as string,
|
||||
description: (row.description as string | null) ?? null,
|
||||
sections,
|
||||
riskClassification: (row.risk_classification as AIActRiskLevel | null) ?? null,
|
||||
orgName: (row.org_name as string | null) ?? null,
|
||||
footerText: (row.footer_text as string | null) ?? null,
|
||||
createdAt: row.created_at as string,
|
||||
updatedAt: row.updated_at as string,
|
||||
};
|
||||
}
|
||||
}
|
||||
201
packages/core/src/compliance/types.ts
Normal file
201
packages/core/src/compliance/types.ts
Normal file
@@ -0,0 +1,201 @@
|
||||
/**
|
||||
* AI Act Compliance Types — interaction audit, risk classification, report format.
|
||||
*/
|
||||
|
||||
// ── Risk Classification (Art. 26) ──
|
||||
|
||||
export type AIActRiskLevel = 'minimal' | 'limited' | 'high-risk' | 'unacceptable';
|
||||
|
||||
export type HumanAction = 'approved' | 'denied' | 'modified' | 'none';
|
||||
|
||||
// ── Interaction Audit (Art. 12) ──
|
||||
|
||||
export interface AIInteraction {
|
||||
id: number;
|
||||
timestamp: string;
|
||||
workspaceId: string | null;
|
||||
sessionId: string | null;
|
||||
model: string;
|
||||
provider: string;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
costUsd: number;
|
||||
toolsCalled: string[];
|
||||
humanAction: HumanAction;
|
||||
riskContext: string | null;
|
||||
importedFrom: string | null;
|
||||
persona: string | null;
|
||||
// Review Critical #3 (compliance): EU AI Act Art. 12.1(a) requires recording the
|
||||
// actual inputs and outputs of the system, not just token counts. Nullable because
|
||||
// pre-existing DBs may have rows from before the columns were added.
|
||||
inputText: string | null;
|
||||
outputText: string | null;
|
||||
}
|
||||
|
||||
export interface RecordInteractionInput {
|
||||
workspaceId?: string;
|
||||
sessionId?: string;
|
||||
model: string;
|
||||
provider: string;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
costUsd: number;
|
||||
toolsCalled?: string[];
|
||||
humanAction?: HumanAction;
|
||||
riskContext?: string;
|
||||
importedFrom?: string;
|
||||
persona?: string;
|
||||
// Review Critical #3: optional because echo-mode interactions may not have meaningful
|
||||
// content. Callers on the live agent path SHOULD pass these; record() does not reject
|
||||
// absent values but the compliance status checker flags them as a gap.
|
||||
inputText?: string;
|
||||
outputText?: string;
|
||||
}
|
||||
|
||||
// ── Compliance Status ──
|
||||
|
||||
export interface ArticleStatus {
|
||||
status: 'compliant' | 'warning' | 'non-compliant';
|
||||
detail: string;
|
||||
}
|
||||
|
||||
export interface ComplianceStatus {
|
||||
overall: 'compliant' | 'warning' | 'non-compliant';
|
||||
art12Logging: ArticleStatus & { totalInteractions: number };
|
||||
art14Oversight: ArticleStatus & { humanActions: number; approvalRate: number };
|
||||
art19Retention: ArticleStatus & { oldestLogDate: string | null; retentionDays: number };
|
||||
art26Monitoring: ArticleStatus & { activeMonitors: string[] };
|
||||
art50Transparency: ArticleStatus & { modelsDisclosed: boolean };
|
||||
}
|
||||
|
||||
// ── Audit Report ──
|
||||
|
||||
export interface ModelInventoryEntry {
|
||||
model: string;
|
||||
provider: string;
|
||||
calls: number;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
costUsd: number;
|
||||
}
|
||||
|
||||
export interface OversightLogEntry {
|
||||
timestamp: string;
|
||||
action: HumanAction;
|
||||
tool: string;
|
||||
detail: string;
|
||||
user?: string;
|
||||
}
|
||||
|
||||
export interface HarvestProvenanceEntry {
|
||||
source: string;
|
||||
importedAt: string;
|
||||
itemsImported: number;
|
||||
framesCreated: number;
|
||||
}
|
||||
|
||||
export interface AuditReportRequest {
|
||||
workspaceId?: string;
|
||||
from: string;
|
||||
to: string;
|
||||
format: 'json' | 'pdf' | 'both';
|
||||
include: {
|
||||
interactions: boolean;
|
||||
oversight: boolean;
|
||||
models: boolean;
|
||||
provenance: boolean;
|
||||
riskAssessment: boolean;
|
||||
fria: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
export interface AuditReport {
|
||||
report: {
|
||||
version: string;
|
||||
generatedAt: string;
|
||||
period: { from: string; to: string };
|
||||
generatedBy: string;
|
||||
};
|
||||
workspace: {
|
||||
id: string | null;
|
||||
name: string;
|
||||
riskLevel: AIActRiskLevel;
|
||||
riskClassifiedAt: string | null;
|
||||
} | null;
|
||||
complianceStatus: ComplianceStatus;
|
||||
modelInventory: ModelInventoryEntry[];
|
||||
humanOversightLog: OversightLogEntry[];
|
||||
harvestProvenance: HarvestProvenanceEntry[];
|
||||
interactionCount: number;
|
||||
}
|
||||
|
||||
// ── Compliance Templates (M-03) ──
|
||||
//
|
||||
// Templates let a user save a preferred audit-report shape (sections + risk class +
|
||||
// org/footer overrides) and re-apply it to exports. Sections MERGE (union) with the
|
||||
// user's runtime section toggles — a template only turns sections ON, never off.
|
||||
// `riskClassification` overrides the workspace's risk level for that one report.
|
||||
// Logo support is deferred to Bucket 2 (post-launch).
|
||||
|
||||
export interface ComplianceTemplateSections {
|
||||
interactions: boolean;
|
||||
oversight: boolean;
|
||||
models: boolean;
|
||||
provenance: boolean;
|
||||
riskAssessment: boolean;
|
||||
fria: boolean;
|
||||
}
|
||||
|
||||
export interface ComplianceTemplate {
|
||||
id: number;
|
||||
name: string;
|
||||
description: string | null;
|
||||
sections: ComplianceTemplateSections;
|
||||
riskClassification: AIActRiskLevel | null;
|
||||
orgName: string | null;
|
||||
footerText: string | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface CreateComplianceTemplateInput {
|
||||
name: string;
|
||||
description?: string | null;
|
||||
sections: ComplianceTemplateSections;
|
||||
riskClassification?: AIActRiskLevel | null;
|
||||
orgName?: string | null;
|
||||
footerText?: string | null;
|
||||
}
|
||||
|
||||
export interface UpdateComplianceTemplateInput {
|
||||
name?: string;
|
||||
description?: string | null;
|
||||
sections?: ComplianceTemplateSections;
|
||||
riskClassification?: AIActRiskLevel | null;
|
||||
orgName?: string | null;
|
||||
footerText?: string | null;
|
||||
}
|
||||
|
||||
// ── Template auto-suggestion ──
|
||||
|
||||
export const TEMPLATE_RISK_MAP: Record<string, AIActRiskLevel> = {
|
||||
'legal-review': 'high-risk',
|
||||
'hr-management': 'high-risk',
|
||||
'recruiting': 'high-risk',
|
||||
'finance': 'high-risk',
|
||||
'insurance': 'high-risk',
|
||||
'credit-scoring': 'high-risk',
|
||||
'sales-pipeline': 'limited',
|
||||
'marketing-campaign': 'limited',
|
||||
'research-project': 'minimal',
|
||||
'code-review': 'minimal',
|
||||
'product-launch': 'limited',
|
||||
'agency-consulting': 'limited',
|
||||
'blank': 'minimal',
|
||||
'content-creation': 'minimal',
|
||||
'customer-support': 'limited',
|
||||
'data-analysis': 'limited',
|
||||
'project-management': 'minimal',
|
||||
'education': 'limited',
|
||||
'healthcare': 'high-risk',
|
||||
};
|
||||
281
packages/core/src/config.ts
Normal file
281
packages/core/src/config.ts
Normal file
@@ -0,0 +1,281 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import type { EmbeddingProviderConfig, EmbeddingProviderType } from '@waggle/hive-mind-core';
|
||||
|
||||
export interface ProviderEntry {
|
||||
apiKey: string;
|
||||
models: string[];
|
||||
baseUrl?: string;
|
||||
}
|
||||
|
||||
export interface TeamServerConfig {
|
||||
url: string;
|
||||
token?: string;
|
||||
userId?: string;
|
||||
displayName?: string;
|
||||
}
|
||||
|
||||
export interface CliConfig {
|
||||
allowlist?: string[];
|
||||
}
|
||||
|
||||
interface ConfigData {
|
||||
defaultModel: string;
|
||||
providers: Record<string, ProviderEntry>;
|
||||
mindPath?: string;
|
||||
teamServer?: TeamServerConfig;
|
||||
/** Governed CLI programs the agent may execute. */
|
||||
cli?: CliConfig;
|
||||
/** F8: Daily cost budget in dollars. null = no limit. */
|
||||
dailyBudget?: number | null;
|
||||
/** When true, exceeding dailyBudget blocks agent. When false, warns only. */
|
||||
budgetHardCap?: boolean;
|
||||
/** Model Pilot: fallback model when primary fails (429/500/timeout) */
|
||||
fallbackModel?: string;
|
||||
/** Model Pilot: budget-saver model when daily spend hits threshold */
|
||||
budgetModel?: string;
|
||||
/** Model Pilot: budget threshold as 0.0-1.0 fraction. Default 0.8 */
|
||||
budgetThreshold?: number;
|
||||
/** Agent Intelligence: max LLM iterations per conversation. Default 90. */
|
||||
maxIterations?: number;
|
||||
/** M2-7: Telemetry opt-in (default: false — privacy first) */
|
||||
telemetryEnabled?: boolean;
|
||||
/** M2-1: Embedding provider configuration */
|
||||
embedding?: {
|
||||
provider?: EmbeddingProviderType | 'auto';
|
||||
ollamaUrl?: string;
|
||||
ollamaModel?: string;
|
||||
inprocessModel?: string;
|
||||
};
|
||||
/** Steal #6: on-demand relevance gating for MCP tools. */
|
||||
mcpToolRetrieval?: {
|
||||
enabled?: boolean;
|
||||
threshold?: number;
|
||||
topK?: number;
|
||||
};
|
||||
}
|
||||
|
||||
/** Resolved MCP tool-retrieval config (all fields present). */
|
||||
export interface McpToolRetrievalSettings {
|
||||
enabled: boolean;
|
||||
threshold: number;
|
||||
topK: number;
|
||||
}
|
||||
|
||||
const DEFAULT_MODEL = 'claude-sonnet-4-6';
|
||||
|
||||
function getDefaultConfigDir(): string {
|
||||
const home = process.env.HOME || process.env.USERPROFILE || os.homedir();
|
||||
return path.join(home, '.waggle');
|
||||
}
|
||||
|
||||
export class WaggleConfig {
|
||||
private readonly configDir: string;
|
||||
private readonly configPath: string;
|
||||
private data: ConfigData;
|
||||
|
||||
constructor(configDir?: string) {
|
||||
this.configDir = configDir ?? getDefaultConfigDir();
|
||||
this.configPath = path.join(this.configDir, 'config.json');
|
||||
|
||||
// Ensure config directory exists
|
||||
if (!fs.existsSync(this.configDir)) {
|
||||
fs.mkdirSync(this.configDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Load existing config or use defaults
|
||||
this.data = this.load();
|
||||
}
|
||||
|
||||
private load(): ConfigData {
|
||||
if (fs.existsSync(this.configPath)) {
|
||||
const raw = fs.readFileSync(this.configPath, 'utf-8');
|
||||
return JSON.parse(raw) as ConfigData;
|
||||
}
|
||||
return {
|
||||
defaultModel: DEFAULT_MODEL,
|
||||
providers: {},
|
||||
};
|
||||
}
|
||||
|
||||
save(): void {
|
||||
fs.writeFileSync(this.configPath, JSON.stringify(this.data, null, 2), 'utf-8');
|
||||
}
|
||||
|
||||
getDefaultModel(): string {
|
||||
return this.data.defaultModel;
|
||||
}
|
||||
|
||||
setDefaultModel(model: string): void {
|
||||
this.data.defaultModel = model;
|
||||
}
|
||||
|
||||
getProviders(): Record<string, ProviderEntry> {
|
||||
return { ...(this.data.providers ?? {}) };
|
||||
}
|
||||
|
||||
setProvider(name: string, entry: ProviderEntry): void {
|
||||
if (!this.data.providers) this.data.providers = {};
|
||||
this.data.providers[name] = entry;
|
||||
}
|
||||
|
||||
removeProvider(name: string): void {
|
||||
delete this.data.providers[name];
|
||||
}
|
||||
|
||||
getMindPath(): string {
|
||||
return this.data.mindPath ?? path.join(this.configDir, 'default.mind');
|
||||
}
|
||||
|
||||
getConfigDir(): string {
|
||||
return this.configDir;
|
||||
}
|
||||
|
||||
// F8: Daily cost budget
|
||||
getDailyBudget(): number | null {
|
||||
return this.data.dailyBudget ?? null;
|
||||
}
|
||||
|
||||
setDailyBudget(budget: number | null): void {
|
||||
this.data.dailyBudget = budget;
|
||||
}
|
||||
|
||||
getBudgetHardCap(): boolean {
|
||||
return this.data.budgetHardCap ?? false;
|
||||
}
|
||||
|
||||
setBudgetHardCap(enabled: boolean): void {
|
||||
this.data.budgetHardCap = enabled;
|
||||
}
|
||||
|
||||
// --- Model Pilot ---
|
||||
|
||||
getFallbackModel(): string | null {
|
||||
return this.data.fallbackModel ?? null;
|
||||
}
|
||||
|
||||
setFallbackModel(model: string): void {
|
||||
this.data.fallbackModel = model;
|
||||
}
|
||||
|
||||
clearFallbackModel(): void {
|
||||
delete this.data.fallbackModel;
|
||||
}
|
||||
|
||||
getBudgetModel(): string | null {
|
||||
return this.data.budgetModel ?? null;
|
||||
}
|
||||
|
||||
setBudgetModel(model: string): void {
|
||||
this.data.budgetModel = model;
|
||||
}
|
||||
|
||||
clearBudgetModel(): void {
|
||||
delete this.data.budgetModel;
|
||||
}
|
||||
|
||||
getBudgetThreshold(): number {
|
||||
return this.data.budgetThreshold ?? 0.8;
|
||||
}
|
||||
|
||||
setBudgetThreshold(threshold: number): void {
|
||||
this.data.budgetThreshold = Math.max(0.5, Math.min(0.95, threshold));
|
||||
}
|
||||
|
||||
// --- Agent Intelligence ---
|
||||
|
||||
getMaxIterations(): number {
|
||||
return this.data.maxIterations ?? 90;
|
||||
}
|
||||
|
||||
setMaxIterations(max: number): void {
|
||||
this.data.maxIterations = Math.max(5, Math.min(500, max));
|
||||
}
|
||||
|
||||
// --- Team Server (Phase 5) ---
|
||||
|
||||
getTeamServer(): TeamServerConfig | null {
|
||||
return this.data.teamServer ?? null;
|
||||
}
|
||||
|
||||
setTeamServer(config: TeamServerConfig): void {
|
||||
this.data.teamServer = config;
|
||||
}
|
||||
|
||||
clearTeamServer(): void {
|
||||
delete this.data.teamServer;
|
||||
}
|
||||
|
||||
isTeamConnected(): boolean {
|
||||
return this.data.teamServer !== null && this.data.teamServer !== undefined && typeof this.data.teamServer.url === 'string' && this.data.teamServer.url.length > 0;
|
||||
}
|
||||
|
||||
// --- Governed CLI access ---
|
||||
|
||||
getCliAllowlist(): string[] {
|
||||
return [...(this.data.cli?.allowlist ?? [])];
|
||||
}
|
||||
|
||||
setCliAllowlist(allowlist: string[]): void {
|
||||
const seen = new Set<string>();
|
||||
const next = allowlist.reduce<string[]>((result, entry) => {
|
||||
const value = entry.trim();
|
||||
const key = value.toLowerCase();
|
||||
if (value && !seen.has(key)) {
|
||||
seen.add(key);
|
||||
result.push(value);
|
||||
}
|
||||
return result;
|
||||
}, []);
|
||||
this.data.cli = { ...(this.data.cli ?? {}), allowlist: next };
|
||||
}
|
||||
|
||||
// --- Telemetry (M2-7) ---
|
||||
|
||||
getTelemetryEnabled(): boolean {
|
||||
return this.data.telemetryEnabled ?? false;
|
||||
}
|
||||
|
||||
setTelemetryEnabled(enabled: boolean): void {
|
||||
this.data.telemetryEnabled = enabled;
|
||||
this.save();
|
||||
}
|
||||
|
||||
// --- Embedding Provider (M2-1) ---
|
||||
|
||||
getEmbeddingConfig(): EmbeddingProviderConfig {
|
||||
const emb = this.data.embedding;
|
||||
const config: EmbeddingProviderConfig = {
|
||||
provider: (process.env.EMBEDDING_PROVIDER as EmbeddingProviderType | 'auto' | undefined) ?? emb?.provider ?? 'auto',
|
||||
targetDimensions: 1024,
|
||||
inprocess: {
|
||||
model: process.env.EMBEDDING_MODEL ?? emb?.inprocessModel,
|
||||
cacheDir: path.join(this.configDir, 'models'),
|
||||
},
|
||||
ollama: {
|
||||
baseUrl: process.env.OLLAMA_HOST ?? emb?.ollamaUrl,
|
||||
model: process.env.OLLAMA_EMBED_MODEL ?? emb?.ollamaModel,
|
||||
},
|
||||
// API keys injected separately from Vault — not stored in config.json
|
||||
};
|
||||
return config;
|
||||
}
|
||||
|
||||
setEmbeddingProvider(provider: EmbeddingProviderType | 'auto'): void {
|
||||
if (!this.data.embedding) this.data.embedding = {};
|
||||
this.data.embedding.provider = provider;
|
||||
}
|
||||
|
||||
// --- MCP tool retrieval (Steal #6) ---
|
||||
|
||||
/** Resolve MCP tool-retrieval settings, filling defaults (ON, threshold 20, top-k 10). */
|
||||
getMcpToolRetrieval(): McpToolRetrievalSettings {
|
||||
const cfg = this.data.mcpToolRetrieval;
|
||||
return {
|
||||
enabled: cfg?.enabled ?? true,
|
||||
threshold: cfg?.threshold ?? 20,
|
||||
topK: cfg?.topK ?? 10,
|
||||
};
|
||||
}
|
||||
}
|
||||
588
packages/core/src/cron-store.ts
Normal file
588
packages/core/src/cron-store.ts
Normal file
@@ -0,0 +1,588 @@
|
||||
/**
|
||||
* Cron Store — SQLite persistence for solo cron schedules.
|
||||
*
|
||||
* Stores cron job definitions with expression validation via cron-parser,
|
||||
* due-job queries, and run tracking. Follows the same pattern as
|
||||
* InstallAuditStore — operates on the .mind DB with lazy table creation.
|
||||
*/
|
||||
|
||||
import cronParser from 'cron-parser';
|
||||
const { parseExpression } = cronParser;
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
export type CronJobType = 'agent_task' | 'memory_consolidation' | 'workspace_health' | 'proactive' | 'prompt_optimization' | 'monthly_assessment' | 'connector_fetch' | 'loop';
|
||||
|
||||
export const VALID_JOB_TYPES: Set<string> = new Set([
|
||||
'agent_task',
|
||||
'memory_consolidation',
|
||||
'workspace_health',
|
||||
'proactive',
|
||||
'prompt_optimization',
|
||||
'monthly_assessment',
|
||||
'connector_fetch',
|
||||
// Loop v0: a stateful, memory-powered, report-only (L1) scheduled automation.
|
||||
// Composes recall + maker (toolless LLM) + checker (LLMJudge) + memory write.
|
||||
// job_type TEXT has no CHECK constraint, so this is additive — no migration.
|
||||
'loop',
|
||||
]);
|
||||
|
||||
export interface CronSchedule {
|
||||
id: number;
|
||||
name: string;
|
||||
cron_expr: string;
|
||||
job_type: CronJobType;
|
||||
job_config: string;
|
||||
workspace_id: string | null;
|
||||
enabled: number; // SQLite integer boolean
|
||||
last_run_at: string | null;
|
||||
next_run_at: string | null;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export interface CreateScheduleInput {
|
||||
name: string;
|
||||
cronExpr: string;
|
||||
jobType: CronJobType;
|
||||
jobConfig?: Record<string, unknown>;
|
||||
workspaceId?: string;
|
||||
enabled?: boolean;
|
||||
}
|
||||
|
||||
export interface CronExecutionRow {
|
||||
id: number;
|
||||
schedule_id: number;
|
||||
schedule_name: string;
|
||||
executed_at: string;
|
||||
duration_ms: number | null;
|
||||
success: number; // SQLite integer boolean
|
||||
result_summary: string | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export interface CronRunLeaseRow {
|
||||
id: number;
|
||||
schedule_id: number;
|
||||
schedule_name: string | null;
|
||||
started_at: string;
|
||||
pid: number | null;
|
||||
}
|
||||
|
||||
export interface NotificationRow {
|
||||
id: number;
|
||||
title: string;
|
||||
body: string;
|
||||
category: string;
|
||||
action_url: string | null;
|
||||
read: number; // SQLite integer boolean
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
// ── L2 assisted loops: durable "held action" approval queue ──────────────
|
||||
// A held action is a self-contained proposed tool call drafted by a headless
|
||||
// run (e.g. an assist-mode Loop) that needs a human's one-click approval before
|
||||
// it executes. Durable so it survives a sidecar restart (the in-memory live
|
||||
// approval Promise does not). Execute-on-approve, never mid-run suspend/resume.
|
||||
|
||||
export type PendingActionStatus = 'held' | 'approved' | 'denied' | 'executed' | 'failed' | 'expired';
|
||||
|
||||
export interface PendingActionRow {
|
||||
id: string; // requestId (uuid)
|
||||
workspace_id: string | null;
|
||||
source: string; // e.g. 'loop:<scheduleId>'
|
||||
tool_name: string;
|
||||
args_json: string;
|
||||
summary: string | null; // the maker's plain-language rationale
|
||||
risk_level: string; // low | medium | high | critical
|
||||
approval_class: string;
|
||||
status: PendingActionStatus;
|
||||
result_summary: string | null;
|
||||
error: string | null;
|
||||
created_at: string;
|
||||
decided_at: string | null;
|
||||
executed_at: string | null;
|
||||
expires_at: string | null;
|
||||
}
|
||||
|
||||
export interface SavePendingActionInput {
|
||||
id: string;
|
||||
workspaceId: string | null;
|
||||
source: string;
|
||||
toolName: string;
|
||||
argsJson: string;
|
||||
summary?: string;
|
||||
riskLevel: string;
|
||||
approvalClass: string;
|
||||
expiresAt?: string;
|
||||
}
|
||||
|
||||
// ── Table DDL ──────────────────────────────────────────────────────────
|
||||
|
||||
export const CRON_SCHEDULES_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS cron_schedules (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
cron_expr TEXT NOT NULL,
|
||||
job_type TEXT NOT NULL,
|
||||
job_config TEXT NOT NULL DEFAULT '{}',
|
||||
workspace_id TEXT,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
last_run_at TEXT,
|
||||
next_run_at TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_cron_enabled_next ON cron_schedules (enabled, next_run_at);
|
||||
`;
|
||||
|
||||
// W5.12: Cron execution history table
|
||||
export const CRON_HISTORY_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS cron_execution_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
schedule_id INTEGER NOT NULL,
|
||||
schedule_name TEXT NOT NULL,
|
||||
executed_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
duration_ms INTEGER,
|
||||
success INTEGER NOT NULL DEFAULT 1,
|
||||
result_summary TEXT,
|
||||
error TEXT,
|
||||
FOREIGN KEY (schedule_id) REFERENCES cron_schedules(id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_cron_history_schedule ON cron_execution_history (schedule_id, executed_at);
|
||||
`;
|
||||
|
||||
export const CRON_RUN_LEASES_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS cron_run_leases (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
schedule_id INTEGER NOT NULL,
|
||||
schedule_name TEXT,
|
||||
started_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
pid INTEGER
|
||||
);
|
||||
`;
|
||||
|
||||
// W5.10: Notification persistence table
|
||||
export const NOTIFICATIONS_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS notifications (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
title TEXT NOT NULL,
|
||||
body TEXT NOT NULL,
|
||||
category TEXT NOT NULL DEFAULT 'system',
|
||||
action_url TEXT,
|
||||
read INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_notifications_created ON notifications (created_at);
|
||||
`;
|
||||
|
||||
// L2: durable held-action approval queue
|
||||
export const PENDING_ACTIONS_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS pending_actions (
|
||||
id TEXT PRIMARY KEY,
|
||||
workspace_id TEXT,
|
||||
source TEXT NOT NULL,
|
||||
tool_name TEXT NOT NULL,
|
||||
args_json TEXT NOT NULL,
|
||||
summary TEXT,
|
||||
risk_level TEXT NOT NULL,
|
||||
approval_class TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'held',
|
||||
result_summary TEXT,
|
||||
error TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
decided_at TEXT,
|
||||
executed_at TEXT,
|
||||
expires_at TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_pending_actions_status ON pending_actions (status, created_at);
|
||||
`;
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
/** Parse a cron expression and return the next run time as ISO string. Throws on invalid expr. */
|
||||
function computeNextRun(cronExpr: string): string {
|
||||
const interval = parseExpression(cronExpr);
|
||||
return interval.next().toISOString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a cron expression with the SAME parser create()/update() use.
|
||||
* Returns null when parseable, else the parser's error message. Lets route
|
||||
* layers (e.g. the automations /test preview) reject an expression the store
|
||||
* would refuse to persist, without duplicating the parser dependency.
|
||||
*/
|
||||
export function cronExprError(cronExpr: string): string | null {
|
||||
try {
|
||||
parseExpression(cronExpr);
|
||||
return null;
|
||||
} catch (err) {
|
||||
return err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Store ──────────────────────────────────────────────────────────────
|
||||
|
||||
export class CronStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
const exists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='cron_schedules'",
|
||||
).get();
|
||||
if (!exists) {
|
||||
raw.exec(CRON_SCHEDULES_TABLE_SQL);
|
||||
}
|
||||
// W5.12: Ensure cron execution history table
|
||||
const histExists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='cron_execution_history'",
|
||||
).get();
|
||||
if (!histExists) {
|
||||
raw.exec(CRON_HISTORY_TABLE_SQL);
|
||||
}
|
||||
const leaseExists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='cron_run_leases'",
|
||||
).get();
|
||||
if (!leaseExists) {
|
||||
raw.exec(CRON_RUN_LEASES_TABLE_SQL);
|
||||
}
|
||||
// W5.10: Ensure notifications table
|
||||
const notifExists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='notifications'",
|
||||
).get();
|
||||
if (!notifExists) {
|
||||
raw.exec(NOTIFICATIONS_TABLE_SQL);
|
||||
}
|
||||
// L2: Ensure pending_actions (held-action approval queue) table
|
||||
const pendingExists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='pending_actions'",
|
||||
).get();
|
||||
if (!pendingExists) {
|
||||
raw.exec(PENDING_ACTIONS_TABLE_SQL);
|
||||
}
|
||||
}
|
||||
|
||||
/** Create a new cron schedule. Validates cron expression and job type. */
|
||||
create(input: CreateScheduleInput): CronSchedule {
|
||||
// Validate job type
|
||||
if (!VALID_JOB_TYPES.has(input.jobType)) {
|
||||
throw new Error(`Invalid job type: "${input.jobType}". Must be one of: ${[...VALID_JOB_TYPES].join(', ')}`);
|
||||
}
|
||||
|
||||
// agent_task requires workspaceId
|
||||
if (input.jobType === 'agent_task' && !input.workspaceId) {
|
||||
throw new Error('agent_task jobs require a workspace ID');
|
||||
}
|
||||
|
||||
// Validate cron expression (throws on invalid)
|
||||
const nextRun = computeNextRun(input.cronExpr);
|
||||
|
||||
const raw = this.db.getDatabase();
|
||||
const enabled = input.enabled === false ? 0 : 1;
|
||||
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO cron_schedules (name, cron_expr, job_type, job_config, workspace_id, enabled, next_run_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
input.name,
|
||||
input.cronExpr,
|
||||
input.jobType,
|
||||
JSON.stringify(input.jobConfig ?? {}),
|
||||
input.workspaceId ?? null,
|
||||
enabled,
|
||||
nextRun,
|
||||
);
|
||||
|
||||
return raw.prepare(
|
||||
'SELECT * FROM cron_schedules WHERE id = ?',
|
||||
).get(result.lastInsertRowid) as CronSchedule;
|
||||
}
|
||||
|
||||
/** List all schedules ordered by name. */
|
||||
list(): CronSchedule[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM cron_schedules ORDER BY name ASC',
|
||||
).all() as CronSchedule[];
|
||||
}
|
||||
|
||||
/** Get a schedule by ID. Returns undefined if not found. */
|
||||
getById(id: number): CronSchedule | undefined {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM cron_schedules WHERE id = ?',
|
||||
).get(id) as CronSchedule | undefined;
|
||||
}
|
||||
|
||||
/** Update a schedule. Recomputes next_run_at if cronExpr changes. Returns updated schedule. */
|
||||
update(id: number, changes: Partial<Pick<CreateScheduleInput, 'name' | 'cronExpr' | 'jobConfig' | 'workspaceId' | 'enabled'>>): CronSchedule {
|
||||
const setClauses: string[] = [];
|
||||
const values: unknown[] = [];
|
||||
|
||||
if (changes.name !== undefined) {
|
||||
setClauses.push('name = ?');
|
||||
values.push(changes.name);
|
||||
}
|
||||
if (changes.cronExpr !== undefined) {
|
||||
const nextRun = computeNextRun(changes.cronExpr);
|
||||
setClauses.push('cron_expr = ?');
|
||||
values.push(changes.cronExpr);
|
||||
setClauses.push('next_run_at = ?');
|
||||
values.push(nextRun);
|
||||
}
|
||||
if (changes.jobConfig !== undefined) {
|
||||
setClauses.push('job_config = ?');
|
||||
values.push(JSON.stringify(changes.jobConfig));
|
||||
}
|
||||
if (changes.workspaceId !== undefined) {
|
||||
setClauses.push('workspace_id = ?');
|
||||
values.push(changes.workspaceId);
|
||||
}
|
||||
if (changes.enabled !== undefined) {
|
||||
setClauses.push('enabled = ?');
|
||||
values.push(changes.enabled ? 1 : 0);
|
||||
}
|
||||
|
||||
if (setClauses.length > 0) {
|
||||
values.push(id);
|
||||
this.db.getDatabase().prepare(
|
||||
`UPDATE cron_schedules SET ${setClauses.join(', ')} WHERE id = ?`,
|
||||
).run(...values);
|
||||
}
|
||||
|
||||
return this.getById(id)!;
|
||||
}
|
||||
|
||||
/** Delete a schedule by ID. */
|
||||
delete(id: number): void {
|
||||
this.db.getDatabase().prepare(
|
||||
'DELETE FROM cron_schedules WHERE id = ?',
|
||||
).run(id);
|
||||
}
|
||||
|
||||
/** Get all enabled schedules whose next_run_at is in the past. */
|
||||
getDue(): CronSchedule[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
"SELECT * FROM cron_schedules WHERE enabled = 1 AND next_run_at <= datetime('now')",
|
||||
).all() as CronSchedule[];
|
||||
}
|
||||
|
||||
/** Mark a schedule as having just run. Updates last_run_at and recomputes next_run_at. */
|
||||
markRun(id: number): void {
|
||||
const schedule = this.getById(id);
|
||||
if (!schedule) return;
|
||||
|
||||
const nextRun = computeNextRun(schedule.cron_expr);
|
||||
this.db.getDatabase().prepare(
|
||||
"UPDATE cron_schedules SET last_run_at = datetime('now'), next_run_at = ? WHERE id = ?",
|
||||
).run(nextRun, id);
|
||||
}
|
||||
|
||||
/** Clear all schedules (for testing). */
|
||||
clear(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM cron_schedules').run();
|
||||
}
|
||||
|
||||
// ── W5.12: Cron Execution History ──────────────────────────────────
|
||||
|
||||
/** Record a cron job execution result. */
|
||||
recordExecution(scheduleId: number, scheduleName: string, opts: {
|
||||
executedAt?: string;
|
||||
durationMs?: number;
|
||||
success: boolean;
|
||||
resultSummary?: string;
|
||||
error?: string;
|
||||
}): void {
|
||||
this.db.getDatabase().prepare(
|
||||
`INSERT INTO cron_execution_history (schedule_id, schedule_name, executed_at, duration_ms, success, result_summary, error)
|
||||
VALUES (?, ?, COALESCE(?, datetime('now')), ?, ?, ?, ?)`,
|
||||
).run(scheduleId, scheduleName, opts.executedAt ?? null, opts.durationMs ?? null, opts.success ? 1 : 0, opts.resultSummary ?? null, opts.error ?? null);
|
||||
}
|
||||
|
||||
/** #17: count today's (UTC) executions for a schedule — ai_task daily cap. */
|
||||
countExecutionsToday(scheduleId: number): number {
|
||||
const row = this.db.getDatabase().prepare(
|
||||
"SELECT COUNT(*) AS n FROM cron_execution_history WHERE schedule_id = ? AND executed_at >= date('now')",
|
||||
).get(scheduleId) as { n: number };
|
||||
return row.n;
|
||||
}
|
||||
|
||||
/** Get execution history for a schedule (most recent first). */
|
||||
getExecutionHistory(scheduleId: number, limit = 20): CronExecutionRow[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM cron_execution_history WHERE schedule_id = ? ORDER BY executed_at DESC LIMIT ?',
|
||||
).all(scheduleId, limit) as CronExecutionRow[];
|
||||
}
|
||||
|
||||
/** Get the most recent executions for boot-time failure-state recovery. */
|
||||
getRecentExecutions(scheduleId: number, limit = 5): CronExecutionRow[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM cron_execution_history WHERE schedule_id = ? ORDER BY executed_at DESC, id DESC LIMIT ?',
|
||||
).all(scheduleId, limit) as CronExecutionRow[];
|
||||
}
|
||||
|
||||
/** Prune execution-history rows older than N days. recordExecution writes a
|
||||
* row per tick (UX-Refactor Phase 3, Journey 16), so without retention the
|
||||
* table grows unbounded (a per-minute job ≈ 525k rows/year). Mirrors
|
||||
* optStore.pruneOlderThan(30). Returns the number of rows deleted. */
|
||||
pruneExecutionHistory(olderThanDays: number): number {
|
||||
const days = Math.max(1, Math.floor(olderThanDays));
|
||||
const result = this.db.getDatabase().prepare(
|
||||
"DELETE FROM cron_execution_history WHERE executed_at < datetime('now', ?)",
|
||||
).run(`-${days} days`);
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
// ── Interrupted-run leases ────────────────────────────────────────
|
||||
|
||||
/** Acquire a durable lease before a scheduled job begins execution. */
|
||||
acquireRunLease(scheduleId: number, name: string, pid: number): number {
|
||||
const result = this.db.getDatabase().prepare(
|
||||
'INSERT INTO cron_run_leases (schedule_id, schedule_name, pid) VALUES (?, ?, ?)',
|
||||
).run(scheduleId, name, pid);
|
||||
return Number(result.lastInsertRowid);
|
||||
}
|
||||
|
||||
/** Release a run lease after its scheduled job finishes. */
|
||||
releaseRunLease(leaseId: number): void {
|
||||
this.db.getDatabase().prepare(
|
||||
'DELETE FROM cron_run_leases WHERE id = ?',
|
||||
).run(leaseId);
|
||||
}
|
||||
|
||||
/** List leases left behind by an interrupted process. */
|
||||
listStaleRunLeases(): CronRunLeaseRow[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM cron_run_leases ORDER BY started_at ASC, id ASC',
|
||||
).all() as CronRunLeaseRow[];
|
||||
}
|
||||
|
||||
/** Clear all interrupted-run leases after boot recovery. */
|
||||
clearRunLeases(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM cron_run_leases').run();
|
||||
}
|
||||
|
||||
// ── W5.10: Notification Persistence ────────────────────────────────
|
||||
|
||||
/** Save a notification. */
|
||||
saveNotification(title: string, body: string, category = 'system', actionUrl?: string): number {
|
||||
const result = this.db.getDatabase().prepare(
|
||||
'INSERT INTO notifications (title, body, category, action_url) VALUES (?, ?, ?, ?)',
|
||||
).run(title, body, category, actionUrl ?? null);
|
||||
return Number(result.lastInsertRowid);
|
||||
}
|
||||
|
||||
/** Get recent notifications (newest first). */
|
||||
getNotifications(opts?: { since?: string; limit?: number; unreadOnly?: boolean }): NotificationRow[] {
|
||||
const limit = opts?.limit ?? 50;
|
||||
let sql = 'SELECT * FROM notifications';
|
||||
const conditions: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
if (opts?.since) { conditions.push('created_at > ?'); params.push(opts.since); }
|
||||
if (opts?.unreadOnly) { conditions.push('read = 0'); }
|
||||
if (conditions.length > 0) sql += ' WHERE ' + conditions.join(' AND ');
|
||||
sql += ' ORDER BY created_at DESC LIMIT ?';
|
||||
params.push(limit);
|
||||
return this.db.getDatabase().prepare(sql).all(...params) as NotificationRow[];
|
||||
}
|
||||
|
||||
/** Mark a notification as read. */
|
||||
markNotificationRead(id: number): void {
|
||||
this.db.getDatabase().prepare('UPDATE notifications SET read = 1 WHERE id = ?').run(id);
|
||||
}
|
||||
|
||||
/** Mark all notifications as read. Returns the number of rows updated. */
|
||||
markAllRead(): number {
|
||||
const result = this.db.getDatabase().prepare('UPDATE notifications SET read = 1 WHERE read = 0').run();
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
/** Count unread notifications. */
|
||||
countUnread(): number {
|
||||
const row = this.db.getDatabase().prepare('SELECT COUNT(*) as cnt FROM notifications WHERE read = 0').get() as { cnt: number };
|
||||
return row.cnt;
|
||||
}
|
||||
|
||||
// ── L2: Held-action approval queue ─────────────────────────────────
|
||||
|
||||
/** Enqueue a held action (status 'held'). Returns the persisted row. */
|
||||
savePendingAction(input: SavePendingActionInput): PendingActionRow {
|
||||
this.db.getDatabase().prepare(`
|
||||
INSERT INTO pending_actions (id, workspace_id, source, tool_name, args_json, summary, risk_level, approval_class, status, expires_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'held', ?)
|
||||
`).run(
|
||||
input.id,
|
||||
input.workspaceId,
|
||||
input.source,
|
||||
input.toolName,
|
||||
input.argsJson,
|
||||
input.summary ?? null,
|
||||
input.riskLevel,
|
||||
input.approvalClass,
|
||||
input.expiresAt ?? null,
|
||||
);
|
||||
return this.getPendingAction(input.id)!;
|
||||
}
|
||||
|
||||
/** List held actions (newest first) — defaults to the 'held' queue. */
|
||||
listPendingActions(status: PendingActionStatus = 'held'): PendingActionRow[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM pending_actions WHERE status = ? ORDER BY created_at DESC',
|
||||
).all(status) as PendingActionRow[];
|
||||
}
|
||||
|
||||
/** Get one held action by id. */
|
||||
getPendingAction(id: string): PendingActionRow | undefined {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM pending_actions WHERE id = ?',
|
||||
).get(id) as PendingActionRow | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomically claim a held action — transition 'held' → 'approved' | 'denied'.
|
||||
* This is the idempotency gate: exactly ONE caller wins (the `WHERE status =
|
||||
* 'held'` makes it atomic in SQLite), so a double-approve / approve-after-deny
|
||||
* is a no-op. Returns the updated row if THIS call won the claim, else
|
||||
* undefined (already decided or missing).
|
||||
*/
|
||||
claimPendingAction(id: string, status: 'approved' | 'denied', decidedAt: string): PendingActionRow | undefined {
|
||||
const result = this.db.getDatabase().prepare(
|
||||
"UPDATE pending_actions SET status = ?, decided_at = ? WHERE id = ? AND status = 'held'",
|
||||
).run(status, decidedAt, id);
|
||||
if (result.changes === 0) return undefined;
|
||||
return this.getPendingAction(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Record the terminal outcome of a claimed action ('approved' → 'executed' |
|
||||
* 'failed'). Unconditional — the caller already won claimPendingAction(), so
|
||||
* it owns the row and no further guard is needed.
|
||||
*/
|
||||
updatePendingActionResult(id: string, changes: { status: 'executed' | 'failed'; resultSummary?: string; error?: string; executedAt?: string }): void {
|
||||
const sets: string[] = ['status = ?'];
|
||||
const vals: unknown[] = [changes.status];
|
||||
if (changes.resultSummary !== undefined) { sets.push('result_summary = ?'); vals.push(changes.resultSummary); }
|
||||
if (changes.error !== undefined) { sets.push('error = ?'); vals.push(changes.error); }
|
||||
if (changes.executedAt !== undefined) { sets.push('executed_at = ?'); vals.push(changes.executedAt); }
|
||||
vals.push(id);
|
||||
this.db.getDatabase().prepare(
|
||||
`UPDATE pending_actions SET ${sets.join(', ')} WHERE id = ?`,
|
||||
).run(...vals);
|
||||
}
|
||||
|
||||
/** Flip held actions past their expires_at to 'expired'. Returns rows changed. */
|
||||
expireStalePendingActions(): number {
|
||||
const result = this.db.getDatabase().prepare(
|
||||
"UPDATE pending_actions SET status = 'expired' WHERE status = 'held' AND expires_at IS NOT NULL AND expires_at < datetime('now')",
|
||||
).run();
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
/** Clear all pending actions (for testing). */
|
||||
clearPendingActions(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM pending_actions').run();
|
||||
}
|
||||
}
|
||||
259
packages/core/src/file-indexer.ts
Normal file
259
packages/core/src/file-indexer.ts
Normal file
@@ -0,0 +1,259 @@
|
||||
/**
|
||||
* FileIndexer — auto-index workspace files into their workspace mind (L-20).
|
||||
*
|
||||
* Design decisions (see docs/plans/FILE-TOOLS-AUDIT-2026-04-20.md):
|
||||
* • Trigger: on-upload / on-overwrite (called from route handlers)
|
||||
* • Target mind: per-workspace (the mind you pass in the constructor)
|
||||
* • Granularity: 1 frame per file
|
||||
* • Formats: .md, .markdown, .txt only for v1 (Bucket 2 adds PDF/DOCX)
|
||||
* • Cleanup: re-index on overwrite, delete index on file remove
|
||||
*
|
||||
* A `file_index` table tracks the file_path → frame_id link so overwrites can
|
||||
* swap the old frame cleanly and moves/deletes can locate the right frame
|
||||
* without scanning every frame in the workspace.
|
||||
*
|
||||
* Large files are truncated to MAX_CONTENT_BYTES with a marker appended; the
|
||||
* full file is still on disk, only the indexed frame is shortened. Keeps the
|
||||
* mind DB size bounded even if a user drops a 5MB markdown dump.
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
import { FrameStore } from '@waggle/hive-mind-core';
|
||||
import { SessionStore } from '@waggle/hive-mind-core';
|
||||
|
||||
const FILE_INDEX_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS file_index (
|
||||
file_path TEXT PRIMARY KEY,
|
||||
frame_id INTEGER NOT NULL,
|
||||
mime_type TEXT,
|
||||
size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
content_hash TEXT NOT NULL,
|
||||
indexed_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
)
|
||||
`;
|
||||
const FILE_INDEX_IDX_SQL = `
|
||||
CREATE INDEX IF NOT EXISTS idx_file_index_frame ON file_index (frame_id)
|
||||
`;
|
||||
|
||||
const INDEX_SESSION_GOP_ID = 'session:file-index';
|
||||
const INDEX_SESSION_PROJECT = 'file-indexer';
|
||||
|
||||
/** Maximum bytes of file content we store in a single frame. Beyond this, the
|
||||
* content is truncated with a marker. Keeps mind size bounded on large dumps. */
|
||||
export const MAX_CONTENT_BYTES = 64 * 1024;
|
||||
|
||||
/** Extensions we index for v1. PDF/DOCX/XLSX land in Bucket 2. */
|
||||
const INDEXABLE_EXTENSIONS = new Set(['.md', '.markdown', '.txt']);
|
||||
|
||||
export interface FileIndexRow {
|
||||
filePath: string;
|
||||
frameId: number;
|
||||
mimeType: string | null;
|
||||
sizeBytes: number;
|
||||
contentHash: string;
|
||||
indexedAt: string;
|
||||
}
|
||||
|
||||
export type FileIndexResult =
|
||||
| { skipped: true; reason: 'unsupported_format' | 'unchanged' | 'empty' }
|
||||
| { skipped: false; frameId: number; truncated: boolean };
|
||||
|
||||
export class FileIndexer {
|
||||
private readonly db: MindDB;
|
||||
private readonly frames: FrameStore;
|
||||
private readonly sessions: SessionStore;
|
||||
private cachedGopId: string | null = null;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.frames = new FrameStore(db);
|
||||
this.sessions = new SessionStore(db);
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
const existsRow = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='file_index'",
|
||||
).get();
|
||||
if (existsRow) return;
|
||||
raw.prepare(FILE_INDEX_TABLE_SQL).run();
|
||||
raw.prepare(FILE_INDEX_IDX_SQL).run();
|
||||
}
|
||||
|
||||
/** True iff the path's extension is one we index in v1. */
|
||||
static shouldIndex(filePath: string): boolean {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
return INDEXABLE_EXTENSIONS.has(ext);
|
||||
}
|
||||
|
||||
private ensureSession(): string {
|
||||
if (this.cachedGopId) return this.cachedGopId;
|
||||
const session = this.sessions.ensure(
|
||||
INDEX_SESSION_GOP_ID,
|
||||
INDEX_SESSION_PROJECT,
|
||||
'Auto-indexed workspace files',
|
||||
);
|
||||
this.cachedGopId = session.gop_id;
|
||||
return session.gop_id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Index a file. Idempotent on unchanged content (same hash → skip). On
|
||||
* overwrite with new content, deletes the old frame and inserts a fresh one.
|
||||
*/
|
||||
indexFile(filePath: string, content: Buffer, mime?: string): FileIndexResult {
|
||||
if (!FileIndexer.shouldIndex(filePath)) {
|
||||
return { skipped: true, reason: 'unsupported_format' };
|
||||
}
|
||||
if (content.length === 0) {
|
||||
// Still remove any stale index so an emptied file doesn't leave an
|
||||
// orphan frame pointing at old content.
|
||||
this.removeFile(filePath);
|
||||
return { skipped: true, reason: 'empty' };
|
||||
}
|
||||
|
||||
const hash = createHash('sha256').update(content).digest('hex');
|
||||
const raw = this.db.getDatabase();
|
||||
const existingRow = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(filePath) as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
|
||||
if (existingRow && existingRow.content_hash === hash) {
|
||||
return { skipped: true, reason: 'unchanged' };
|
||||
}
|
||||
|
||||
const normalized = this.normalizeText(content);
|
||||
const truncated = content.length > MAX_CONTENT_BYTES;
|
||||
const frameBody = this.buildFrameContent(filePath, normalized, truncated, mime);
|
||||
const gopId = this.ensureSession();
|
||||
// Atomicity contract (L-20 BLOCKER-1 fix): createIFrame → (conditional
|
||||
// old-frame delete) → UPDATE/INSERT on file_index must commit as one unit.
|
||||
// If any step throws mid-callback, better-sqlite3 rolls back the new frame
|
||||
// and the index-row write, leaving the table in its pre-call state. Without
|
||||
// this, a crash between the delete and the UPDATE would leave file_index
|
||||
// pointing at a deleted frame_id (dangling) or an uncommitted new frame
|
||||
// orphaned in the frames table.
|
||||
const mutate = raw.transaction(() => {
|
||||
const frame = this.frames.createIFrame(gopId, frameBody, 'normal', 'system');
|
||||
|
||||
if (existingRow) {
|
||||
// Overwrite path: delete the old frame before we swap the row, unless
|
||||
// the dedup coalesced two paths onto the same frame (leave that frame
|
||||
// alone in that case — the other path still references it).
|
||||
const oldFrameId = Number(existingRow.frame_id);
|
||||
if (oldFrameId !== frame.id) {
|
||||
const otherRef = raw.prepare('SELECT 1 FROM file_index WHERE frame_id = ? AND file_path != ? LIMIT 1').get(oldFrameId, filePath);
|
||||
if (!otherRef) {
|
||||
this.frames.delete(oldFrameId);
|
||||
}
|
||||
}
|
||||
raw.prepare(`
|
||||
UPDATE file_index SET
|
||||
frame_id = ?,
|
||||
mime_type = ?,
|
||||
size_bytes = ?,
|
||||
content_hash = ?,
|
||||
indexed_at = datetime('now')
|
||||
WHERE file_path = ?
|
||||
`).run(frame.id, mime ?? null, content.length, hash, filePath);
|
||||
} else {
|
||||
raw.prepare(`
|
||||
INSERT INTO file_index (file_path, frame_id, mime_type, size_bytes, content_hash)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`).run(filePath, frame.id, mime ?? null, content.length, hash);
|
||||
}
|
||||
|
||||
return frame.id;
|
||||
});
|
||||
|
||||
const frameId = mutate();
|
||||
return { skipped: false, frameId, truncated };
|
||||
}
|
||||
|
||||
/** Remove a file's frame + index row. Returns true if anything was removed. */
|
||||
removeFile(filePath: string): boolean {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT frame_id FROM file_index WHERE file_path = ?').get(filePath) as
|
||||
| { frame_id: number }
|
||||
| undefined;
|
||||
if (!row) return false;
|
||||
|
||||
const otherRef = raw.prepare('SELECT 1 FROM file_index WHERE frame_id = ? AND file_path != ? LIMIT 1').get(row.frame_id, filePath);
|
||||
if (!otherRef) {
|
||||
this.frames.delete(row.frame_id);
|
||||
}
|
||||
raw.prepare('DELETE FROM file_index WHERE file_path = ?').run(filePath);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the recorded path for a moved file. The frame content itself still
|
||||
* carries the old path in its header — that's acceptable for v1 since the
|
||||
* frame body is already the file's own content (not a description). A later
|
||||
* reader sees the stale header only if they dig into raw frame content; the
|
||||
* index table, which is what every other code path queries, is correct.
|
||||
*/
|
||||
moveFile(from: string, to: string): boolean {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(from) as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
if (!row) return false;
|
||||
|
||||
// If `to` already indexed, drop its entry (file was replaced by the move).
|
||||
raw.prepare('DELETE FROM file_index WHERE file_path = ?').run(to);
|
||||
raw.prepare('UPDATE file_index SET file_path = ?, indexed_at = datetime(\'now\') WHERE file_path = ?').run(to, from);
|
||||
return true;
|
||||
}
|
||||
|
||||
getRow(filePath: string): FileIndexRow | null {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT * FROM file_index WHERE file_path = ?').get(filePath) as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
if (!row) return null;
|
||||
return {
|
||||
filePath: row.file_path as string,
|
||||
frameId: row.frame_id as number,
|
||||
mimeType: (row.mime_type as string | null) ?? null,
|
||||
sizeBytes: row.size_bytes as number,
|
||||
contentHash: row.content_hash as string,
|
||||
indexedAt: row.indexed_at as string,
|
||||
};
|
||||
}
|
||||
|
||||
listAll(): FileIndexRow[] {
|
||||
const raw = this.db.getDatabase();
|
||||
const rows = raw.prepare('SELECT * FROM file_index ORDER BY indexed_at DESC').all() as Record<string, unknown>[];
|
||||
return rows.map(r => ({
|
||||
filePath: r.file_path as string,
|
||||
frameId: r.frame_id as number,
|
||||
mimeType: (r.mime_type as string | null) ?? null,
|
||||
sizeBytes: r.size_bytes as number,
|
||||
contentHash: r.content_hash as string,
|
||||
indexedAt: r.indexed_at as string,
|
||||
}));
|
||||
}
|
||||
|
||||
private normalizeText(content: Buffer): string {
|
||||
// UTF-8 decode. Malformed bytes become U+FFFD; acceptable for index use.
|
||||
const full = content.toString('utf-8');
|
||||
if (Buffer.byteLength(full, 'utf-8') <= MAX_CONTENT_BYTES) return full;
|
||||
// Truncate in character-safe chunks. Walk down until we're under the byte
|
||||
// budget; avoids cutting a multi-byte character in half.
|
||||
let candidate = full.slice(0, MAX_CONTENT_BYTES);
|
||||
while (Buffer.byteLength(candidate, 'utf-8') > MAX_CONTENT_BYTES && candidate.length > 0) {
|
||||
candidate = candidate.slice(0, candidate.length - 1);
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
private buildFrameContent(filePath: string, body: string, truncated: boolean, mime?: string): string {
|
||||
const header = `[FILE: ${filePath}${mime ? ` · ${mime}` : ''}]`;
|
||||
const suffix = truncated ? `\n\n[…truncated at ${MAX_CONTENT_BYTES} bytes for indexing]` : '';
|
||||
return `${header}\n\n${body}${suffix}`;
|
||||
}
|
||||
}
|
||||
639
packages/core/src/file-store.ts
Normal file
639
packages/core/src/file-store.ts
Normal file
@@ -0,0 +1,639 @@
|
||||
/**
|
||||
* FileStore — workspace file storage abstraction.
|
||||
*
|
||||
* Provides a clean interface for reading/writing workspace files that abstracts
|
||||
* over the underlying storage backend. Two implementations:
|
||||
*
|
||||
* - LocalFileStore: manages files in ~/.waggle/workspaces/{id}/files/ (virtual storage)
|
||||
* - LinkedDirStore: reads/writes to workspace.directory (linked to external folder)
|
||||
* - S3FileStore: MinIO/S3-backed storage for team/cloud deployments (lazy SDK import)
|
||||
*
|
||||
* All operations enforce path traversal protection — files cannot escape
|
||||
* the workspace boundary.
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { glob } from 'glob';
|
||||
// Type-only import: erased at compile time, so it does not force @aws-sdk/client-s3
|
||||
// (a devDependency) into the runtime bundle for local-only users.
|
||||
import type { S3Client } from '@aws-sdk/client-s3';
|
||||
|
||||
// ── Interface ───────────────────────────────────────────────────────
|
||||
|
||||
export interface FileEntry {
|
||||
name: string;
|
||||
path: string; // relative to workspace root
|
||||
size: number;
|
||||
modified: string; // ISO timestamp
|
||||
isDirectory: boolean;
|
||||
}
|
||||
|
||||
export interface StorageInfo {
|
||||
usedBytes: number;
|
||||
fileCount: number;
|
||||
storageType: 'virtual' | 'linked';
|
||||
}
|
||||
|
||||
export interface FileStore {
|
||||
// Read
|
||||
readFile(relativePath: string): Promise<Buffer>;
|
||||
listFiles(directory?: string): Promise<FileEntry[]>;
|
||||
searchFiles(pattern: string): Promise<FileEntry[]>;
|
||||
|
||||
// Write
|
||||
writeFile(relativePath: string, content: Buffer | string): Promise<void>;
|
||||
deleteFile(relativePath: string): Promise<void>;
|
||||
moveFile(from: string, to: string): Promise<void>;
|
||||
|
||||
// Meta
|
||||
getStorageInfo(): Promise<StorageInfo>;
|
||||
getRootPath(): string;
|
||||
getStorageType(): 'virtual' | 'linked';
|
||||
}
|
||||
|
||||
// ── Path safety ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Directory segments that, anywhere in a path, almost always hold secrets.
|
||||
* Matched (normalized, case-insensitive) against each path segment.
|
||||
*/
|
||||
const SENSITIVE_DIR_SEGMENTS = new Set([
|
||||
'.ssh', '.aws', '.gnupg', '.gpg', '.docker', '.kube', '.azure', '.terraform', '.terraform.d',
|
||||
]);
|
||||
|
||||
/** Exact basenames (normalized) that are secret material. */
|
||||
const SENSITIVE_BASENAMES = new Set([
|
||||
'id_rsa', 'id_dsa', 'id_ecdsa', 'id_ed25519', 'authorized_keys', 'known_hosts',
|
||||
'.netrc', '.pgpass', '.npmrc', '.pypirc', '.git-credentials',
|
||||
'credentials', 'credentials.json', 'service-account.json',
|
||||
'terraform.tfstate', 'terraform.tfstate.backup',
|
||||
]);
|
||||
|
||||
/** Extensions that are (almost always) private-key material. */
|
||||
const SENSITIVE_EXTENSIONS = new Set(['.pem']);
|
||||
|
||||
/** Backup/copy suffixes — strip and re-test the base (id_rsa.bak → id_rsa). */
|
||||
const BACKUP_SUFFIX_RE = /\.(bak|old|backup|orig|copy|save|swp)$/i;
|
||||
|
||||
/** `.env` files are secrets — but the documented, checked-in templates are not. */
|
||||
const ENV_TEMPLATE_ALLOW = new Set(['.env.example', '.env.sample', '.env.template', '.env.dist', '.env.defaults']);
|
||||
|
||||
/**
|
||||
* Normalize one path segment to the name the OS will actually open: lower-case
|
||||
* (case-insensitive FS), strip a Windows NTFS alternate-data-stream suffix
|
||||
* (`id_rsa::$DATA` → `id_rsa`) and any trailing dots/spaces (`id_rsa.`, `.env `
|
||||
* → the base) which Windows silently removes when opening.
|
||||
*/
|
||||
function normalizeSegment(seg: string): string {
|
||||
return seg.toLowerCase().replace(/::.*$/, '').replace(/[. ]+$/, '');
|
||||
}
|
||||
|
||||
function isSensitiveBase(base: string): boolean {
|
||||
if (SENSITIVE_BASENAMES.has(base)) return true;
|
||||
const dot = base.lastIndexOf('.');
|
||||
if (dot > 0 && SENSITIVE_EXTENSIONS.has(base.slice(dot))) return true;
|
||||
if (base === '.env' || base.startsWith('.env.')) return !ENV_TEMPLATE_ALLOW.has(base);
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a path points at well-known secret material (SSH/GPG keys, cloud
|
||||
* credentials, dotenv files, terraform state, …). Used to deny reads/writes
|
||||
* inside LINKED external folders so an agent given a project directory cannot
|
||||
* exfiltrate or clobber the user's secrets.
|
||||
*
|
||||
* It is a BLOCKLIST (defense-in-depth), not a sandbox: it raises the bar against
|
||||
* obvious secrets but cannot enumerate every secret a home dir holds. Conservative
|
||||
* on extensions (only *.pem) to avoid denying legitimate files. Path-separator
|
||||
* agnostic; segments are normalized for case + Windows ADS/trailing-char tricks.
|
||||
*/
|
||||
export function isSensitiveFilePath(relativePath: string): boolean {
|
||||
const segments = relativePath.replace(/\\/g, '/').split('/').map(normalizeSegment).filter(Boolean);
|
||||
if (segments.length === 0) return false;
|
||||
for (const seg of segments) {
|
||||
if (SENSITIVE_DIR_SEGMENTS.has(seg)) return true;
|
||||
}
|
||||
const base = segments[segments.length - 1];
|
||||
if (isSensitiveBase(base)) return true;
|
||||
if (BACKUP_SUFFIX_RE.test(base) && isSensitiveBase(base.replace(BACKUP_SUFFIX_RE, ''))) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
interface ResolveSafeOptions {
|
||||
/** Reject paths flagged by isSensitiveFilePath (linked external dirs only). */
|
||||
denySensitive?: boolean;
|
||||
}
|
||||
|
||||
/** realpathSync, falling back to the input if it can't be resolved (e.g. not created yet). */
|
||||
function safeRealpath(p: string): string {
|
||||
try { return fs.realpathSync(p); } catch { return p; }
|
||||
}
|
||||
|
||||
/** Walk up to the deepest ancestor of `p` that exists on disk. */
|
||||
function deepestExisting(p: string): string {
|
||||
let cur = p;
|
||||
while (cur !== path.dirname(cur) && !fs.existsSync(cur)) cur = path.dirname(cur);
|
||||
return cur;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve `relativePath` under `root` and assert it cannot escape the boundary.
|
||||
*
|
||||
* Two layers:
|
||||
* 1. LEXICAL segment-boundary containment — the resolved path is the root or
|
||||
* sits under `root + sep` (a bare `startsWith(root)` wrongly admits a sibling
|
||||
* like `${root}-evil/secret`).
|
||||
* 2. SYMLINK-aware containment — `path.resolve` is lexical but `fs.*` follows
|
||||
* symlinks, so we realpath the deepest EXISTING ancestor of the target and
|
||||
* re-check it is still under the realpath'd root. This blocks a benign-named
|
||||
* symlink that points outside the boundary, while still permitting in-root
|
||||
* symlinks (e.g. monorepo package links). The sensitive-file deny then runs
|
||||
* on BOTH the lexical and the real in-root path, so an in-root symlink to a
|
||||
* secret (`alias → ./.env`) cannot launder it past a benign basename.
|
||||
*/
|
||||
function resolveSafe(root: string, relativePath: string, opts: ResolveSafeOptions = {}): string {
|
||||
const resolvedRoot = path.resolve(root);
|
||||
const resolved = path.resolve(resolvedRoot, relativePath);
|
||||
if (resolved !== resolvedRoot && !resolved.startsWith(resolvedRoot + path.sep)) {
|
||||
throw new Error(`Path traversal denied: ${relativePath}`);
|
||||
}
|
||||
|
||||
// Symlink-aware containment only applies once the root exists on disk — if it
|
||||
// doesn't, nothing inside it exists to be a symlink, and walking the deepest
|
||||
// existing ancestor above the (not-yet-created) root would compare against an
|
||||
// unrelated real dir (e.g. an OS temp-dir symlink). Lexical containment holds.
|
||||
const realRoot = safeRealpath(resolvedRoot);
|
||||
let realTarget = realRoot;
|
||||
if (fs.existsSync(resolvedRoot)) {
|
||||
realTarget = safeRealpath(deepestExisting(resolved));
|
||||
if (realTarget !== realRoot && !realTarget.startsWith(realRoot + path.sep)) {
|
||||
throw new Error(`Path traversal denied (symlink): ${relativePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (opts.denySensitive) {
|
||||
const lexicalRel = path.relative(resolvedRoot, resolved);
|
||||
const realRel = path.relative(realRoot, realTarget);
|
||||
if (isSensitiveFilePath(lexicalRel) || isSensitiveFilePath(realRel)) {
|
||||
throw new Error(`Access to sensitive file denied: ${relativePath}`);
|
||||
}
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
/** Keep only glob matches that resolve back inside `root` (glob `../` patterns can escape cwd). */
|
||||
function containGlobMatches(root: string, matches: string[]): string[] {
|
||||
const resolvedRoot = path.resolve(root);
|
||||
return matches.filter(m => {
|
||||
const abs = path.resolve(resolvedRoot, m);
|
||||
return abs === resolvedRoot || abs.startsWith(resolvedRoot + path.sep);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a `*`/`?` glob to a RegExp that is SAFE against catastrophic
|
||||
* backtracking (ReDoS). Every regex metacharacter is escaped first, so the only
|
||||
* specials left are the two wildcards we re-introduce (`.*`, `.`) — neither can
|
||||
* nest a quantifier, so a hostile pattern like `(a+)+x` becomes a harmless
|
||||
* literal. Used by S3FileStore.searchFiles where the pattern is caller-controlled.
|
||||
* Unanchored (substring match) to preserve the prior behavior.
|
||||
*/
|
||||
function globToRegExp(pattern: string): RegExp {
|
||||
const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const body = escaped.replace(/\\\*/g, '.*').replace(/\\\?/g, '.');
|
||||
return new RegExp(body, 'i');
|
||||
}
|
||||
|
||||
/**
|
||||
* Reject an S3 key relative-path that would escape its workspace prefix. S3 keys
|
||||
* are literal, but MinIO (path-style, filesystem-backed) can normalize `..`, so a
|
||||
* `..` segment could cross into a sibling workspace's prefix. Makes the file
|
||||
* header's "all operations enforce path traversal protection" true for S3 too.
|
||||
*/
|
||||
function assertSafeRelativeKey(relativePath: string): void {
|
||||
const segments = relativePath.replace(/\\/g, '/').split('/');
|
||||
if (segments.some(s => s === '..')) {
|
||||
throw new Error(`Path traversal denied: ${relativePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
function ensureDir(dirPath: string): void {
|
||||
if (!fs.existsSync(dirPath)) {
|
||||
fs.mkdirSync(dirPath, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
// ── LocalFileStore (virtual workspace storage) ──────────────────────
|
||||
|
||||
export class LocalFileStore implements FileStore {
|
||||
private readonly root: string;
|
||||
|
||||
constructor(dataDir: string, workspaceId: string) {
|
||||
this.root = path.join(dataDir, 'workspaces', workspaceId, 'files');
|
||||
}
|
||||
|
||||
getRootPath(): string { return this.root; }
|
||||
getStorageType(): 'virtual' { return 'virtual'; }
|
||||
|
||||
async readFile(relativePath: string): Promise<Buffer> {
|
||||
const fullPath = resolveSafe(this.root, relativePath);
|
||||
return fs.readFileSync(fullPath);
|
||||
}
|
||||
|
||||
async listFiles(directory?: string): Promise<FileEntry[]> {
|
||||
const dir = directory ? resolveSafe(this.root, directory) : this.root;
|
||||
if (!fs.existsSync(dir)) return [];
|
||||
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
return entries.map(entry => {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
const stat = fs.statSync(fullPath);
|
||||
return {
|
||||
name: entry.name,
|
||||
path: path.relative(this.root, fullPath).replace(/\\/g, '/'),
|
||||
size: stat.size,
|
||||
modified: stat.mtime.toISOString(),
|
||||
isDirectory: entry.isDirectory(),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async searchFiles(pattern: string): Promise<FileEntry[]> {
|
||||
if (!fs.existsSync(this.root)) return [];
|
||||
const matches = await glob(pattern, {
|
||||
cwd: this.root,
|
||||
nodir: true,
|
||||
ignore: ['node_modules/**', '.git/**'],
|
||||
});
|
||||
// A glob pattern with `../` can escape cwd — keep only matches inside root.
|
||||
return containGlobMatches(this.root, matches).slice(0, 200).map(match => {
|
||||
const fullPath = path.join(this.root, match);
|
||||
try {
|
||||
const stat = fs.statSync(fullPath);
|
||||
return {
|
||||
name: path.basename(match),
|
||||
path: match.replace(/\\/g, '/'),
|
||||
size: stat.size,
|
||||
modified: stat.mtime.toISOString(),
|
||||
isDirectory: false,
|
||||
};
|
||||
} catch {
|
||||
return { name: path.basename(match), path: match, size: 0, modified: '', isDirectory: false };
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async writeFile(relativePath: string, content: Buffer | string): Promise<void> {
|
||||
const fullPath = resolveSafe(this.root, relativePath);
|
||||
// Lazy directory creation
|
||||
ensureDir(path.dirname(fullPath));
|
||||
fs.writeFileSync(fullPath, content);
|
||||
}
|
||||
|
||||
async deleteFile(relativePath: string): Promise<void> {
|
||||
const fullPath = resolveSafe(this.root, relativePath);
|
||||
if (fs.existsSync(fullPath)) {
|
||||
const stat = fs.statSync(fullPath);
|
||||
if (stat.isDirectory()) {
|
||||
fs.rmSync(fullPath, { recursive: true });
|
||||
} else {
|
||||
fs.unlinkSync(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async moveFile(from: string, to: string): Promise<void> {
|
||||
const fromPath = resolveSafe(this.root, from);
|
||||
const toPath = resolveSafe(this.root, to);
|
||||
ensureDir(path.dirname(toPath));
|
||||
fs.renameSync(fromPath, toPath);
|
||||
}
|
||||
|
||||
async getStorageInfo(): Promise<StorageInfo> {
|
||||
if (!fs.existsSync(this.root)) {
|
||||
return { usedBytes: 0, fileCount: 0, storageType: 'virtual' };
|
||||
}
|
||||
let usedBytes = 0;
|
||||
let fileCount = 0;
|
||||
const walk = (dir: string) => {
|
||||
try {
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
walk(fullPath);
|
||||
} else {
|
||||
try {
|
||||
usedBytes += fs.statSync(fullPath).size;
|
||||
fileCount++;
|
||||
} catch { /* skip unreadable */ }
|
||||
}
|
||||
}
|
||||
} catch { /* skip unreadable dirs */ }
|
||||
};
|
||||
walk(this.root);
|
||||
return { usedBytes, fileCount, storageType: 'virtual' };
|
||||
}
|
||||
}
|
||||
|
||||
// ── LinkedDirStore (linked to external directory) ───────────────────
|
||||
|
||||
export class LinkedDirStore implements FileStore {
|
||||
private readonly root: string;
|
||||
|
||||
constructor(directory: string) {
|
||||
this.root = directory;
|
||||
}
|
||||
|
||||
getRootPath(): string { return this.root; }
|
||||
getStorageType(): 'linked' { return 'linked'; }
|
||||
|
||||
// Linked stores point at a REAL external folder (a code project, even the home
|
||||
// dir), so every path op denies well-known secret files — the agent cannot
|
||||
// read or clobber ~/.ssh, .env, cloud credentials, etc. (LocalFileStore is a
|
||||
// sandboxed virtual dir and needs no such deny.)
|
||||
private static readonly DENY = { denySensitive: true } as const;
|
||||
|
||||
async readFile(relativePath: string): Promise<Buffer> {
|
||||
const fullPath = resolveSafe(this.root, relativePath, LinkedDirStore.DENY);
|
||||
return fs.readFileSync(fullPath);
|
||||
}
|
||||
|
||||
async listFiles(directory?: string): Promise<FileEntry[]> {
|
||||
const dir = directory ? resolveSafe(this.root, directory, LinkedDirStore.DENY) : this.root;
|
||||
if (!fs.existsSync(dir)) return [];
|
||||
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
return entries
|
||||
.filter(e => !e.name.startsWith('.') && e.name !== 'node_modules')
|
||||
// Don't disclose the existence/size/mtime of non-dot secrets (credentials.json,
|
||||
// id_rsa, known_hosts, …) — symmetric with searchFiles + the read deny.
|
||||
.filter(e => !isSensitiveFilePath(e.name))
|
||||
.map(entry => {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
try {
|
||||
const stat = fs.statSync(fullPath);
|
||||
return {
|
||||
name: entry.name,
|
||||
path: path.relative(this.root, fullPath).replace(/\\/g, '/'),
|
||||
size: stat.size,
|
||||
modified: stat.mtime.toISOString(),
|
||||
isDirectory: entry.isDirectory(),
|
||||
};
|
||||
} catch {
|
||||
return { name: entry.name, path: entry.name, size: 0, modified: '', isDirectory: false };
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async searchFiles(pattern: string): Promise<FileEntry[]> {
|
||||
const matches = await glob(pattern, {
|
||||
cwd: this.root,
|
||||
nodir: true,
|
||||
ignore: ['node_modules/**', '.git/**', 'dist/**', 'build/**', '**/.ssh/**', '**/.aws/**', '**/.gnupg/**'],
|
||||
});
|
||||
// Contain `../`-escaping globs to the root, then drop any secret a creative
|
||||
// pattern still matched — search never discloses the existence/path of secrets.
|
||||
return containGlobMatches(this.root, matches).filter(m => !isSensitiveFilePath(m)).slice(0, 200).map(match => {
|
||||
const fullPath = path.join(this.root, match);
|
||||
try {
|
||||
const stat = fs.statSync(fullPath);
|
||||
return {
|
||||
name: path.basename(match),
|
||||
path: match.replace(/\\/g, '/'),
|
||||
size: stat.size,
|
||||
modified: stat.mtime.toISOString(),
|
||||
isDirectory: false,
|
||||
};
|
||||
} catch {
|
||||
return { name: path.basename(match), path: match, size: 0, modified: '', isDirectory: false };
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async writeFile(relativePath: string, content: Buffer | string): Promise<void> {
|
||||
const fullPath = resolveSafe(this.root, relativePath, LinkedDirStore.DENY);
|
||||
ensureDir(path.dirname(fullPath));
|
||||
fs.writeFileSync(fullPath, content);
|
||||
}
|
||||
|
||||
async deleteFile(relativePath: string): Promise<void> {
|
||||
const fullPath = resolveSafe(this.root, relativePath, LinkedDirStore.DENY);
|
||||
if (fs.existsSync(fullPath)) {
|
||||
fs.unlinkSync(fullPath);
|
||||
}
|
||||
}
|
||||
|
||||
async moveFile(from: string, to: string): Promise<void> {
|
||||
const fromPath = resolveSafe(this.root, from, LinkedDirStore.DENY);
|
||||
const toPath = resolveSafe(this.root, to, LinkedDirStore.DENY);
|
||||
ensureDir(path.dirname(toPath));
|
||||
fs.renameSync(fromPath, toPath);
|
||||
}
|
||||
|
||||
async getStorageInfo(): Promise<StorageInfo> {
|
||||
if (!fs.existsSync(this.root)) {
|
||||
return { usedBytes: 0, fileCount: 0, storageType: 'linked' };
|
||||
}
|
||||
let usedBytes = 0;
|
||||
let fileCount = 0;
|
||||
const walk = (dir: string, depth = 0) => {
|
||||
if (depth > 5) return; // Don't walk too deep in linked dirs
|
||||
try {
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (entry.name === 'node_modules' || entry.name === '.git') continue;
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
walk(fullPath, depth + 1);
|
||||
} else {
|
||||
try {
|
||||
usedBytes += fs.statSync(fullPath).size;
|
||||
fileCount++;
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
}
|
||||
} catch { /* skip */ }
|
||||
};
|
||||
walk(this.root);
|
||||
return { usedBytes, fileCount, storageType: 'linked' };
|
||||
}
|
||||
}
|
||||
|
||||
// ── S3FileStore (MinIO/S3 for team workspace storage) ───────────────
|
||||
|
||||
export interface S3Config {
|
||||
endpoint: string;
|
||||
bucket: string;
|
||||
accessKey: string;
|
||||
secretKey: string;
|
||||
prefix: string; // e.g., "workspaces/{id}/"
|
||||
region?: string;
|
||||
}
|
||||
|
||||
export class S3FileStore implements FileStore {
|
||||
private config: S3Config;
|
||||
// Lazily constructed S3 client; the SDK is imported on first use to avoid
|
||||
// bundling it for local-only users.
|
||||
private client: S3Client | undefined;
|
||||
|
||||
constructor(config: S3Config) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
private async getClient(): Promise<S3Client> {
|
||||
if (this.client) return this.client;
|
||||
const { S3Client } = await import('@aws-sdk/client-s3');
|
||||
this.client = new S3Client({
|
||||
endpoint: `http://${this.config.endpoint}`,
|
||||
region: this.config.region ?? 'us-east-1',
|
||||
credentials: {
|
||||
accessKeyId: this.config.accessKey,
|
||||
secretAccessKey: this.config.secretKey,
|
||||
},
|
||||
forcePathStyle: true, // Required for MinIO
|
||||
});
|
||||
return this.client;
|
||||
}
|
||||
|
||||
getRootPath(): string { return `s3://${this.config.bucket}/${this.config.prefix}`; }
|
||||
getStorageType(): 'virtual' | 'linked' { return 'virtual'; }
|
||||
|
||||
async readFile(relativePath: string): Promise<Buffer> {
|
||||
assertSafeRelativeKey(relativePath);
|
||||
const client = await this.getClient();
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
const key = this.config.prefix + relativePath;
|
||||
const response = await client.send(new GetObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
Key: key,
|
||||
}));
|
||||
const body = response.Body;
|
||||
if (!body) throw new Error(`S3 object has no body: ${key}`);
|
||||
const chunks: Uint8Array[] = [];
|
||||
// In Node.js the S3 streaming body is an async-iterable readable stream.
|
||||
for await (const chunk of body as AsyncIterable<Uint8Array>) chunks.push(chunk);
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
async writeFile(relativePath: string, content: Buffer | string): Promise<void> {
|
||||
assertSafeRelativeKey(relativePath);
|
||||
const client = await this.getClient();
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
const key = this.config.prefix + relativePath;
|
||||
await client.send(new PutObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
Key: key,
|
||||
Body: typeof content === 'string' ? Buffer.from(content) : content,
|
||||
}));
|
||||
}
|
||||
|
||||
async deleteFile(relativePath: string): Promise<void> {
|
||||
assertSafeRelativeKey(relativePath);
|
||||
const client = await this.getClient();
|
||||
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
const key = this.config.prefix + relativePath;
|
||||
await client.send(new DeleteObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
Key: key,
|
||||
}));
|
||||
}
|
||||
|
||||
async listFiles(directory?: string): Promise<FileEntry[]> {
|
||||
if (directory) assertSafeRelativeKey(directory);
|
||||
const client = await this.getClient();
|
||||
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
|
||||
const pfx = this.config.prefix + (directory ? directory + '/' : '');
|
||||
const response = await client.send(new ListObjectsV2Command({
|
||||
Bucket: this.config.bucket,
|
||||
Prefix: pfx,
|
||||
Delimiter: '/',
|
||||
}));
|
||||
const entries: FileEntry[] = [];
|
||||
for (const obj of response.Contents ?? []) {
|
||||
if (!obj.Key) continue;
|
||||
const relPath = obj.Key.slice(this.config.prefix.length);
|
||||
entries.push({
|
||||
name: relPath.split('/').pop() ?? relPath,
|
||||
path: relPath,
|
||||
size: obj.Size ?? 0,
|
||||
modified: obj.LastModified?.toISOString() ?? '',
|
||||
isDirectory: false,
|
||||
});
|
||||
}
|
||||
for (const cpfx of response.CommonPrefixes ?? []) {
|
||||
if (!cpfx.Prefix) continue;
|
||||
const relPath = cpfx.Prefix.slice(this.config.prefix.length).replace(/\/$/, '');
|
||||
entries.push({
|
||||
name: relPath.split('/').pop() ?? relPath,
|
||||
path: relPath,
|
||||
size: 0,
|
||||
modified: '',
|
||||
isDirectory: true,
|
||||
});
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
async searchFiles(pattern: string): Promise<FileEntry[]> {
|
||||
const all = await this.listFiles();
|
||||
const regex = globToRegExp(pattern); // ReDoS-safe glob→regex (pattern is caller-controlled)
|
||||
return all.filter(f => regex.test(f.path) || regex.test(f.name));
|
||||
}
|
||||
|
||||
async moveFile(from: string, to: string): Promise<void> {
|
||||
assertSafeRelativeKey(from);
|
||||
assertSafeRelativeKey(to);
|
||||
const client = await this.getClient();
|
||||
const { CopyObjectCommand, DeleteObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
const fromKey = this.config.prefix + from;
|
||||
const toKey = this.config.prefix + to;
|
||||
await client.send(new CopyObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
CopySource: `${this.config.bucket}/${fromKey}`,
|
||||
Key: toKey,
|
||||
}));
|
||||
await client.send(new DeleteObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
Key: fromKey,
|
||||
}));
|
||||
}
|
||||
|
||||
async getStorageInfo(): Promise<StorageInfo> {
|
||||
const client = await this.getClient();
|
||||
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
|
||||
const response = await client.send(new ListObjectsV2Command({
|
||||
Bucket: this.config.bucket,
|
||||
Prefix: this.config.prefix,
|
||||
}));
|
||||
let usedBytes = 0;
|
||||
let fileCount = 0;
|
||||
for (const obj of response.Contents ?? []) {
|
||||
usedBytes += obj.Size ?? 0;
|
||||
fileCount++;
|
||||
}
|
||||
return { usedBytes, fileCount, storageType: 'virtual' };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Factory ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Create the appropriate FileStore for a workspace.
|
||||
* If s3Config is provided, use S3FileStore (team/cloud deployment).
|
||||
* If the workspace has a linked directory, use LinkedDirStore.
|
||||
* Otherwise, use LocalFileStore (virtual managed storage).
|
||||
*/
|
||||
export function createFileStore(
|
||||
dataDir: string,
|
||||
workspaceId: string,
|
||||
linkedDirectory?: string,
|
||||
s3Config?: S3Config,
|
||||
): FileStore {
|
||||
if (s3Config) return new S3FileStore(s3Config);
|
||||
if (linkedDirectory && fs.existsSync(linkedDirectory)) {
|
||||
return new LinkedDirStore(linkedDirectory);
|
||||
}
|
||||
return new LocalFileStore(dataDir, workspaceId);
|
||||
}
|
||||
159
packages/core/src/index.ts
Normal file
159
packages/core/src/index.ts
Normal file
@@ -0,0 +1,159 @@
|
||||
// @waggle/core — Waggle-specific orchestration layer.
|
||||
//
|
||||
// As of CC Sesija B 2026-04-30 (PM Q3 Plan A ratification), the substrate
|
||||
// (mind/ + harvest/ + logger + injection-scanner) lives in @waggle/hive-mind-core
|
||||
// and is distributed as Apache 2.0 OSS via subtree-split. This barrel re-exports
|
||||
// substrate symbols for backward compatibility — existing consumers (packages/agent,
|
||||
// apps/web, packages/server, etc.) keep their `import { ... } from '@waggle/core'`
|
||||
// imports unchanged.
|
||||
//
|
||||
// Waggle-specific (non-extracted) modules below the substrate re-exports stay
|
||||
// in this package: config, multi-mind, workspace orchestration, vault, telemetry,
|
||||
// install-audit, cron-store, skill-hashes, team-sync, file-store, file-indexer,
|
||||
// memory-import, optimization-log, compliance/.
|
||||
|
||||
// ── Substrate re-exports from @waggle/hive-mind-core (Apache 2.0 OSS) ──
|
||||
export {
|
||||
// Logger + injection scanner
|
||||
createCoreLogger, type CoreLogger,
|
||||
scanForInjection, type ScanResult,
|
||||
// mind/ — memory substrate
|
||||
MindDB, EmbeddingDimMismatchError,
|
||||
type EmbeddingFingerprint, type FingerprintCheck,
|
||||
IdentityLayer, type Identity,
|
||||
AwarenessLayer, type AwarenessItem, type AwarenessCategory,
|
||||
FrameStore, stripHmPrefix, hashFrameContent,
|
||||
RawArchive, hashRaw, readArchiveUids, withArchiveUid, type RawArchiveRow, type ArchiveInput,
|
||||
MindErasure, type EraseResult,
|
||||
SuppressionStore, type SuppressedSubject,
|
||||
type MemoryFrame, type FrameType, type Importance, type FrameSource,
|
||||
SessionStore, type Session,
|
||||
HybridSearch, type SearchResult,
|
||||
// D1 (oss-drift triage, 2026-06-11) — chunk-level retrieval (flag-gated, default OFF)
|
||||
chunkRetrievalEnabled, rechunkAllFrames, type RechunkResult,
|
||||
chunkText, type ChunkOptions, type FrameChunk,
|
||||
KnowledgeGraph, type Entity, type Relation, type ValidationSchema,
|
||||
SCHEMA_SQL, VEC_TABLE_SQL, CHUNKS_VEC_TABLE_SQL, SCHEMA_VERSION,
|
||||
vecTableSqlForDim, chunksVecTableSqlForDim,
|
||||
computeRelevance,
|
||||
computeTemporalScore,
|
||||
computePopularityScore,
|
||||
computeContextualScore,
|
||||
computeImportanceScore,
|
||||
SCORING_PROFILES,
|
||||
type ScoringProfile,
|
||||
type ScoringWeights,
|
||||
type Embedder,
|
||||
createLiteLLMEmbedder, type LiteLLMEmbedderConfig,
|
||||
createInProcessEmbedder, normalizeDimensions, type InProcessEmbedderConfig,
|
||||
createOllamaEmbedder, type OllamaEmbedderConfig,
|
||||
createApiEmbedder, type ApiEmbedderConfig,
|
||||
createEmbeddingProvider, EmbeddingQuotaExceededError, getMinimumTierForProvider,
|
||||
type EmbeddingProviderConfig, type EmbeddingProviderStatus, type EmbeddingProviderType,
|
||||
type EmbeddingProviderInstance, type EmbeddingQuotaStatus,
|
||||
normalizeEntityName, findDuplicates, isNoiseName, isLikelyAcronym,
|
||||
Ontology, validateEntity, type EntitySchema, type ValidationResult,
|
||||
ImprovementSignalStore,
|
||||
type ImprovementSignal, type ActionableSignal, type SignalCategory, type ActionableThresholds,
|
||||
ExecutionTraceStore, EXECUTION_TRACES_TABLE_SQL,
|
||||
type ExecutionTrace, type ParsedExecutionTrace, type TraceOutcome,
|
||||
type TracePayload, type TraceToolCall, type TraceReasoningStep,
|
||||
type StartTraceInput, type FinalizeTraceInput, type TraceQueryFilter,
|
||||
EvolutionRunStore, EVOLUTION_RUNS_TABLE_SQL,
|
||||
type EvolutionRun, type EvolutionRunStatus, type EvolutionRunTarget,
|
||||
type CreateEvolutionRunInput, type EvolutionRunFilter,
|
||||
reconcileIndexes, reconcileFtsIndex, reconcileVecIndex, cleanOrphanVectors, cleanOrphanFts,
|
||||
type ReconcileResult,
|
||||
ConceptTracker, CONCEPT_MASTERY_TABLE_SQL,
|
||||
type ConceptEntry, type ConceptUpdate,
|
||||
// recall-context — temporal rendering helpers (W4.1 production port)
|
||||
TEMPORAL_GUIDANCE, toDatePrefix, renderDatedSnippet, referenceDate, renderReferenceDateLine,
|
||||
parseDateWindow, type DateWindow,
|
||||
resolveRelativeDate, type ResolvedDate,
|
||||
createInProcessReranker, type Reranker, type InProcessRerankerConfig,
|
||||
// harvest/ — universal memory ingestion pipeline
|
||||
HarvestSourceStore,
|
||||
HarvestRunStore, type HarvestRun, type HarvestRunStatus,
|
||||
ChatGPTAdapter,
|
||||
ClaudeAdapter,
|
||||
ClaudeCodeAdapter,
|
||||
GeminiAdapter,
|
||||
UniversalAdapter,
|
||||
MarkdownAdapter,
|
||||
PlaintextAdapter,
|
||||
UrlAdapter,
|
||||
PdfAdapter,
|
||||
HarvestPipeline, type LLMCallFn, type PipelineOptions,
|
||||
extractMemoryLanes, writeMemoryLaneFrames,
|
||||
MIND_FACT_PREFIX, MIND_EVENT_PREFIX, MIND_PROFILE_PREFIX,
|
||||
type MemoryLaneExtraction, type ExtractedEvent, type ExtractedFact, type ExtractedProfile,
|
||||
type WriteLaneFramesResult,
|
||||
// D2 — LLM-based KG entity extraction (oss-drift triage, 2026-06-11)
|
||||
extractKgEntities, writeKgEntities, KG_ENTITY_TYPES,
|
||||
type KgEntity, type KgEntityType, type KgEntityExtraction, type WriteKgEntitiesResult,
|
||||
// W4.6 — raw-turn storage + RAWDETAIL recall lane
|
||||
writeRawTurnFrames, rawTurnHeader, parseRawTurnHeader, rawTurnConvKey,
|
||||
MIND_RAWTURN_PREFIX, MAX_TURNS_PER_ITEM, RAWDETAIL_KILL_SWITCH,
|
||||
type WriteRawTurnsResult, type ParsedRawTurnHeader,
|
||||
fetchRawDetailLane, rawTurnBody, RAW_DETAIL_K,
|
||||
type RawTurnHit, type RawDetailLaneOptions,
|
||||
dedup, harvestSetHash,
|
||||
HARVEST_FRAME_CONTENT_CAP,
|
||||
type ImportSourceType, type ImportItemType, type UniversalImportItem, type DistilledKnowledge,
|
||||
type HarvestPipelineResult, type HarvestSource, type SourceAdapter, type FilesystemAdapter,
|
||||
type ClassifiedItem, type ExtractedContent, type KnowledgeProvenance,
|
||||
// Plan A AMENDMENT 2026-04-30 — multi-workspace orchestration
|
||||
MultiMind, type MultiMindSearchResult, type MindSource, type SearchScope,
|
||||
MultiMindCache, type MultiMindCacheConfig,
|
||||
WorkspaceManager, type WorkspaceConfig, type CreateWorkspaceOptions,
|
||||
// Supersession (P) + bridge (B) frame producer (mono-parity 2026-07-05)
|
||||
detectSupersessionChains, detectEntityGroups, applyConsolidation,
|
||||
collectObservations, getCurrentValues,
|
||||
type ConsolidationLlm, type Observation, type SupersessionChain,
|
||||
type EntityGroup, type ConsolidationResult, type CollectObservationsOptions,
|
||||
} from '@waggle/hive-mind-core';
|
||||
|
||||
// ── Waggle-specific orchestration (stays in @waggle/core) ──
|
||||
export { WaggleConfig, type ProviderEntry, type TeamServerConfig } from './config.js';
|
||||
export { needsMigration, migrateToMultiMind } from './migration.js';
|
||||
export { TeamSync, frameToEntity, entityToSyncedFrame, type TeamSyncConfig, type SyncedFrame } from './team-sync.js';
|
||||
export {
|
||||
InstallAuditStore, INSTALL_AUDIT_TABLE_SQL,
|
||||
type InstallAuditEntry, type RecordAuditInput,
|
||||
type AuditAction, type AuditRiskLevel, type AuditTrustSource,
|
||||
type AuditApprovalClass, type AuditInitiator, type AuditCapabilityType,
|
||||
} from './install-audit.js';
|
||||
export {
|
||||
CronStore, CRON_SCHEDULES_TABLE_SQL, CRON_RUN_LEASES_TABLE_SQL, VALID_JOB_TYPES, cronExprError,
|
||||
type CronSchedule, type CreateScheduleInput, type CronJobType,
|
||||
type CronExecutionRow, type CronRunLeaseRow,
|
||||
type PendingActionRow, type PendingActionStatus, type SavePendingActionInput,
|
||||
} from './cron-store.js';
|
||||
export { VaultStore, type VaultEntry } from './vault.js';
|
||||
export { TelemetryStore, TelemetryCollector, TELEMETRY_EVENTS, type TelemetryEvent, type TelemetrySummary } from './telemetry.js';
|
||||
export {
|
||||
SkillHashStore, computeSkillHash, SKILL_HASHES_TABLE_SQL,
|
||||
type SkillHash,
|
||||
} from './skill-hashes.js';
|
||||
export { processImport, parseChatGPTExport, parseClaudeExport, extractKnowledge } from './memory-import.js';
|
||||
export { createFileStore, LocalFileStore, LinkedDirStore, S3FileStore, isSensitiveFilePath, type FileStore, type FileEntry, type StorageInfo, type S3Config } from './file-store.js';
|
||||
export { FileIndexer, MAX_CONTENT_BYTES, type FileIndexRow, type FileIndexResult } from './file-indexer.js';
|
||||
export type { ImportSource, ImportResult, ExtractedKnowledge, ParsedConversation, ConversationMessage } from './memory-import.js';
|
||||
export {
|
||||
OptimizationLogStore, OPTIMIZATION_LOG_TABLE_SQL,
|
||||
type OptimizationLogEntry, type CreateOptimizationLogInput,
|
||||
} from './optimization-log.js';
|
||||
|
||||
// ── Compliance (AI Act) — stays in @waggle/core (NOT extracted per .github/sync.md) ──
|
||||
export { InteractionStore } from './compliance/interaction-store.js';
|
||||
export { ComplianceStatusChecker } from './compliance/status-checker.js';
|
||||
export { ReportGenerator, type ReportGeneratorDeps } from './compliance/report-generator.js';
|
||||
export { ComplianceTemplateStore, KVARK_TEMPLATE_NAME } from './compliance/template-store.js';
|
||||
export { TEMPLATE_RISK_MAP } from './compliance/types.js';
|
||||
export type {
|
||||
AIActRiskLevel, HumanAction, AIInteraction, RecordInteractionInput,
|
||||
ComplianceStatus, ArticleStatus, AuditReport, AuditReportRequest,
|
||||
ModelInventoryEntry, OversightLogEntry, HarvestProvenanceEntry,
|
||||
ComplianceTemplate, ComplianceTemplateSections,
|
||||
CreateComplianceTemplateInput, UpdateComplianceTemplateInput,
|
||||
} from './compliance/types.js';
|
||||
221
packages/core/src/install-audit.ts
Normal file
221
packages/core/src/install-audit.ts
Normal file
@@ -0,0 +1,221 @@
|
||||
/**
|
||||
* Install Audit Store — persistent audit trail for capability install events.
|
||||
*
|
||||
* Records every install-relevant action (proposed, approved, installed, rejected,
|
||||
* failed) so there is a verifiable history of what was installed, when, why,
|
||||
* and by whom.
|
||||
*
|
||||
* Follows the same pattern as ImprovementSignalStore — operates on the .mind DB.
|
||||
*/
|
||||
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
import {
|
||||
sqlInList, RISK_LEVELS, APPROVAL_CLASSES, AUDIT_ACTIONS,
|
||||
AUDIT_CAPABILITY_TYPES, AUDIT_INITIATORS, TRUST_SOURCES,
|
||||
} from '@waggle/shared';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
// P7/D15 A2b: the audit vocabulary is now canonical in @waggle/shared. The
|
||||
// Audit*-prefixed names are kept as aliases (re-exported) so every downstream
|
||||
// `import { AuditAction, ... } from '@waggle/core'` keeps working unchanged.
|
||||
// All six sets were already byte-identical to the shared ones (incl. P5/D4's
|
||||
// 'uninstalled'), so this is a pure structural re-point — no value change.
|
||||
export type {
|
||||
AuditAction, AuditCapabilityType, AuditInitiator,
|
||||
RiskLevel as AuditRiskLevel, ApprovalClass as AuditApprovalClass, TrustSource as AuditTrustSource,
|
||||
} from '@waggle/shared';
|
||||
import type {
|
||||
AuditAction, AuditCapabilityType, AuditInitiator,
|
||||
RiskLevel as AuditRiskLevel, ApprovalClass as AuditApprovalClass, TrustSource as AuditTrustSource,
|
||||
} from '@waggle/shared';
|
||||
|
||||
export interface InstallAuditEntry {
|
||||
id: number;
|
||||
timestamp: string;
|
||||
capability_name: string;
|
||||
capability_type: AuditCapabilityType;
|
||||
source: string;
|
||||
version: string | null;
|
||||
risk_level: AuditRiskLevel;
|
||||
trust_source: AuditTrustSource;
|
||||
approval_class: AuditApprovalClass;
|
||||
action: AuditAction;
|
||||
initiator: AuditInitiator;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
export interface RecordAuditInput {
|
||||
capabilityName: string;
|
||||
capabilityType: AuditCapabilityType;
|
||||
source: string;
|
||||
version?: string | null;
|
||||
riskLevel: AuditRiskLevel;
|
||||
trustSource: AuditTrustSource;
|
||||
approvalClass: AuditApprovalClass;
|
||||
action: AuditAction;
|
||||
initiator: AuditInitiator;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
// ── Table DDL ──────────────────────────────────────────────────────────
|
||||
|
||||
// P7/D15 A3: the CHECK lists are generated from the canonical @waggle/shared
|
||||
// arrays via sqlInList, so the SQLite constraint and the TS union can no longer
|
||||
// drift (divergence #14 — the old comment admitted "drift silently crashes
|
||||
// record()"). The mirror DDL in hive-mind-core/src/mind/schema.ts stays a
|
||||
// standalone literal (it's the OSS substrate, §7.5) but is locked to these same
|
||||
// canonical lists by the parity test in install-audit-check-parity.test.ts.
|
||||
// P7/D15 #15: trust_source now also carries a CHECK (was unconstrained at the DB
|
||||
// while the TS type claimed a closed set). Every historical value came from the
|
||||
// typed AuditTrustSource (the pre-security-gate 6-set ⊂ the current 7-set), so
|
||||
// the rebuild migration's row copy can never violate it.
|
||||
export const INSTALL_AUDIT_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS install_audit (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
capability_name TEXT NOT NULL,
|
||||
capability_type TEXT NOT NULL CHECK (capability_type IN (${sqlInList(AUDIT_CAPABILITY_TYPES)})),
|
||||
source TEXT NOT NULL,
|
||||
version TEXT,
|
||||
risk_level TEXT NOT NULL CHECK (risk_level IN (${sqlInList(RISK_LEVELS)})),
|
||||
trust_source TEXT NOT NULL CHECK (trust_source IN (${sqlInList(TRUST_SOURCES)})),
|
||||
approval_class TEXT NOT NULL CHECK (approval_class IN (${sqlInList(APPROVAL_CLASSES)})),
|
||||
action TEXT NOT NULL CHECK (action IN (${sqlInList(AUDIT_ACTIONS)})),
|
||||
initiator TEXT NOT NULL CHECK (initiator IN (${sqlInList(AUDIT_INITIATORS)})),
|
||||
detail TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_capability ON install_audit (capability_name, action);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_timestamp ON install_audit (timestamp DESC);
|
||||
`;
|
||||
|
||||
// ── Store ──────────────────────────────────────────────────────────────
|
||||
|
||||
export class InstallAuditStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
const existing = raw.prepare(
|
||||
"SELECT sql FROM sqlite_master WHERE type='table' AND name='install_audit'",
|
||||
).get() as { sql: string } | undefined;
|
||||
if (!existing) {
|
||||
raw.exec(INSTALL_AUDIT_TABLE_SQL);
|
||||
return;
|
||||
}
|
||||
// P5/D4 migration: pre-'uninstalled' installs carry a narrower action CHECK
|
||||
// baked into the table DDL. SQLite can't ALTER a CHECK, so rebuild the table
|
||||
// when the stored DDL lacks the new value. Idempotent — a no-op once migrated.
|
||||
// #15: also rebuild when the stored DDL has no trust_source CHECK (the column
|
||||
// was previously unconstrained). "CHECK (trust_source IN" is a safe sentinel —
|
||||
// it appears nowhere else in this DDL.
|
||||
const needsActionWiden = !existing.sql.includes("'uninstalled'");
|
||||
const needsTrustSourceCheck = !existing.sql.includes('CHECK (trust_source IN');
|
||||
if (needsActionWiden || needsTrustSourceCheck) {
|
||||
this.rebuildForWidenedActionCheck(raw);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild install_audit with the widened `action` CHECK, preserving all rows.
|
||||
* Classic SQLite 12-step table redefinition, wrapped in a transaction so a
|
||||
* crash mid-rebuild leaves the original table intact.
|
||||
*/
|
||||
private rebuildForWidenedActionCheck(raw: ReturnType<MindDB['getDatabase']>): void {
|
||||
const migrate = raw.transaction(() => {
|
||||
raw.exec('ALTER TABLE install_audit RENAME TO install_audit_legacy');
|
||||
// SQLite carries indexes along with RENAME (still named idx_audit_*), so
|
||||
// INSTALL_AUDIT_TABLE_SQL's CREATE INDEX IF NOT EXISTS would no-op and the
|
||||
// DROP below would take the indexes with the legacy table. Drop them first
|
||||
// (mirrors hive-mind-core db.ts) so they get recreated on the new table.
|
||||
raw.exec('DROP INDEX IF EXISTS idx_audit_capability');
|
||||
raw.exec('DROP INDEX IF EXISTS idx_audit_timestamp');
|
||||
raw.exec(INSTALL_AUDIT_TABLE_SQL);
|
||||
raw.exec(`
|
||||
INSERT INTO install_audit (
|
||||
id, timestamp, capability_name, capability_type, source, version,
|
||||
risk_level, trust_source, approval_class, action, initiator, detail
|
||||
)
|
||||
SELECT id, timestamp, capability_name, capability_type, source, version,
|
||||
risk_level, trust_source, approval_class, action, initiator, detail
|
||||
FROM install_audit_legacy
|
||||
`);
|
||||
raw.exec('DROP TABLE install_audit_legacy');
|
||||
});
|
||||
migrate();
|
||||
}
|
||||
|
||||
/** Record an install audit event. */
|
||||
record(input: RecordAuditInput): InstallAuditEntry {
|
||||
const raw = this.db.getDatabase();
|
||||
|
||||
raw.prepare(`
|
||||
INSERT INTO install_audit (
|
||||
capability_name, capability_type, source, version,
|
||||
risk_level, trust_source, approval_class, action, initiator, detail
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
input.capabilityName,
|
||||
input.capabilityType,
|
||||
input.source,
|
||||
input.version ?? null,
|
||||
input.riskLevel,
|
||||
input.trustSource,
|
||||
input.approvalClass,
|
||||
input.action,
|
||||
input.initiator,
|
||||
input.detail ?? '',
|
||||
);
|
||||
|
||||
// Return the inserted row
|
||||
return raw.prepare(
|
||||
'SELECT * FROM install_audit ORDER BY id DESC LIMIT 1',
|
||||
).get() as InstallAuditEntry;
|
||||
}
|
||||
|
||||
/** Get audit history for a specific capability. */
|
||||
getByCapability(name: string): InstallAuditEntry[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM install_audit WHERE capability_name = ? ORDER BY id DESC',
|
||||
).all(name) as InstallAuditEntry[];
|
||||
}
|
||||
|
||||
/** Get audit history filtered by action type. */
|
||||
getByAction(action: AuditAction): InstallAuditEntry[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM install_audit WHERE action = ? ORDER BY id DESC',
|
||||
).all(action) as InstallAuditEntry[];
|
||||
}
|
||||
|
||||
/** Get recent audit entries (most recent first). */
|
||||
getRecent(limit: number = 20): InstallAuditEntry[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM install_audit ORDER BY id DESC LIMIT ?',
|
||||
).all(limit) as InstallAuditEntry[];
|
||||
}
|
||||
|
||||
/** Get recent entries for one capability type (most recent first) — backs
|
||||
* the shared Extend-layer audit read (GET /api/extend/audit?type=, C18). */
|
||||
getRecentByType(type: AuditCapabilityType, limit: number = 20): InstallAuditEntry[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM install_audit WHERE capability_type = ? ORDER BY id DESC LIMIT ?',
|
||||
).all(type, limit) as InstallAuditEntry[];
|
||||
}
|
||||
|
||||
/** Get all entries (for testing). */
|
||||
getAll(): InstallAuditEntry[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM install_audit ORDER BY id ASC',
|
||||
).all() as InstallAuditEntry[];
|
||||
}
|
||||
|
||||
/** Clear all entries (for testing). */
|
||||
clear(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM install_audit').run();
|
||||
}
|
||||
}
|
||||
292
packages/core/src/memory-import.ts
Normal file
292
packages/core/src/memory-import.ts
Normal file
@@ -0,0 +1,292 @@
|
||||
/**
|
||||
* Memory Import — parse ChatGPT and Claude exports, extract knowledge.
|
||||
*
|
||||
* Flow: raw JSON → parse conversations → extract knowledge items → ImportResult
|
||||
*/
|
||||
|
||||
export type ImportSource = 'chatgpt' | 'claude';
|
||||
|
||||
export interface ConversationMessage {
|
||||
role: 'user' | 'assistant';
|
||||
text: string;
|
||||
timestamp?: string;
|
||||
}
|
||||
|
||||
export interface ParsedConversation {
|
||||
title: string;
|
||||
messages: ConversationMessage[];
|
||||
createdAt?: string;
|
||||
source: ImportSource;
|
||||
}
|
||||
|
||||
export interface ExtractedKnowledge {
|
||||
type: 'decision' | 'fact' | 'preference' | 'topic';
|
||||
content: string;
|
||||
source: ImportSource;
|
||||
conversationTitle: string;
|
||||
importance: 'important' | 'normal';
|
||||
}
|
||||
|
||||
export interface ImportResult {
|
||||
source: ImportSource;
|
||||
conversationsFound: number;
|
||||
conversationsParsed: number;
|
||||
knowledgeExtracted: ExtractedKnowledge[];
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
// ── Raw export shapes (untrusted external JSON; all fields optional) ──
|
||||
|
||||
/** A ChatGPT export node carries an optional message in its `mapping` entry. */
|
||||
interface ChatGPTMessage {
|
||||
author?: { role?: string };
|
||||
content?: { parts?: unknown[] };
|
||||
create_time?: number;
|
||||
}
|
||||
|
||||
interface ChatGPTNode {
|
||||
message?: ChatGPTMessage;
|
||||
}
|
||||
|
||||
interface ChatGPTConversation {
|
||||
title?: string;
|
||||
mapping?: Record<string, ChatGPTNode>;
|
||||
create_time?: number;
|
||||
}
|
||||
|
||||
interface ClaudeMessage {
|
||||
sender?: string;
|
||||
role?: string;
|
||||
text?: string;
|
||||
content?: string;
|
||||
created_at?: string;
|
||||
timestamp?: string;
|
||||
}
|
||||
|
||||
interface ClaudeConversation {
|
||||
name?: string;
|
||||
title?: string;
|
||||
chat_messages?: ClaudeMessage[];
|
||||
messages?: ClaudeMessage[];
|
||||
created_at?: string;
|
||||
create_time?: string;
|
||||
}
|
||||
|
||||
/** Pull the `conversations` array out of an export that may be the array itself. */
|
||||
function extractConversations(json: unknown): unknown[] {
|
||||
if (Array.isArray(json)) return json;
|
||||
if (json && typeof json === 'object' && 'conversations' in json) {
|
||||
const c = (json as { conversations?: unknown }).conversations;
|
||||
if (Array.isArray(c)) return c;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
// ── Parsers ───────────────────────────────────────
|
||||
|
||||
export function parseChatGPTExport(json: unknown): ParsedConversation[] {
|
||||
const conversations = extractConversations(json);
|
||||
|
||||
return conversations.map((raw) => {
|
||||
const conv = raw as ChatGPTConversation;
|
||||
const title = conv.title || 'Untitled';
|
||||
const messages: ConversationMessage[] = [];
|
||||
|
||||
// ChatGPT uses a mapping object with node IDs
|
||||
if (conv.mapping && typeof conv.mapping === 'object') {
|
||||
const nodes = Object.values(conv.mapping);
|
||||
// Sort by create_time for chronological order
|
||||
const sorted = nodes
|
||||
.filter((n) => (n?.message?.content?.parts?.length ?? 0) > 0)
|
||||
.sort((a, b) => (a.message?.create_time ?? 0) - (b.message?.create_time ?? 0));
|
||||
|
||||
for (const node of sorted) {
|
||||
const msg = node.message;
|
||||
if (!msg || !msg.author?.role) continue;
|
||||
const role = msg.author.role === 'user' ? 'user' : 'assistant';
|
||||
if (msg.author.role === 'system') continue; // Skip system messages
|
||||
|
||||
const text = (msg.content?.parts ?? [])
|
||||
.filter((p): p is string => typeof p === 'string')
|
||||
.join('\n')
|
||||
.trim();
|
||||
if (!text) continue;
|
||||
|
||||
messages.push({
|
||||
role,
|
||||
text,
|
||||
timestamp: msg.create_time ? new Date(msg.create_time * 1000).toISOString() : undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
title,
|
||||
messages,
|
||||
createdAt: conv.create_time ? new Date(conv.create_time * 1000).toISOString() : undefined,
|
||||
source: 'chatgpt' as ImportSource,
|
||||
};
|
||||
}).filter(c => c.messages.length > 0);
|
||||
}
|
||||
|
||||
export function parseClaudeExport(json: unknown): ParsedConversation[] {
|
||||
const conversations = extractConversations(json);
|
||||
|
||||
return conversations.map((raw) => {
|
||||
const conv = raw as ClaudeConversation;
|
||||
const title = conv.name || conv.title || 'Untitled';
|
||||
const messages: ConversationMessage[] = [];
|
||||
|
||||
const chatMessages = conv.chat_messages ?? conv.messages ?? [];
|
||||
for (const msg of chatMessages) {
|
||||
const role = (msg.sender === 'human' || msg.role === 'user') ? 'user' : 'assistant';
|
||||
const text = (msg.text ?? msg.content ?? '').trim();
|
||||
if (!text) continue;
|
||||
|
||||
messages.push({
|
||||
role,
|
||||
text,
|
||||
timestamp: msg.created_at ?? msg.timestamp,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
title,
|
||||
messages,
|
||||
createdAt: conv.created_at ?? conv.create_time,
|
||||
source: 'claude' as ImportSource,
|
||||
};
|
||||
}).filter(c => c.messages.length > 0);
|
||||
}
|
||||
|
||||
// ── Knowledge Extraction ──────────────────────────
|
||||
|
||||
const DECISION_PATTERNS = [
|
||||
/\b(?:decided|chose|going with|we'll use|confirmed|agreed|settled on|picked|selected)\b/i,
|
||||
/\b(?:decision|choice):\s/i,
|
||||
/\blet's go with\b/i,
|
||||
];
|
||||
|
||||
const PREFERENCE_PATTERNS = [
|
||||
/\b(?:I prefer|I like|I always|I never|I don't like|don't use|use .+ instead)\b/i,
|
||||
/\b(?:my preference|I'd rather|please always|please never|from now on)\b/i,
|
||||
/\b(?:my style is|I want you to|format it as)\b/i,
|
||||
];
|
||||
|
||||
const FACT_PATTERNS = [
|
||||
/\b(?:my name is|I work at|I'm a|I am a|my company|my team|my role|our product|our stack)\b/i,
|
||||
/\b(?:we use|our tech|we're building|the project is|the codebase)\b/i,
|
||||
];
|
||||
|
||||
export function extractKnowledge(conversations: ParsedConversation[]): ExtractedKnowledge[] {
|
||||
const knowledge: ExtractedKnowledge[] = [];
|
||||
const seen = new Set<string>(); // Deduplicate
|
||||
|
||||
for (const conv of conversations) {
|
||||
// Extract conversation topic
|
||||
if (conv.title && conv.title !== 'Untitled' && conv.title.length > 5) {
|
||||
const topicKey = `topic:${conv.title.toLowerCase().trim()}`;
|
||||
if (!seen.has(topicKey)) {
|
||||
seen.add(topicKey);
|
||||
knowledge.push({
|
||||
type: 'topic',
|
||||
content: `Conversation topic: ${conv.title}`,
|
||||
source: conv.source,
|
||||
conversationTitle: conv.title,
|
||||
importance: 'normal',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Scan user messages for decisions, preferences, facts
|
||||
for (const msg of conv.messages) {
|
||||
if (msg.role !== 'user') continue;
|
||||
if (msg.text.length < 10 || msg.text.length > 500) continue; // Skip too short/long
|
||||
|
||||
const sentences = msg.text.split(/[.!?\n]+/).map(s => s.trim()).filter(s => s.length > 10);
|
||||
|
||||
for (const sentence of sentences) {
|
||||
const contentKey = sentence.toLowerCase().slice(0, 80);
|
||||
if (seen.has(contentKey)) continue;
|
||||
|
||||
// Check for decisions
|
||||
if (DECISION_PATTERNS.some(p => p.test(sentence))) {
|
||||
seen.add(contentKey);
|
||||
knowledge.push({
|
||||
type: 'decision',
|
||||
content: `Decision: ${sentence}`,
|
||||
source: conv.source,
|
||||
conversationTitle: conv.title,
|
||||
importance: 'important',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check for preferences
|
||||
if (PREFERENCE_PATTERNS.some(p => p.test(sentence))) {
|
||||
seen.add(contentKey);
|
||||
knowledge.push({
|
||||
type: 'preference',
|
||||
content: `Preference: ${sentence}`,
|
||||
source: conv.source,
|
||||
conversationTitle: conv.title,
|
||||
importance: 'important',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check for facts
|
||||
if (FACT_PATTERNS.some(p => p.test(sentence))) {
|
||||
seen.add(contentKey);
|
||||
knowledge.push({
|
||||
type: 'fact',
|
||||
content: `${sentence}`,
|
||||
source: conv.source,
|
||||
conversationTitle: conv.title,
|
||||
importance: 'normal',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cap at 100 items to avoid polluting memory
|
||||
return knowledge.slice(0, 100);
|
||||
}
|
||||
|
||||
// ── Main Import Function ──────────────────────────
|
||||
|
||||
export function processImport(jsonData: unknown, source: ImportSource): ImportResult {
|
||||
const errors: string[] = [];
|
||||
|
||||
let conversations: ParsedConversation[];
|
||||
try {
|
||||
conversations = source === 'chatgpt'
|
||||
? parseChatGPTExport(jsonData)
|
||||
: parseClaudeExport(jsonData);
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
return {
|
||||
source,
|
||||
conversationsFound: 0,
|
||||
conversationsParsed: 0,
|
||||
knowledgeExtracted: [],
|
||||
errors: [`Parse error: ${message}`],
|
||||
};
|
||||
}
|
||||
|
||||
if (conversations.length === 0) {
|
||||
errors.push('No conversations found in export');
|
||||
}
|
||||
|
||||
const knowledge = extractKnowledge(conversations);
|
||||
|
||||
return {
|
||||
source,
|
||||
conversationsFound: conversations.length,
|
||||
conversationsParsed: conversations.filter(c => c.messages.length > 0).length,
|
||||
knowledgeExtracted: knowledge,
|
||||
errors,
|
||||
};
|
||||
}
|
||||
45
packages/core/src/migration.ts
Normal file
45
packages/core/src/migration.ts
Normal file
@@ -0,0 +1,45 @@
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
/**
|
||||
* Check if a migration from default.mind to personal.mind is needed.
|
||||
* Returns true if default.mind exists AND personal.mind does NOT exist.
|
||||
*/
|
||||
export function needsMigration(waggleDir: string): boolean {
|
||||
const defaultMind = path.join(waggleDir, 'default.mind');
|
||||
const personalMind = path.join(waggleDir, 'personal.mind');
|
||||
return fs.existsSync(defaultMind) && !fs.existsSync(personalMind);
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrate from the old single-mind layout (default.mind) to the
|
||||
* M4 multi-mind layout (personal.mind + workspaces/).
|
||||
*
|
||||
* Steps when migration is needed:
|
||||
* 1. Copy default.mind -> personal.mind
|
||||
* 2. Rename default.mind -> default.mind.bak
|
||||
* 3. Create workspaces/ directory
|
||||
*
|
||||
* This is a pure file operation -- no data transformation is performed.
|
||||
*/
|
||||
export function migrateToMultiMind(waggleDir: string): { migrated: boolean; message: string } {
|
||||
if (!needsMigration(waggleDir)) {
|
||||
return { migrated: false, message: 'No migration needed' };
|
||||
}
|
||||
|
||||
const defaultMind = path.join(waggleDir, 'default.mind');
|
||||
const personalMind = path.join(waggleDir, 'personal.mind');
|
||||
const backupMind = path.join(waggleDir, 'default.mind.bak');
|
||||
const workspacesDir = path.join(waggleDir, 'workspaces');
|
||||
|
||||
// 1. Copy default.mind -> personal.mind
|
||||
fs.copyFileSync(defaultMind, personalMind);
|
||||
|
||||
// 2. Rename default.mind -> default.mind.bak
|
||||
fs.renameSync(defaultMind, backupMind);
|
||||
|
||||
// 3. Create workspaces/ directory (idempotent)
|
||||
fs.mkdirSync(workspacesDir, { recursive: true });
|
||||
|
||||
return { migrated: true, message: 'Migrated default.mind to personal.mind' };
|
||||
}
|
||||
154
packages/core/src/optimization-log.ts
Normal file
154
packages/core/src/optimization-log.ts
Normal file
@@ -0,0 +1,154 @@
|
||||
/**
|
||||
* Optimization Log Store — SQLite persistence for GEPA prompt optimization data.
|
||||
*
|
||||
* Captures agent interaction metadata (system prompts, tools used, turn counts,
|
||||
* correction signals) for background analysis by the PromptOptimizer.
|
||||
*
|
||||
* Follows the same lazy-table pattern as CronStore and InstallAuditStore —
|
||||
* operates on the .mind DB with auto-table creation.
|
||||
*/
|
||||
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
export interface OptimizationLogEntry {
|
||||
id: number;
|
||||
session_id: string;
|
||||
workspace_id: string;
|
||||
system_prompt: string;
|
||||
tools_used: string; // JSON array
|
||||
turn_count: number;
|
||||
was_correction: number; // SQLite integer boolean
|
||||
input_tokens: number;
|
||||
output_tokens: number;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
export interface CreateOptimizationLogInput {
|
||||
sessionId: string;
|
||||
workspaceId: string;
|
||||
systemPrompt: string;
|
||||
toolsUsed: string[];
|
||||
turnCount: number;
|
||||
wasCorrection: boolean;
|
||||
inputTokens?: number;
|
||||
outputTokens?: number;
|
||||
}
|
||||
|
||||
// ── Table DDL ──────────────────────────────────────────────────────────
|
||||
|
||||
export const OPTIMIZATION_LOG_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS optimization_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT NOT NULL,
|
||||
workspace_id TEXT NOT NULL,
|
||||
system_prompt TEXT NOT NULL,
|
||||
tools_used TEXT NOT NULL DEFAULT '[]',
|
||||
turn_count INTEGER NOT NULL DEFAULT 0,
|
||||
was_correction INTEGER NOT NULL DEFAULT 0,
|
||||
input_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
output_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
timestamp TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_optlog_workspace ON optimization_log (workspace_id, timestamp DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_optlog_timestamp ON optimization_log (timestamp DESC);
|
||||
`;
|
||||
|
||||
// ── Store ──────────────────────────────────────────────────────────────
|
||||
|
||||
export class OptimizationLogStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
const exists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='optimization_log'",
|
||||
).get();
|
||||
if (!exists) {
|
||||
raw.exec(OPTIMIZATION_LOG_TABLE_SQL);
|
||||
}
|
||||
}
|
||||
|
||||
/** Insert a new optimization log entry capturing an agent interaction. */
|
||||
insert(input: CreateOptimizationLogInput): OptimizationLogEntry {
|
||||
const raw = this.db.getDatabase();
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO optimization_log (session_id, workspace_id, system_prompt, tools_used, turn_count, was_correction, input_tokens, output_tokens)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(
|
||||
input.sessionId,
|
||||
input.workspaceId,
|
||||
input.systemPrompt,
|
||||
JSON.stringify(input.toolsUsed),
|
||||
input.turnCount,
|
||||
input.wasCorrection ? 1 : 0,
|
||||
input.inputTokens ?? 0,
|
||||
input.outputTokens ?? 0,
|
||||
);
|
||||
|
||||
return raw.prepare(
|
||||
'SELECT * FROM optimization_log WHERE id = ?',
|
||||
).get(result.lastInsertRowid) as OptimizationLogEntry;
|
||||
}
|
||||
|
||||
/** Get recent optimization log entries, ordered by most recent first. */
|
||||
getRecent(limit: number = 50): OptimizationLogEntry[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM optimization_log ORDER BY timestamp DESC, id DESC LIMIT ?',
|
||||
).all(limit) as OptimizationLogEntry[];
|
||||
}
|
||||
|
||||
/** Get log entries for a specific workspace. */
|
||||
getByWorkspace(workspaceId: string, limit: number = 50): OptimizationLogEntry[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM optimization_log WHERE workspace_id = ? ORDER BY timestamp DESC, id DESC LIMIT ?',
|
||||
).all(workspaceId, limit) as OptimizationLogEntry[];
|
||||
}
|
||||
|
||||
/** Get aggregate stats: total entries, correction rate, avg turn count, total tokens. */
|
||||
getStats(): { total: number; correctionRate: number; avgTurnCount: number; totalInputTokens: number; totalOutputTokens: number } {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare(`
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
COALESCE(AVG(was_correction * 1.0), 0) as correction_rate,
|
||||
COALESCE(AVG(turn_count), 0) as avg_turn_count,
|
||||
COALESCE(SUM(input_tokens), 0) as total_input_tokens,
|
||||
COALESCE(SUM(output_tokens), 0) as total_output_tokens
|
||||
FROM optimization_log
|
||||
`).get() as {
|
||||
total: number;
|
||||
correction_rate: number;
|
||||
avg_turn_count: number;
|
||||
total_input_tokens: number;
|
||||
total_output_tokens: number;
|
||||
};
|
||||
|
||||
return {
|
||||
total: row.total,
|
||||
correctionRate: row.correction_rate,
|
||||
avgTurnCount: row.avg_turn_count,
|
||||
totalInputTokens: row.total_input_tokens,
|
||||
totalOutputTokens: row.total_output_tokens,
|
||||
};
|
||||
}
|
||||
|
||||
/** Delete entries older than the given number of days. */
|
||||
pruneOlderThan(days: number): number {
|
||||
const result = this.db.getDatabase().prepare(
|
||||
`DELETE FROM optimization_log WHERE timestamp < datetime('now', '-' || ? || ' days')`,
|
||||
).run(days);
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
/** Clear all entries (for testing). */
|
||||
clear(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM optimization_log').run();
|
||||
}
|
||||
}
|
||||
122
packages/core/src/skill-hashes.ts
Normal file
122
packages/core/src/skill-hashes.ts
Normal file
@@ -0,0 +1,122 @@
|
||||
/**
|
||||
* Skill Hash Store — SHA-256 content hashing for skill update detection.
|
||||
*
|
||||
* Stores hashes of skill file content in the .mind DB. On server startup,
|
||||
* compares current file hashes to stored hashes to detect changes.
|
||||
* Follows the same ensureTable pattern as CronStore / InstallAuditStore.
|
||||
*/
|
||||
|
||||
import * as crypto from 'node:crypto';
|
||||
import type { MindDB } from '@waggle/hive-mind-core';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
export interface SkillHash {
|
||||
name: string;
|
||||
hash: string;
|
||||
verified_at: string;
|
||||
}
|
||||
|
||||
// ── Table DDL ──────────────────────────────────────────────────────────
|
||||
|
||||
export const SKILL_HASHES_TABLE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS skill_hashes (
|
||||
name TEXT PRIMARY KEY,
|
||||
hash TEXT NOT NULL,
|
||||
verified_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
`;
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
/** Compute SHA-256 hex digest of skill content. */
|
||||
export function computeSkillHash(content: string): string {
|
||||
return crypto.createHash('sha256').update(content).digest('hex');
|
||||
}
|
||||
|
||||
// ── Store ──────────────────────────────────────────────────────────────
|
||||
|
||||
export class SkillHashStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
const exists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='skill_hashes'",
|
||||
).get();
|
||||
if (!exists) {
|
||||
raw.exec(SKILL_HASHES_TABLE_SQL);
|
||||
}
|
||||
}
|
||||
|
||||
/** Store or update hash for a skill. */
|
||||
setHash(name: string, hash: string): void {
|
||||
this.db.getDatabase().prepare(`
|
||||
INSERT INTO skill_hashes (name, hash) VALUES (?, ?)
|
||||
ON CONFLICT (name) DO UPDATE SET hash = ?, verified_at = datetime('now')
|
||||
`).run(name, hash, hash);
|
||||
}
|
||||
|
||||
/** Get stored hash for a skill. */
|
||||
getHash(name: string): SkillHash | undefined {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT * FROM skill_hashes WHERE name = ?',
|
||||
).get(name) as SkillHash | undefined;
|
||||
}
|
||||
|
||||
/** Remove hash (when skill is deleted). */
|
||||
removeHash(name: string): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM skill_hashes WHERE name = ?').run(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check all skills against stored hashes.
|
||||
* Returns lists of changed, added, and removed skill names.
|
||||
*/
|
||||
checkAll(currentSkills: Array<{ name: string; content: string }>): {
|
||||
changed: string[];
|
||||
added: string[];
|
||||
removed: string[];
|
||||
} {
|
||||
const storedHashes = this.db.getDatabase().prepare(
|
||||
'SELECT name, hash FROM skill_hashes',
|
||||
).all() as SkillHash[];
|
||||
const storedMap = new Map(storedHashes.map(h => [h.name, h.hash]));
|
||||
const currentNames = new Set(currentSkills.map(s => s.name));
|
||||
|
||||
const changed: string[] = [];
|
||||
const added: string[] = [];
|
||||
|
||||
for (const skill of currentSkills) {
|
||||
const currentHash = computeSkillHash(skill.content);
|
||||
const storedHash = storedMap.get(skill.name);
|
||||
|
||||
if (!storedHash) {
|
||||
added.push(skill.name);
|
||||
} else if (storedHash !== currentHash) {
|
||||
changed.push(skill.name);
|
||||
}
|
||||
}
|
||||
|
||||
const removed = storedHashes
|
||||
.filter(h => !currentNames.has(h.name))
|
||||
.map(h => h.name);
|
||||
|
||||
return { changed, added, removed };
|
||||
}
|
||||
|
||||
/** Mark a skill as verified (update hash to current content). */
|
||||
verify(name: string, content: string): void {
|
||||
this.setHash(name, computeSkillHash(content));
|
||||
}
|
||||
|
||||
/** Clear all hashes (for testing). */
|
||||
clear(): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM skill_hashes').run();
|
||||
}
|
||||
}
|
||||
186
packages/core/src/team-sync.ts
Normal file
186
packages/core/src/team-sync.ts
Normal file
@@ -0,0 +1,186 @@
|
||||
/**
|
||||
* TeamSync — Syncs memory frames between local .mind and team server.
|
||||
*
|
||||
* Uses the team server's entities API (entityType='memory_frame') for storage.
|
||||
* Push-on-write: new frames are pushed to team server after local write.
|
||||
* Pull-on-activate: frames are pulled from team server when workspace is activated.
|
||||
* Attribution: each frame carries the author's userId and displayName.
|
||||
*
|
||||
* Sync protocol:
|
||||
* - Frames are identified by a composite key: gopId + t (group-of-pictures ID + temporal index)
|
||||
* - Pull uses ?type=memory_frame&since=<timestamp> to get only new frames
|
||||
* - Push sends each frame as a teamEntity with entityType='memory_frame'
|
||||
* - Last-write-wins at frame level (frames are append-mostly, conflicts are rare)
|
||||
*/
|
||||
|
||||
import type { MemoryFrame, FrameType, Importance } from '@waggle/hive-mind-core';
|
||||
import { createCoreLogger } from '@waggle/hive-mind-core';
|
||||
const log = createCoreLogger('team-sync');
|
||||
|
||||
export interface TeamSyncConfig {
|
||||
teamServerUrl: string;
|
||||
teamSlug: string;
|
||||
authToken: string;
|
||||
userId: string;
|
||||
displayName: string;
|
||||
}
|
||||
|
||||
export interface SyncedFrame {
|
||||
/** Server-side entity ID (UUID). */
|
||||
remoteId: string;
|
||||
gopId: string;
|
||||
t: number;
|
||||
frameType: FrameType;
|
||||
content: string;
|
||||
importance: Importance;
|
||||
authorId: string;
|
||||
authorName: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a local MemoryFrame to the team server entity format.
|
||||
*/
|
||||
export function frameToEntity(frame: MemoryFrame, authorId: string, authorName: string) {
|
||||
return {
|
||||
entityType: 'memory_frame',
|
||||
name: frame.gop_id,
|
||||
properties: {
|
||||
frameType: frame.frame_type,
|
||||
t: frame.t,
|
||||
baseFrameId: frame.base_frame_id,
|
||||
content: frame.content,
|
||||
importance: frame.importance,
|
||||
authorId,
|
||||
authorName,
|
||||
localId: frame.id,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a team server entity back to a SyncedFrame.
|
||||
*/
|
||||
export function entityToSyncedFrame(entity: {
|
||||
id: string;
|
||||
name: string;
|
||||
properties: Record<string, unknown>;
|
||||
createdAt: string;
|
||||
}): SyncedFrame {
|
||||
const props = entity.properties;
|
||||
return {
|
||||
remoteId: entity.id,
|
||||
gopId: entity.name,
|
||||
t: (props.t as number) ?? 0,
|
||||
frameType: (props.frameType as FrameType) ?? 'I',
|
||||
content: (props.content as string) ?? '',
|
||||
importance: (props.importance as Importance) ?? 'normal',
|
||||
authorId: (props.authorId as string) ?? '',
|
||||
authorName: (props.authorName as string) ?? '',
|
||||
createdAt: entity.createdAt,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* TeamSync handles push/pull of memory frames to/from the team server.
|
||||
* This is a stateless client — call pushFrame() after writing locally,
|
||||
* call pullFrames() on workspace activation.
|
||||
*/
|
||||
export class TeamSync {
|
||||
private config: TeamSyncConfig;
|
||||
private lastSyncTimestamp: string | null = null;
|
||||
|
||||
constructor(config: TeamSyncConfig) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Push a single frame to the team server.
|
||||
* Call this after writing a frame to the local workspace .mind.
|
||||
*/
|
||||
async pushFrame(frame: MemoryFrame): Promise<{ remoteId: string } | null> {
|
||||
const entity = frameToEntity(frame, this.config.userId, this.config.displayName);
|
||||
|
||||
try {
|
||||
const response = await fetch(
|
||||
`${this.config.teamServerUrl}/api/teams/${this.config.teamSlug}/entities`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${this.config.authToken}`,
|
||||
},
|
||||
body: JSON.stringify(entity),
|
||||
}
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
log.error(`Push failed: ${response.status} ${response.statusText}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const created = await response.json() as { id: string };
|
||||
return { remoteId: created.id };
|
||||
} catch (err) {
|
||||
log.error('Push error:', err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull all memory frames from the team server.
|
||||
* Returns frames sorted by creation time (newest first).
|
||||
* Optionally pass `since` to get only frames created after that timestamp.
|
||||
*/
|
||||
async pullFrames(since?: string): Promise<SyncedFrame[]> {
|
||||
try {
|
||||
const url = `${this.config.teamServerUrl}/api/teams/${this.config.teamSlug}/entities?type=memory_frame`;
|
||||
// Note: `since` filtering would require server-side support. For now, pull all and filter client-side.
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
'Authorization': `Bearer ${this.config.authToken}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
log.error(`Pull failed: ${response.status} ${response.statusText}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const entities = await response.json() as Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
properties: Record<string, unknown>;
|
||||
createdAt: string;
|
||||
}>;
|
||||
|
||||
const frames = entities.map(entityToSyncedFrame);
|
||||
|
||||
// Client-side filter by timestamp if requested
|
||||
if (since) {
|
||||
return frames.filter(f => f.createdAt > since);
|
||||
}
|
||||
|
||||
this.lastSyncTimestamp = new Date().toISOString();
|
||||
return frames;
|
||||
} catch (err) {
|
||||
log.error('Pull error:', err);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the last sync timestamp (for incremental sync).
|
||||
*/
|
||||
getLastSyncTimestamp(): string | null {
|
||||
return this.lastSyncTimestamp;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the last sync timestamp (e.g. loaded from persistent storage).
|
||||
*/
|
||||
setLastSyncTimestamp(ts: string): void {
|
||||
this.lastSyncTimestamp = ts;
|
||||
}
|
||||
}
|
||||
324
packages/core/src/telemetry.ts
Normal file
324
packages/core/src/telemetry.ts
Normal file
@@ -0,0 +1,324 @@
|
||||
/**
|
||||
* TelemetryStore — local, privacy-first event tracking.
|
||||
*
|
||||
* All data stays in ~/.waggle/telemetry.db (SQLite).
|
||||
* Default: OFF. User opts in via Settings toggle.
|
||||
* No cloud reporting in M2 — data is queryable via local API only.
|
||||
*
|
||||
* NEVER tracked: message content, memory content, file paths,
|
||||
* API keys, personal info, IP addresses, device IDs.
|
||||
*/
|
||||
|
||||
import Database from 'better-sqlite3';
|
||||
import type { Database as DatabaseType } from 'better-sqlite3';
|
||||
import path from 'node:path';
|
||||
import fs from 'node:fs';
|
||||
|
||||
/* ── Schema ── */
|
||||
const TELEMETRY_SCHEMA = `
|
||||
CREATE TABLE IF NOT EXISTS telemetry_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
event TEXT NOT NULL,
|
||||
properties TEXT NOT NULL DEFAULT '{}',
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_telemetry_event ON telemetry_events (event, created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_telemetry_date ON telemetry_events (created_at);
|
||||
`;
|
||||
|
||||
/* ── Event constants ── */
|
||||
export const TELEMETRY_EVENTS = {
|
||||
// Onboarding funnel
|
||||
ONBOARDING_START: 'onboarding_start',
|
||||
ONBOARDING_STEP: 'onboarding_step',
|
||||
ONBOARDING_COMPLETE: 'onboarding_complete',
|
||||
ONBOARDING_SKIP: 'onboarding_skip',
|
||||
|
||||
// Session engagement
|
||||
SESSION_START: 'session_start',
|
||||
SESSION_END: 'session_end',
|
||||
|
||||
// Infrastructure
|
||||
EMBEDDING_PROVIDER: 'embedding_provider',
|
||||
LLM_PROVIDER: 'llm_provider',
|
||||
|
||||
// Feature usage
|
||||
TEMPLATE_SELECTED: 'template_selected',
|
||||
WORKSPACE_CREATED: 'workspace_created',
|
||||
FIRST_AGENT_RESPONSE: 'first_agent_response',
|
||||
SLASH_COMMAND_USED: 'slash_command_used',
|
||||
|
||||
// App lifecycle
|
||||
APP_START: 'app_start',
|
||||
APP_ERROR: 'app_error',
|
||||
} as const;
|
||||
|
||||
/* ── Types ── */
|
||||
export interface TelemetryEvent {
|
||||
id: number;
|
||||
event: string;
|
||||
properties: Record<string, unknown>;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export interface TelemetrySummary {
|
||||
enabled: boolean;
|
||||
totalEvents: number;
|
||||
firstEvent: string | null;
|
||||
lastEvent: string | null;
|
||||
onboardingCompleted: boolean;
|
||||
totalSessions: number;
|
||||
embeddingProvider: string | null;
|
||||
templatesUsed: string[];
|
||||
eventBreakdown: Record<string, number>;
|
||||
}
|
||||
|
||||
/* ── Store ── */
|
||||
export class TelemetryStore {
|
||||
private db: DatabaseType;
|
||||
private enabled: boolean;
|
||||
|
||||
constructor(dataDir: string, enabled = false) {
|
||||
this.enabled = enabled;
|
||||
const dbPath = path.join(dataDir, 'telemetry.db');
|
||||
this.db = new Database(dbPath);
|
||||
this.db.pragma('journal_mode = WAL');
|
||||
this.db.exec(TELEMETRY_SCHEMA);
|
||||
}
|
||||
|
||||
/** Track an event. No-op if telemetry is disabled. */
|
||||
track(event: string, properties?: Record<string, unknown>): void {
|
||||
if (!this.enabled) return;
|
||||
this.db.prepare(
|
||||
'INSERT INTO telemetry_events (event, properties) VALUES (?, ?)'
|
||||
).run(event, JSON.stringify(properties ?? {}));
|
||||
}
|
||||
|
||||
/** Enable/disable telemetry at runtime. */
|
||||
setEnabled(enabled: boolean): void {
|
||||
this.enabled = enabled;
|
||||
}
|
||||
|
||||
/** Whether telemetry is currently enabled. */
|
||||
isEnabled(): boolean {
|
||||
return this.enabled;
|
||||
}
|
||||
|
||||
/** Get aggregated summary. */
|
||||
getSummary(): TelemetrySummary {
|
||||
const total = (this.db.prepare(
|
||||
'SELECT COUNT(*) as cnt FROM telemetry_events'
|
||||
).get() as { cnt: number }).cnt;
|
||||
|
||||
const first = this.db.prepare(
|
||||
'SELECT MIN(created_at) as d FROM telemetry_events'
|
||||
).get() as { d: string | null };
|
||||
|
||||
const last = this.db.prepare(
|
||||
'SELECT MAX(created_at) as d FROM telemetry_events'
|
||||
).get() as { d: string | null };
|
||||
|
||||
const onboarded = (this.db.prepare(
|
||||
"SELECT COUNT(*) as cnt FROM telemetry_events WHERE event = 'onboarding_complete'"
|
||||
).get() as { cnt: number }).cnt > 0;
|
||||
|
||||
const sessions = (this.db.prepare(
|
||||
"SELECT COUNT(*) as cnt FROM telemetry_events WHERE event = 'session_start'"
|
||||
).get() as { cnt: number }).cnt;
|
||||
|
||||
const embRow = this.db.prepare(
|
||||
"SELECT properties FROM telemetry_events WHERE event = 'embedding_provider' ORDER BY created_at DESC LIMIT 1"
|
||||
).get() as { properties: string } | undefined;
|
||||
const embProvider = embRow
|
||||
? (JSON.parse(embRow.properties) as Record<string, unknown>).provider as string ?? null
|
||||
: null;
|
||||
|
||||
const tplRows = this.db.prepare(
|
||||
"SELECT DISTINCT json_extract(properties, '$.templateId') as tid FROM telemetry_events WHERE event IN ('template_selected', 'workspace_created') AND json_extract(properties, '$.templateId') IS NOT NULL"
|
||||
).all() as Array<{ tid: string }>;
|
||||
|
||||
const breakdownRows = this.db.prepare(
|
||||
'SELECT event, COUNT(*) as cnt FROM telemetry_events GROUP BY event ORDER BY cnt DESC'
|
||||
).all() as Array<{ event: string; cnt: number }>;
|
||||
const breakdown: Record<string, number> = {};
|
||||
for (const row of breakdownRows) {
|
||||
breakdown[row.event] = row.cnt;
|
||||
}
|
||||
|
||||
return {
|
||||
enabled: this.enabled,
|
||||
totalEvents: total,
|
||||
firstEvent: first.d,
|
||||
lastEvent: last.d,
|
||||
onboardingCompleted: onboarded,
|
||||
totalSessions: sessions,
|
||||
embeddingProvider: embProvider,
|
||||
templatesUsed: tplRows.map(r => r.tid),
|
||||
eventBreakdown: breakdown,
|
||||
};
|
||||
}
|
||||
|
||||
/** Get raw events with optional filters. */
|
||||
getEvents(options?: { event?: string; since?: string; until?: string; limit?: number }): TelemetryEvent[] {
|
||||
const conditions: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
|
||||
if (options?.event) {
|
||||
conditions.push('event = ?');
|
||||
params.push(options.event);
|
||||
}
|
||||
if (options?.since) {
|
||||
conditions.push('created_at >= ?');
|
||||
params.push(options.since);
|
||||
}
|
||||
if (options?.until) {
|
||||
conditions.push('created_at <= ?');
|
||||
params.push(options.until);
|
||||
}
|
||||
|
||||
const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
|
||||
const limit = options?.limit ?? 100;
|
||||
|
||||
const rows = this.db.prepare(
|
||||
`SELECT id, event, properties, created_at FROM telemetry_events ${where} ORDER BY created_at DESC LIMIT ?`
|
||||
).all(...params, limit) as Array<{ id: number; event: string; properties: string; created_at: string }>;
|
||||
|
||||
return rows.map(r => ({
|
||||
id: r.id,
|
||||
event: r.event,
|
||||
properties: JSON.parse(r.properties) as Record<string, unknown>,
|
||||
created_at: r.created_at,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Delete all telemetry data (user right-to-delete). */
|
||||
clear(): { deleted: number } {
|
||||
const result = this.db.prepare('DELETE FROM telemetry_events').run();
|
||||
return { deleted: result.changes };
|
||||
}
|
||||
|
||||
/** Close the database connection. */
|
||||
close(): void {
|
||||
this.db.close();
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Collector ── */
|
||||
|
||||
const COLLECTOR_SCHEMA = `
|
||||
CREATE TABLE IF NOT EXISTS collector_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
category TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
count INTEGER NOT NULL DEFAULT 1,
|
||||
date TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_collector_date ON collector_events (date);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_collector_name_date ON collector_events (name, date);
|
||||
`;
|
||||
|
||||
/**
|
||||
* TelemetryCollector — higher-level daily-aggregated event tracker.
|
||||
*
|
||||
* Records tool usage, commands, errors, and capability gaps by day.
|
||||
* Privacy-safe: only category + name + count, no content or PII.
|
||||
*/
|
||||
export class TelemetryCollector {
|
||||
private db: DatabaseType;
|
||||
private enabled: boolean;
|
||||
private dataDir: string;
|
||||
|
||||
constructor(dataDir: string, enabled = false) {
|
||||
this.dataDir = dataDir;
|
||||
this.enabled = enabled;
|
||||
const dbPath = path.join(dataDir, 'telemetry.db');
|
||||
this.db = new Database(dbPath);
|
||||
this.db.pragma('journal_mode = WAL');
|
||||
this.db.exec(COLLECTOR_SCHEMA);
|
||||
}
|
||||
|
||||
recordToolUse(toolName: string): void {
|
||||
if (!this.enabled) return;
|
||||
this.upsertEvent('tool', toolName);
|
||||
}
|
||||
|
||||
recordCommand(command: string): void {
|
||||
if (!this.enabled) return;
|
||||
this.upsertEvent('command', command);
|
||||
}
|
||||
|
||||
recordError(errorName: string): void {
|
||||
if (!this.enabled) return;
|
||||
this.upsertEvent('error', errorName);
|
||||
}
|
||||
|
||||
recordCapabilityGap(capability: string): void {
|
||||
if (!this.enabled) return;
|
||||
this.upsertEvent('capability_gap', capability);
|
||||
}
|
||||
|
||||
recordSession(durationMs: number, interactionCount: number): void {
|
||||
if (!this.enabled) return;
|
||||
const today = new Date().toISOString().split('T')[0];
|
||||
this.db.prepare(
|
||||
`INSERT INTO collector_events (category, name, count, date) VALUES ('session', 'duration_total_ms', ?, ?)
|
||||
ON CONFLICT(name, date) DO UPDATE SET count = count + excluded.count`
|
||||
).run(durationMs, today);
|
||||
this.db.prepare(
|
||||
`INSERT INTO collector_events (category, name, count, date) VALUES ('session', 'interaction_count', ?, ?)
|
||||
ON CONFLICT(name, date) DO UPDATE SET count = count + excluded.count`
|
||||
).run(interactionCount, today);
|
||||
this.db.prepare(
|
||||
`INSERT INTO collector_events (category, name, count, date) VALUES ('session', 'session_count', 1, ?)
|
||||
ON CONFLICT(name, date) DO UPDATE SET count = count + 1`
|
||||
).run(today);
|
||||
}
|
||||
|
||||
private upsertEvent(category: string, name: string): void {
|
||||
const today = new Date().toISOString().split('T')[0];
|
||||
this.db.prepare(
|
||||
`INSERT INTO collector_events (category, name, count, date) VALUES (?, ?, 1, ?)
|
||||
ON CONFLICT(name, date) DO UPDATE SET count = count + 1`
|
||||
).run(category, name, today);
|
||||
}
|
||||
|
||||
getReport(days = 7): {
|
||||
totalEvents: number;
|
||||
events: Array<{ name: string; count: number; category: string; date: string }>;
|
||||
dateRange: { from: string | null };
|
||||
} {
|
||||
const since = new Date();
|
||||
since.setDate(since.getDate() - days);
|
||||
const sinceStr = since.toISOString().split('T')[0];
|
||||
|
||||
const rows = this.db.prepare(
|
||||
`SELECT name, SUM(count) as total_count, category, date
|
||||
FROM collector_events WHERE date >= ?
|
||||
GROUP BY name, category, date ORDER BY date DESC, name`
|
||||
).all(sinceStr) as Array<{ name: string; total_count: number; category: string; date: string }>;
|
||||
|
||||
const totalEvents = rows.reduce((sum, r) => sum + r.total_count, 0);
|
||||
const events = rows.map(r => ({ name: r.name, count: r.total_count, category: r.category, date: r.date }));
|
||||
const firstRow = this.db.prepare(
|
||||
'SELECT MIN(date) as first_date FROM collector_events'
|
||||
).get() as { first_date: string | null };
|
||||
|
||||
return { totalEvents, events, dateRange: { from: firstRow.first_date } };
|
||||
}
|
||||
|
||||
/** Write events to telemetry.json for external consumption. */
|
||||
flush(): void {
|
||||
const rows = this.db.prepare(
|
||||
'SELECT name, count, category, date FROM collector_events ORDER BY date DESC, name'
|
||||
).all() as Array<{ name: string; count: number; category: string; date: string }>;
|
||||
fs.writeFileSync(
|
||||
path.join(this.dataDir, 'telemetry.json'),
|
||||
JSON.stringify(rows, null, 2),
|
||||
);
|
||||
}
|
||||
|
||||
setEnabled(enabled: boolean): void { this.enabled = enabled; }
|
||||
isEnabled(): boolean { return this.enabled; }
|
||||
close(): void { this.db.close(); }
|
||||
}
|
||||
299
packages/core/src/vault.ts
Normal file
299
packages/core/src/vault.ts
Normal file
@@ -0,0 +1,299 @@
|
||||
/**
|
||||
* VaultStore — encrypted local secrets store.
|
||||
*
|
||||
* Encrypts secrets using AES-256-GCM (Node.js built-in crypto).
|
||||
* Stores encrypted data in vault.json with a machine-local key file.
|
||||
* Each entry is independently encrypted so individual secrets can be
|
||||
* updated without re-encrypting everything.
|
||||
*/
|
||||
|
||||
import * as crypto from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { createCoreLogger } from '@waggle/hive-mind-core';
|
||||
|
||||
const log = createCoreLogger('vault');
|
||||
const ALGORITHM = 'aes-256-gcm';
|
||||
const KEY_LENGTH = 32;
|
||||
const IV_LENGTH = 16;
|
||||
|
||||
export interface VaultEntry {
|
||||
name: string;
|
||||
value: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
interface VaultRecord {
|
||||
encrypted: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export class VaultStore {
|
||||
private dataDir: string;
|
||||
private vaultPath: string;
|
||||
private keyPath: string;
|
||||
private encryptionKey: Buffer;
|
||||
|
||||
constructor(dataDir: string) {
|
||||
this.dataDir = dataDir;
|
||||
this.vaultPath = path.join(dataDir, 'vault.json');
|
||||
this.keyPath = path.join(dataDir, '.vault-key');
|
||||
|
||||
// Ensure data directory exists
|
||||
if (!fs.existsSync(this.dataDir)) {
|
||||
fs.mkdirSync(this.dataDir, { recursive: true });
|
||||
}
|
||||
|
||||
this.encryptionKey = this.ensureKey();
|
||||
}
|
||||
|
||||
/** Ensure the encryption key exists. Generate if missing. */
|
||||
private ensureKey(): Buffer {
|
||||
if (fs.existsSync(this.keyPath)) {
|
||||
const key = Buffer.from(fs.readFileSync(this.keyPath, 'utf-8').trim(), 'hex');
|
||||
if (key.length !== KEY_LENGTH) {
|
||||
throw new Error(
|
||||
`Vault key file is corrupted — expected ${KEY_LENGTH} bytes, got ${key.length}. Delete ${this.keyPath} to regenerate.`
|
||||
);
|
||||
}
|
||||
return key;
|
||||
}
|
||||
const key = crypto.randomBytes(KEY_LENGTH);
|
||||
fs.writeFileSync(this.keyPath, key.toString('hex'), { mode: 0o600 });
|
||||
// Review Critical #1: On Windows, restrict key file access to current user only.
|
||||
// Previously used `require('node:child_process')` inline which fails silently under
|
||||
// ESM (`type: module` in the sidecar) — the try/catch swallowed the ReferenceError
|
||||
// and every Windows install left the vault key with no ACL restriction.
|
||||
// Now imported statically at the top of the file; the try/catch only covers actual
|
||||
// icacls failures (e.g. icacls.exe not on PATH in a minimal Windows image).
|
||||
if (process.platform === 'win32') {
|
||||
try {
|
||||
// Resolve the current user via whoami (safer than process.env.USERNAME
|
||||
// which can be absent or spoofed in containerized/scripted setups)
|
||||
const currentUser = execFileSync('whoami', { encoding: 'utf-8' }).trim();
|
||||
execFileSync('icacls', [
|
||||
this.keyPath,
|
||||
'/inheritance:r',
|
||||
'/grant:r',
|
||||
`${currentUser}:F`,
|
||||
], { stdio: 'ignore' });
|
||||
} catch (err) {
|
||||
log.warn('Could not restrict key file permissions via icacls — vault key may be readable by other users', err);
|
||||
}
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
/** Encrypt a plaintext string. Returns iv:authTag:ciphertext (all hex). */
|
||||
private encrypt(plaintext: string): string {
|
||||
const iv = crypto.randomBytes(IV_LENGTH);
|
||||
const cipher = crypto.createCipheriv(ALGORITHM, this.encryptionKey, iv);
|
||||
const encrypted = Buffer.concat([cipher.update(plaintext, 'utf-8'), cipher.final()]);
|
||||
const authTag = cipher.getAuthTag();
|
||||
return `${iv.toString('hex')}:${authTag.toString('hex')}:${encrypted.toString('hex')}`;
|
||||
}
|
||||
|
||||
/** Decrypt an encoded string (iv:authTag:ciphertext, all hex). */
|
||||
private decrypt(encoded: string): string {
|
||||
const parts = encoded.split(':');
|
||||
if (parts.length !== 3) {
|
||||
throw new Error('Vault entry format invalid — expected iv:authTag:ciphertext');
|
||||
}
|
||||
const [ivHex, authTagHex, ciphertextHex] = parts;
|
||||
const iv = Buffer.from(ivHex, 'hex');
|
||||
const authTag = Buffer.from(authTagHex, 'hex');
|
||||
const ciphertext = Buffer.from(ciphertextHex, 'hex');
|
||||
const decipher = crypto.createDecipheriv(ALGORITHM, this.encryptionKey, iv);
|
||||
decipher.setAuthTag(authTag);
|
||||
return decipher.update(ciphertext) + decipher.final('utf-8');
|
||||
}
|
||||
|
||||
/** Read the vault file (encrypted entries). */
|
||||
private readVault(): Record<string, VaultRecord> {
|
||||
if (!fs.existsSync(this.vaultPath)) return {};
|
||||
try {
|
||||
// Review Critical #3: use a null-prototype object so a malicious `__proto__`
|
||||
// key in vault.json cannot pollute Object.prototype when accessed. Matches
|
||||
// the `name` validation at the route layer.
|
||||
const parsed = JSON.parse(fs.readFileSync(this.vaultPath, 'utf-8'));
|
||||
return Object.assign(Object.create(null), parsed);
|
||||
} catch (err) {
|
||||
// M5: corrupt vault.json — back up the corrupt file so data isn't silently lost
|
||||
log.error('Failed to parse vault.json — backing up corrupt file', err);
|
||||
const bakPath = this.vaultPath + '.bak';
|
||||
try { fs.copyFileSync(this.vaultPath, bakPath); } catch { /* best effort */ }
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/** Write the vault file atomically (write to .tmp, then replace). */
|
||||
private writeVault(vault: Record<string, VaultRecord>): void {
|
||||
const tmpPath = this.vaultPath + '.tmp';
|
||||
const data = JSON.stringify(vault, null, 2);
|
||||
fs.writeFileSync(tmpPath, data, { mode: 0o600 });
|
||||
try {
|
||||
try {
|
||||
fs.renameSync(tmpPath, this.vaultPath);
|
||||
} catch {
|
||||
// M6: On Windows, rename fails with EPERM when target exists.
|
||||
// Delete target first, then rename — preserves atomicity better than direct write.
|
||||
try { fs.unlinkSync(this.vaultPath); } catch { /* target may not exist */ }
|
||||
try {
|
||||
fs.renameSync(tmpPath, this.vaultPath);
|
||||
} catch (renameErr) {
|
||||
// Last resort: direct write (non-atomic) — log the degradation
|
||||
log.error('Atomic vault write failed, falling back to direct write', renameErr);
|
||||
fs.writeFileSync(this.vaultPath, data, { mode: 0o600 });
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
// M12: clean up .tmp if it still exists after any failure path
|
||||
try { if (fs.existsSync(tmpPath)) fs.unlinkSync(tmpPath); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
/** Set a secret (encrypts value, stores metadata alongside). Uses atomic write. */
|
||||
set(name: string, value: string, metadata?: Record<string, unknown>): void {
|
||||
const vault = this.readVault();
|
||||
vault[name] = {
|
||||
encrypted: this.encrypt(value),
|
||||
metadata,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
this.writeVault(vault);
|
||||
}
|
||||
|
||||
/**
|
||||
* Async set — delegates to the synchronous set().
|
||||
* Review Critical #2: the previous implementation used a promise chain
|
||||
* (.writeLock.then(...)) which deferred the actual write to a microtask.
|
||||
* A sync set() between the chain call and microtask execution would be
|
||||
* overwritten when the deferred write fired. Since all vault I/O is
|
||||
* synchronous (fs.readFileSync/writeFileSync), the lock is unnecessary —
|
||||
* each callback runs to completion before the next microtask. Delegating
|
||||
* to the sync method eliminates the interleaving window entirely.
|
||||
*/
|
||||
async setAsync(name: string, value: string, metadata?: Record<string, unknown>): Promise<void> {
|
||||
this.set(name, value, metadata);
|
||||
}
|
||||
|
||||
/** Get a decrypted secret by name. Returns null if not found or decryption fails. */
|
||||
get(name: string): VaultEntry | null {
|
||||
const vault = this.readVault();
|
||||
const entry = vault[name];
|
||||
if (!entry) return null;
|
||||
try {
|
||||
return {
|
||||
name,
|
||||
value: this.decrypt(entry.encrypted),
|
||||
metadata: entry.metadata,
|
||||
updatedAt: entry.updatedAt,
|
||||
};
|
||||
} catch {
|
||||
return null; // Decryption failed (key mismatch, corrupted)
|
||||
}
|
||||
}
|
||||
|
||||
/** Delete a secret. Returns true if it existed. Uses atomic write. */
|
||||
delete(name: string): boolean {
|
||||
const vault = this.readVault();
|
||||
if (!vault[name]) return false;
|
||||
delete vault[name];
|
||||
this.writeVault(vault);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Async delete — delegates to the synchronous delete().
|
||||
* Review Critical #2: same rationale as setAsync — sync I/O means no
|
||||
* interleaving risk. The previous promise-chain approach deferred the
|
||||
* actual delete, creating a window where a sync caller's write could
|
||||
* be overwritten.
|
||||
*/
|
||||
async deleteAsync(name: string): Promise<boolean> {
|
||||
return this.delete(name);
|
||||
}
|
||||
|
||||
/** List secret names (without values). */
|
||||
list(): Array<{ name: string; metadata?: Record<string, unknown>; updatedAt: string }> {
|
||||
const vault = this.readVault();
|
||||
return Object.entries(vault).map(([name, entry]) => ({
|
||||
name,
|
||||
metadata: entry.metadata,
|
||||
updatedAt: entry.updatedAt,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Check if a secret exists. */
|
||||
has(name: string): boolean {
|
||||
const vault = this.readVault();
|
||||
return name in vault;
|
||||
}
|
||||
|
||||
/** Set a connector credential with typed metadata */
|
||||
setConnectorCredential(connectorId: string, credential: {
|
||||
type: 'api_key' | 'oauth2' | 'bearer' | 'basic';
|
||||
value: string;
|
||||
refreshToken?: string;
|
||||
expiresAt?: string;
|
||||
scopes?: string[];
|
||||
}): void {
|
||||
this.set(`connector:${connectorId}`, credential.value, {
|
||||
credentialType: credential.type,
|
||||
expiresAt: credential.expiresAt,
|
||||
scopes: credential.scopes,
|
||||
});
|
||||
// Store refresh token as a separate encrypted entry (never in plaintext metadata)
|
||||
if (credential.refreshToken) {
|
||||
this.set(`connector:${connectorId}:refresh`, credential.refreshToken);
|
||||
} else {
|
||||
// Clear any previously stored refresh token if not provided
|
||||
this.delete(`connector:${connectorId}:refresh`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Get a connector credential with typed metadata */
|
||||
getConnectorCredential(connectorId: string): {
|
||||
value: string;
|
||||
type: string;
|
||||
refreshToken?: string;
|
||||
expiresAt?: string;
|
||||
scopes?: string[];
|
||||
isExpired: boolean;
|
||||
} | null {
|
||||
const entry = this.get(`connector:${connectorId}`);
|
||||
if (!entry) return null;
|
||||
const expiresAt = entry.metadata?.expiresAt as string | undefined;
|
||||
// Retrieve refresh token from its own encrypted entry
|
||||
const refreshEntry = this.get(`connector:${connectorId}:refresh`);
|
||||
return {
|
||||
value: entry.value,
|
||||
type: (entry.metadata?.credentialType as string) ?? 'api_key',
|
||||
// Review Major #7: removed dead plaintext metadata fallback that contradicted
|
||||
// the "never in plaintext metadata" security contract from setConnectorCredential.
|
||||
refreshToken: refreshEntry?.value,
|
||||
expiresAt,
|
||||
scopes: entry.metadata?.scopes as string[] | undefined,
|
||||
isExpired: expiresAt ? new Date(expiresAt) < new Date() : false,
|
||||
};
|
||||
}
|
||||
|
||||
/** Migrate plaintext providers from config.json to vault. Returns count migrated. */
|
||||
migrateFromConfig(config: { providers?: Record<string, { apiKey: string; models?: string[]; baseUrl?: string }> }): number {
|
||||
if (!config.providers) return 0;
|
||||
let migrated = 0;
|
||||
for (const [name, provider] of Object.entries(config.providers)) {
|
||||
if (provider.apiKey && !this.has(name)) {
|
||||
this.set(name, provider.apiKey, {
|
||||
models: provider.models,
|
||||
baseUrl: provider.baseUrl,
|
||||
});
|
||||
migrated++;
|
||||
}
|
||||
}
|
||||
return migrated;
|
||||
}
|
||||
}
|
||||
254
packages/core/tests/compliance/template-store.test.ts
Normal file
254
packages/core/tests/compliance/template-store.test.ts
Normal file
@@ -0,0 +1,254 @@
|
||||
/**
|
||||
* ComplianceTemplateStore unit tests (M-03)
|
||||
*
|
||||
* Covers CRUD + section merge semantics + default section fill-in + risk-class CHECK
|
||||
* constraint + deletion. Following HarvestRunStore test shape (in-memory MindDB).
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { MindDB } from '@waggle/hive-mind-core';
|
||||
import { ComplianceTemplateStore, KVARK_TEMPLATE_NAME } from '../../src/compliance/template-store.js';
|
||||
import type { ComplianceTemplateSections } from '../../src/compliance/types.js';
|
||||
|
||||
const ALL_ON: ComplianceTemplateSections = {
|
||||
interactions: true,
|
||||
oversight: true,
|
||||
models: true,
|
||||
provenance: true,
|
||||
riskAssessment: true,
|
||||
fria: true,
|
||||
};
|
||||
const ALL_OFF: ComplianceTemplateSections = {
|
||||
interactions: false,
|
||||
oversight: false,
|
||||
models: false,
|
||||
provenance: false,
|
||||
riskAssessment: false,
|
||||
fria: false,
|
||||
};
|
||||
|
||||
describe('ComplianceTemplateStore', () => {
|
||||
let db: MindDB;
|
||||
let store: ComplianceTemplateStore;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
store = new ComplianceTemplateStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
describe('create', () => {
|
||||
it('persists a template with all fields', () => {
|
||||
const t = store.create({
|
||||
name: 'KVARK enterprise',
|
||||
description: 'Full AI Act package for enterprise on-prem',
|
||||
sections: ALL_ON,
|
||||
riskClassification: 'high-risk',
|
||||
orgName: 'KVARK Sovereign',
|
||||
footerText: 'Confidential — internal only',
|
||||
});
|
||||
expect(t.id).toBeGreaterThan(0);
|
||||
expect(t.name).toBe('KVARK enterprise');
|
||||
expect(t.description).toBe('Full AI Act package for enterprise on-prem');
|
||||
expect(t.sections).toEqual(ALL_ON);
|
||||
expect(t.riskClassification).toBe('high-risk');
|
||||
expect(t.orgName).toBe('KVARK Sovereign');
|
||||
expect(t.footerText).toBe('Confidential — internal only');
|
||||
expect(t.createdAt).toBeTruthy();
|
||||
expect(t.updatedAt).toBeTruthy();
|
||||
});
|
||||
|
||||
it('defaults optional fields to null', () => {
|
||||
const t = store.create({ name: 'Bare', sections: ALL_OFF });
|
||||
expect(t.description).toBeNull();
|
||||
expect(t.riskClassification).toBeNull();
|
||||
expect(t.orgName).toBeNull();
|
||||
expect(t.footerText).toBeNull();
|
||||
});
|
||||
|
||||
it('trims whitespace from name', () => {
|
||||
const t = store.create({ name: ' Spaced ', sections: ALL_OFF });
|
||||
expect(t.name).toBe('Spaced');
|
||||
});
|
||||
|
||||
it('rejects empty name', () => {
|
||||
expect(() => store.create({ name: ' ', sections: ALL_OFF })).toThrow(/name is required/i);
|
||||
});
|
||||
|
||||
it('fills in missing section keys with defaults', () => {
|
||||
const t = store.create({
|
||||
name: 'Partial',
|
||||
// Only interactions specified; others should fall back to DEFAULT_SECTIONS.
|
||||
sections: { interactions: false } as ComplianceTemplateSections,
|
||||
});
|
||||
expect(t.sections.interactions).toBe(false);
|
||||
expect(t.sections.oversight).toBe(true); // default
|
||||
expect(t.sections.fria).toBe(false); // default
|
||||
});
|
||||
});
|
||||
|
||||
describe('getById', () => {
|
||||
it('returns null for missing id', () => {
|
||||
expect(store.getById(999)).toBeNull();
|
||||
});
|
||||
|
||||
it('round-trips a created template', () => {
|
||||
const a = store.create({ name: 'A', sections: ALL_ON, riskClassification: 'limited' });
|
||||
const b = store.getById(a.id);
|
||||
expect(b).not.toBeNull();
|
||||
expect(b?.name).toBe('A');
|
||||
expect(b?.sections).toEqual(ALL_ON);
|
||||
expect(b?.riskClassification).toBe('limited');
|
||||
});
|
||||
});
|
||||
|
||||
describe('list', () => {
|
||||
it('returns empty array when none exist', () => {
|
||||
expect(store.list()).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns templates newest-updated first', async () => {
|
||||
store.create({ name: 'Old', sections: ALL_OFF });
|
||||
await new Promise(resolve => setTimeout(resolve, 1100)); // datetime('now') is second-resolution
|
||||
store.create({ name: 'New', sections: ALL_OFF });
|
||||
const list = store.list();
|
||||
expect(list).toHaveLength(2);
|
||||
expect(list[0].name).toBe('New');
|
||||
expect(list[1].name).toBe('Old');
|
||||
});
|
||||
});
|
||||
|
||||
describe('update', () => {
|
||||
it('returns null for missing id', () => {
|
||||
expect(store.update(999, { name: 'nope' })).toBeNull();
|
||||
});
|
||||
|
||||
it('updates only the fields provided', () => {
|
||||
const t = store.create({
|
||||
name: 'Orig',
|
||||
description: 'desc',
|
||||
sections: ALL_ON,
|
||||
orgName: 'Acme',
|
||||
});
|
||||
const updated = store.update(t.id, { name: 'Renamed' });
|
||||
expect(updated?.name).toBe('Renamed');
|
||||
expect(updated?.description).toBe('desc'); // preserved
|
||||
expect(updated?.sections).toEqual(ALL_ON); // preserved
|
||||
expect(updated?.orgName).toBe('Acme'); // preserved
|
||||
});
|
||||
|
||||
it('clears a field when explicitly set to null', () => {
|
||||
const t = store.create({
|
||||
name: 'T',
|
||||
sections: ALL_ON,
|
||||
orgName: 'To remove',
|
||||
footerText: 'also remove',
|
||||
});
|
||||
const updated = store.update(t.id, { orgName: null, footerText: null });
|
||||
expect(updated?.orgName).toBeNull();
|
||||
expect(updated?.footerText).toBeNull();
|
||||
});
|
||||
|
||||
it('replaces sections wholesale', () => {
|
||||
const t = store.create({ name: 'T', sections: ALL_ON });
|
||||
const updated = store.update(t.id, { sections: ALL_OFF });
|
||||
expect(updated?.sections).toEqual(ALL_OFF);
|
||||
});
|
||||
|
||||
it('rejects an empty rename', () => {
|
||||
const t = store.create({ name: 'T', sections: ALL_OFF });
|
||||
expect(() => store.update(t.id, { name: ' ' })).toThrow(/name is required/i);
|
||||
});
|
||||
|
||||
it('bumps updated_at', async () => {
|
||||
const t = store.create({ name: 'T', sections: ALL_OFF });
|
||||
await new Promise(resolve => setTimeout(resolve, 1100));
|
||||
const updated = store.update(t.id, { name: 'T2' });
|
||||
expect(updated?.updatedAt).not.toBe(t.updatedAt);
|
||||
});
|
||||
});
|
||||
|
||||
describe('delete', () => {
|
||||
it('returns false for missing id', () => {
|
||||
expect(store.delete(999)).toBe(false);
|
||||
});
|
||||
|
||||
it('removes an existing row and returns true', () => {
|
||||
const t = store.create({ name: 'T', sections: ALL_OFF });
|
||||
expect(store.delete(t.id)).toBe(true);
|
||||
expect(store.getById(t.id)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('risk_classification CHECK constraint', () => {
|
||||
it('accepts all four AIActRiskLevel values', () => {
|
||||
for (const level of ['minimal', 'limited', 'high-risk', 'unacceptable'] as const) {
|
||||
const t = store.create({ name: `T-${level}`, sections: ALL_OFF, riskClassification: level });
|
||||
expect(t.riskClassification).toBe(level);
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects an invalid risk class at the SQL layer', () => {
|
||||
expect(() =>
|
||||
store.create({
|
||||
name: 'Bad',
|
||||
sections: ALL_OFF,
|
||||
riskClassification: 'super-dangerous' as never,
|
||||
}),
|
||||
).toThrow(); // better-sqlite3 surfaces the CHECK violation
|
||||
});
|
||||
});
|
||||
|
||||
describe('seedKvarkTemplateIfMissing (M-06)', () => {
|
||||
it('creates the KVARK template on first run', () => {
|
||||
const seeded = store.seedKvarkTemplateIfMissing();
|
||||
expect(seeded).not.toBeNull();
|
||||
expect(seeded!.name).toBe(KVARK_TEMPLATE_NAME);
|
||||
expect(seeded!.riskClassification).toBe('high-risk');
|
||||
expect(seeded!.sections.fria).toBe(true);
|
||||
expect(seeded!.orgName).toContain('KVARK');
|
||||
expect(seeded!.footerText).toContain('sovereign');
|
||||
});
|
||||
|
||||
it('is idempotent — second call returns null', () => {
|
||||
store.seedKvarkTemplateIfMissing();
|
||||
expect(store.seedKvarkTemplateIfMissing()).toBeNull();
|
||||
// Only one KVARK row exists.
|
||||
const all = store.list().filter(t => t.name === KVARK_TEMPLATE_NAME);
|
||||
expect(all).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('does not seed when the user already created a same-named template', () => {
|
||||
store.create({
|
||||
name: KVARK_TEMPLATE_NAME,
|
||||
sections: ALL_OFF,
|
||||
});
|
||||
expect(store.seedKvarkTemplateIfMissing()).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('mergeSections (static)', () => {
|
||||
it('unions template + runtime flags', () => {
|
||||
const template: ComplianceTemplateSections = { ...ALL_OFF, interactions: true, fria: true };
|
||||
const runtime: ComplianceTemplateSections = { ...ALL_OFF, oversight: true };
|
||||
const merged = ComplianceTemplateStore.mergeSections(template, runtime);
|
||||
expect(merged.interactions).toBe(true); // from template
|
||||
expect(merged.oversight).toBe(true); // from runtime
|
||||
expect(merged.fria).toBe(true); // from template
|
||||
expect(merged.models).toBe(false);
|
||||
});
|
||||
|
||||
it('never hides a section the runtime requested (union semantics)', () => {
|
||||
const merged = ComplianceTemplateStore.mergeSections(ALL_OFF, ALL_ON);
|
||||
expect(merged).toEqual(ALL_ON);
|
||||
});
|
||||
|
||||
it('never hides a section the template requested', () => {
|
||||
const merged = ComplianceTemplateStore.mergeSections(ALL_ON, ALL_OFF);
|
||||
expect(merged).toEqual(ALL_ON);
|
||||
});
|
||||
});
|
||||
});
|
||||
245
packages/core/tests/config.test.ts
Normal file
245
packages/core/tests/config.test.ts
Normal file
@@ -0,0 +1,245 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { WaggleConfig, type ProviderEntry, type TeamServerConfig } from '../src/config.js';
|
||||
|
||||
describe('WaggleConfig', () => {
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
function makeTempDir(): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-config-test-'));
|
||||
tempDirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of tempDirs) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
tempDirs.length = 0;
|
||||
});
|
||||
|
||||
it('creates config directory if missing', () => {
|
||||
const base = makeTempDir();
|
||||
const configDir = path.join(base, 'nested', '.waggle');
|
||||
|
||||
new WaggleConfig(configDir);
|
||||
|
||||
expect(fs.existsSync(configDir)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns default config when no file exists', () => {
|
||||
const configDir = makeTempDir();
|
||||
const config = new WaggleConfig(configDir);
|
||||
|
||||
expect(config.getDefaultModel()).toBe('claude-sonnet-4-6');
|
||||
expect(config.getProviders()).toEqual({});
|
||||
expect(config.getMindPath()).toBe(path.join(configDir, 'default.mind'));
|
||||
});
|
||||
|
||||
it('saves and loads provider config', () => {
|
||||
const configDir = makeTempDir();
|
||||
const config = new WaggleConfig(configDir);
|
||||
|
||||
const provider: ProviderEntry = {
|
||||
apiKey: 'sk-test-key',
|
||||
models: ['claude-sonnet-4-6', 'claude-haiku-3'],
|
||||
baseUrl: 'https://api.anthropic.com',
|
||||
};
|
||||
|
||||
config.setProvider('anthropic', provider);
|
||||
config.save();
|
||||
|
||||
// Load fresh instance from same directory
|
||||
const config2 = new WaggleConfig(configDir);
|
||||
const providers = config2.getProviders();
|
||||
|
||||
expect(providers['anthropic']).toEqual(provider);
|
||||
expect(providers['anthropic'].apiKey).toBe('sk-test-key');
|
||||
expect(providers['anthropic'].models).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('sets and gets default model', () => {
|
||||
const configDir = makeTempDir();
|
||||
const config = new WaggleConfig(configDir);
|
||||
|
||||
config.setDefaultModel('gpt-4o');
|
||||
config.save();
|
||||
|
||||
const config2 = new WaggleConfig(configDir);
|
||||
expect(config2.getDefaultModel()).toBe('gpt-4o');
|
||||
});
|
||||
|
||||
it('returns mind file path', () => {
|
||||
const configDir = makeTempDir();
|
||||
const config = new WaggleConfig(configDir);
|
||||
|
||||
expect(config.getMindPath()).toBe(path.join(configDir, 'default.mind'));
|
||||
});
|
||||
|
||||
it('removes a provider', () => {
|
||||
const configDir = makeTempDir();
|
||||
const config = new WaggleConfig(configDir);
|
||||
|
||||
config.setProvider('anthropic', { apiKey: 'key1', models: ['m1'] });
|
||||
config.setProvider('openai', { apiKey: 'key2', models: ['m2'] });
|
||||
config.removeProvider('anthropic');
|
||||
config.save();
|
||||
|
||||
const config2 = new WaggleConfig(configDir);
|
||||
const providers = config2.getProviders();
|
||||
expect(providers['anthropic']).toBeUndefined();
|
||||
expect(providers['openai']).toBeDefined();
|
||||
});
|
||||
|
||||
it('returns config directory path', () => {
|
||||
const configDir = makeTempDir();
|
||||
const config = new WaggleConfig(configDir);
|
||||
|
||||
expect(config.getConfigDir()).toBe(configDir);
|
||||
});
|
||||
|
||||
describe('team server config', () => {
|
||||
it('returns null when no team server configured', () => {
|
||||
const configDir = makeTempDir();
|
||||
const config = new WaggleConfig(configDir);
|
||||
|
||||
expect(config.getTeamServer()).toBeNull();
|
||||
expect(config.isTeamConnected()).toBe(false);
|
||||
});
|
||||
|
||||
it('sets and gets team server config', () => {
|
||||
const configDir = makeTempDir();
|
||||
const config = new WaggleConfig(configDir);
|
||||
|
||||
const teamConfig: TeamServerConfig = {
|
||||
url: 'https://team.waggle.dev',
|
||||
token: 'clerk-jwt-token',
|
||||
userId: 'user-123',
|
||||
displayName: 'Marko',
|
||||
};
|
||||
|
||||
config.setTeamServer(teamConfig);
|
||||
config.save();
|
||||
|
||||
const config2 = new WaggleConfig(configDir);
|
||||
const loaded = config2.getTeamServer();
|
||||
expect(loaded).toEqual(teamConfig);
|
||||
expect(config2.isTeamConnected()).toBe(true);
|
||||
});
|
||||
|
||||
it('clears team server config', () => {
|
||||
const configDir = makeTempDir();
|
||||
const config = new WaggleConfig(configDir);
|
||||
|
||||
config.setTeamServer({ url: 'https://team.waggle.dev' });
|
||||
config.clearTeamServer();
|
||||
config.save();
|
||||
|
||||
const config2 = new WaggleConfig(configDir);
|
||||
expect(config2.getTeamServer()).toBeNull();
|
||||
expect(config2.isTeamConnected()).toBe(false);
|
||||
});
|
||||
|
||||
it('persists team server through save/load cycle', () => {
|
||||
const configDir = makeTempDir();
|
||||
const config = new WaggleConfig(configDir);
|
||||
|
||||
config.setTeamServer({ url: 'https://example.com', userId: 'u1' });
|
||||
config.save();
|
||||
|
||||
const config2 = new WaggleConfig(configDir);
|
||||
expect(config2.getTeamServer()!.url).toBe('https://example.com');
|
||||
expect(config2.getTeamServer()!.userId).toBe('u1');
|
||||
});
|
||||
});
|
||||
|
||||
describe('governed CLI config', () => {
|
||||
it('normalizes, deduplicates, and persists the CLI allowlist', () => {
|
||||
const configDir = makeTempDir();
|
||||
const config = new WaggleConfig(configDir);
|
||||
|
||||
config.setCliAllowlist([' node ', 'NODE', '', 'git']);
|
||||
expect(config.getCliAllowlist()).toEqual(['node', 'git']);
|
||||
config.save();
|
||||
|
||||
const config2 = new WaggleConfig(configDir);
|
||||
expect(config2.getCliAllowlist()).toEqual(['node', 'git']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Model Pilot config fields', () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-config-pilot-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('returns null for fallbackModel when not set', () => {
|
||||
const config = new WaggleConfig(tmpDir);
|
||||
expect(config.getFallbackModel()).toBeNull();
|
||||
});
|
||||
|
||||
it('persists fallbackModel', () => {
|
||||
const config = new WaggleConfig(tmpDir);
|
||||
config.setFallbackModel('qwen/qwen3.6-plus:free');
|
||||
config.save();
|
||||
const config2 = new WaggleConfig(tmpDir);
|
||||
expect(config2.getFallbackModel()).toBe('qwen/qwen3.6-plus:free');
|
||||
});
|
||||
|
||||
it('returns null for budgetModel when not set', () => {
|
||||
const config = new WaggleConfig(tmpDir);
|
||||
expect(config.getBudgetModel()).toBeNull();
|
||||
});
|
||||
|
||||
it('persists budgetModel', () => {
|
||||
const config = new WaggleConfig(tmpDir);
|
||||
config.setBudgetModel('deepseek/deepseek-chat-v3-0324:free');
|
||||
config.save();
|
||||
const config2 = new WaggleConfig(tmpDir);
|
||||
expect(config2.getBudgetModel()).toBe('deepseek/deepseek-chat-v3-0324:free');
|
||||
});
|
||||
|
||||
it('returns 0.8 as default budgetThreshold', () => {
|
||||
const config = new WaggleConfig(tmpDir);
|
||||
expect(config.getBudgetThreshold()).toBe(0.8);
|
||||
});
|
||||
|
||||
it('persists budgetThreshold', () => {
|
||||
const config = new WaggleConfig(tmpDir);
|
||||
config.setBudgetThreshold(0.6);
|
||||
config.save();
|
||||
const config2 = new WaggleConfig(tmpDir);
|
||||
expect(config2.getBudgetThreshold()).toBe(0.6);
|
||||
});
|
||||
|
||||
it('clearFallbackModel removes the field', () => {
|
||||
const config = new WaggleConfig(tmpDir);
|
||||
config.setFallbackModel('test-model');
|
||||
config.save();
|
||||
config.clearFallbackModel();
|
||||
config.save();
|
||||
const config2 = new WaggleConfig(tmpDir);
|
||||
expect(config2.getFallbackModel()).toBeNull();
|
||||
});
|
||||
|
||||
it('returns 90 as default maxIterations', () => {
|
||||
const config = new WaggleConfig(tmpDir);
|
||||
expect(config.getMaxIterations()).toBe(90);
|
||||
});
|
||||
|
||||
it('persists maxIterations', () => {
|
||||
const config = new WaggleConfig(tmpDir);
|
||||
config.setMaxIterations(50);
|
||||
config.save();
|
||||
const config2 = new WaggleConfig(tmpDir);
|
||||
expect(config2.getMaxIterations()).toBe(50);
|
||||
});
|
||||
});
|
||||
});
|
||||
268
packages/core/tests/cron-store.test.ts
Normal file
268
packages/core/tests/cron-store.test.ts
Normal file
@@ -0,0 +1,268 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { MindDB } from '@waggle/hive-mind-core';
|
||||
import { CronStore, type CreateScheduleInput, type SavePendingActionInput } from '../src/cron-store.js';
|
||||
|
||||
describe('CronStore', () => {
|
||||
let tmpDir: string;
|
||||
let db: MindDB;
|
||||
let store: CronStore;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-cron-'));
|
||||
db = new MindDB(path.join(tmpDir, 'test.mind'));
|
||||
store = new CronStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function makeInput(overrides?: Partial<CreateScheduleInput>): CreateScheduleInput {
|
||||
return {
|
||||
name: 'Daily backup',
|
||||
cronExpr: '0 9 * * *', // every day at 9am
|
||||
jobType: 'memory_consolidation',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
it('create with valid cron computes next_run_at in the future', () => {
|
||||
const schedule = store.create(makeInput());
|
||||
expect(schedule.id).toBeGreaterThan(0);
|
||||
expect(schedule.name).toBe('Daily backup');
|
||||
expect(schedule.cron_expr).toBe('0 9 * * *');
|
||||
expect(schedule.job_type).toBe('memory_consolidation');
|
||||
expect(schedule.enabled).toBe(1);
|
||||
expect(schedule.next_run_at).toBeTruthy();
|
||||
// next_run_at should be in the future
|
||||
expect(new Date(schedule.next_run_at!).getTime()).toBeGreaterThan(Date.now());
|
||||
});
|
||||
|
||||
it('create with invalid cron expression throws', () => {
|
||||
expect(() => store.create(makeInput({ cronExpr: 'not a cron' }))).toThrow();
|
||||
});
|
||||
|
||||
it('create agent_task without workspaceId throws', () => {
|
||||
expect(() =>
|
||||
store.create(makeInput({ jobType: 'agent_task' })),
|
||||
).toThrow(/workspace/i);
|
||||
});
|
||||
|
||||
it('create loop job type round-trips with its job_config', () => {
|
||||
const schedule = store.create(makeInput({
|
||||
jobType: 'loop',
|
||||
jobConfig: { prompt: 'Summarize what changed in this workspace.' },
|
||||
}));
|
||||
expect(schedule.job_type).toBe('loop');
|
||||
expect(JSON.parse(schedule.job_config).prompt).toBe('Summarize what changed in this workspace.');
|
||||
});
|
||||
|
||||
it('create loop without workspaceId succeeds (loops can run on the personal mind)', () => {
|
||||
const schedule = store.create(makeInput({ jobType: 'loop' }));
|
||||
expect(schedule.job_type).toBe('loop');
|
||||
expect(schedule.workspace_id).toBeNull();
|
||||
});
|
||||
|
||||
it('create agent_task with workspaceId succeeds', () => {
|
||||
const schedule = store.create(makeInput({
|
||||
jobType: 'agent_task',
|
||||
workspaceId: 'ws-123',
|
||||
}));
|
||||
expect(schedule.job_type).toBe('agent_task');
|
||||
expect(schedule.workspace_id).toBe('ws-123');
|
||||
});
|
||||
|
||||
it('list returns schedules ordered by name', () => {
|
||||
store.create(makeInput({ name: 'Zebra task' }));
|
||||
store.create(makeInput({ name: 'Alpha task' }));
|
||||
store.create(makeInput({ name: 'Middle task' }));
|
||||
|
||||
const list = store.list();
|
||||
expect(list).toHaveLength(3);
|
||||
expect(list[0].name).toBe('Alpha task');
|
||||
expect(list[1].name).toBe('Middle task');
|
||||
expect(list[2].name).toBe('Zebra task');
|
||||
});
|
||||
|
||||
it('getById returns the schedule when found', () => {
|
||||
const created = store.create(makeInput({ name: 'Findable' }));
|
||||
const found = store.getById(created.id);
|
||||
expect(found).toBeDefined();
|
||||
expect(found!.name).toBe('Findable');
|
||||
});
|
||||
|
||||
it('getById returns undefined when not found', () => {
|
||||
const found = store.getById(99999);
|
||||
expect(found).toBeUndefined();
|
||||
});
|
||||
|
||||
it('update changes name and enabled', () => {
|
||||
const created = store.create(makeInput());
|
||||
store.update(created.id, { name: 'Renamed', enabled: false });
|
||||
|
||||
const updated = store.getById(created.id)!;
|
||||
expect(updated.name).toBe('Renamed');
|
||||
expect(updated.enabled).toBe(0);
|
||||
});
|
||||
|
||||
it('update cronExpr recomputes next_run_at', () => {
|
||||
const created = store.create(makeInput({ cronExpr: '0 9 * * *' }));
|
||||
const originalNext = created.next_run_at;
|
||||
|
||||
// Change to every minute — next_run_at should change
|
||||
store.update(created.id, { cronExpr: '*/1 * * * *' });
|
||||
const updated = store.getById(created.id)!;
|
||||
expect(updated.cron_expr).toBe('*/1 * * * *');
|
||||
expect(updated.next_run_at).toBeTruthy();
|
||||
// The new next_run_at should differ (different schedule)
|
||||
// Both should be valid ISO dates
|
||||
expect(new Date(updated.next_run_at!).getTime()).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('delete removes the schedule', () => {
|
||||
const created = store.create(makeInput());
|
||||
store.delete(created.id);
|
||||
expect(store.getById(created.id)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('getDue returns only enabled past-due schedules', () => {
|
||||
const raw = db.getDatabase();
|
||||
|
||||
// Create two schedules: one past-due, one future
|
||||
store.create(makeInput({ name: 'Future job' }));
|
||||
|
||||
// Manually insert a past-due schedule
|
||||
raw.prepare(`
|
||||
INSERT INTO cron_schedules (name, cron_expr, job_type, job_config, enabled, next_run_at, created_at)
|
||||
VALUES (?, ?, ?, ?, 1, datetime('now', '-1 hour'), datetime('now'))
|
||||
`).run('Past due job', '0 9 * * *', 'memory_consolidation', '{}');
|
||||
|
||||
const due = store.getDue();
|
||||
expect(due).toHaveLength(1);
|
||||
expect(due[0].name).toBe('Past due job');
|
||||
});
|
||||
|
||||
it('getDue excludes disabled schedules', () => {
|
||||
const raw = db.getDatabase();
|
||||
|
||||
// Insert a past-due but disabled schedule
|
||||
raw.prepare(`
|
||||
INSERT INTO cron_schedules (name, cron_expr, job_type, job_config, enabled, next_run_at, created_at)
|
||||
VALUES (?, ?, ?, ?, 0, datetime('now', '-1 hour'), datetime('now'))
|
||||
`).run('Disabled job', '0 9 * * *', 'memory_consolidation', '{}');
|
||||
|
||||
const due = store.getDue();
|
||||
expect(due).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('markRun updates last_run_at and recomputes next_run_at', () => {
|
||||
const created = store.create(makeInput({ cronExpr: '0 9 * * *' }));
|
||||
expect(created.last_run_at).toBeNull();
|
||||
|
||||
store.markRun(created.id);
|
||||
const updated = store.getById(created.id)!;
|
||||
expect(updated.last_run_at).toBeTruthy();
|
||||
expect(updated.next_run_at).toBeTruthy();
|
||||
// next_run_at should be in the future
|
||||
expect(new Date(updated.next_run_at!).getTime()).toBeGreaterThan(Date.now());
|
||||
});
|
||||
|
||||
it('pruneExecutionHistory deletes only rows older than the cutoff', () => {
|
||||
const created = store.create(makeInput());
|
||||
// A fresh row (executed_at = now) must survive the prune.
|
||||
store.recordExecution(created.id, created.name, { success: true });
|
||||
// A back-dated row beyond the 30-day retention must go.
|
||||
db.getDatabase().prepare(`
|
||||
INSERT INTO cron_execution_history (schedule_id, schedule_name, executed_at, success)
|
||||
VALUES (?, ?, datetime('now', '-40 days'), 1)
|
||||
`).run(created.id, created.name);
|
||||
expect(store.getExecutionHistory(created.id)).toHaveLength(2);
|
||||
|
||||
const deleted = store.pruneExecutionHistory(30);
|
||||
expect(deleted).toBe(1);
|
||||
|
||||
const remaining = store.getExecutionHistory(created.id);
|
||||
expect(remaining).toHaveLength(1);
|
||||
expect(remaining[0].success).toBe(1);
|
||||
});
|
||||
|
||||
it('countExecutionsToday counts only today, only this schedule (#17 daily cap)', () => {
|
||||
const created = store.create(makeInput());
|
||||
const other = store.create(makeInput({ name: 'Other job' }));
|
||||
|
||||
store.recordExecution(created.id, created.name, { success: true });
|
||||
store.recordExecution(created.id, created.name, { success: false });
|
||||
store.recordExecution(other.id, other.name, { success: true });
|
||||
// Yesterday's row must not count.
|
||||
db.getDatabase().prepare(`
|
||||
INSERT INTO cron_execution_history (schedule_id, schedule_name, executed_at, success)
|
||||
VALUES (?, ?, datetime('now', '-1 day'), 1)
|
||||
`).run(created.id, created.name);
|
||||
|
||||
expect(store.countExecutionsToday(created.id)).toBe(2);
|
||||
expect(store.countExecutionsToday(other.id)).toBe(1);
|
||||
});
|
||||
|
||||
describe('pending_actions (L2 held-action queue)', () => {
|
||||
function held(over?: Partial<SavePendingActionInput>): SavePendingActionInput {
|
||||
return {
|
||||
id: 'pa-1', workspaceId: null, source: 'loop:1', toolName: 'send_email',
|
||||
argsJson: JSON.stringify({ to: 'x@y.z', subject: 'hi' }), summary: 'Send follow-up',
|
||||
riskLevel: 'medium', approvalClass: 'elevated', ...over,
|
||||
};
|
||||
}
|
||||
|
||||
it('saves and lists held actions, filtered by status', () => {
|
||||
store.savePendingAction(held());
|
||||
store.savePendingAction(held({ id: 'pa-2', toolName: 'write_file' }));
|
||||
const heldRows = store.listPendingActions('held');
|
||||
expect(heldRows).toHaveLength(2);
|
||||
expect(heldRows.map(r => r.tool_name).sort()).toEqual(['send_email', 'write_file']);
|
||||
expect(store.listPendingActions('executed')).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('claimPendingAction is an atomic idempotency gate (double-claim no-ops)', () => {
|
||||
store.savePendingAction(held());
|
||||
const first = store.claimPendingAction('pa-1', 'approved', '2026-06-29T10:00:00Z');
|
||||
expect(first?.status).toBe('approved');
|
||||
expect(first?.decided_at).toBe('2026-06-29T10:00:00Z');
|
||||
// A second claim (double-approve / approve-after-deny) wins nothing.
|
||||
expect(store.claimPendingAction('pa-1', 'approved', '2026-06-29T10:05:00Z')).toBeUndefined();
|
||||
expect(store.claimPendingAction('pa-1', 'denied', '2026-06-29T10:05:00Z')).toBeUndefined();
|
||||
// Unknown id → undefined.
|
||||
expect(store.claimPendingAction('nope', 'approved', '2026-06-29T10:00:00Z')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('updatePendingActionResult records the terminal outcome after a claim', () => {
|
||||
store.savePendingAction(held());
|
||||
store.claimPendingAction('pa-1', 'approved', '2026-06-29T10:00:00Z');
|
||||
store.updatePendingActionResult('pa-1', { status: 'executed', resultSummary: 'sent', executedAt: '2026-06-29T10:01:00Z' });
|
||||
const row = store.getPendingAction('pa-1');
|
||||
expect(row?.status).toBe('executed');
|
||||
expect(row?.result_summary).toBe('sent');
|
||||
expect(row?.executed_at).toBe('2026-06-29T10:01:00Z');
|
||||
});
|
||||
|
||||
it('persists held actions across a DB reopen (durable queue)', () => {
|
||||
store.savePendingAction(held());
|
||||
db.close();
|
||||
db = new MindDB(path.join(tmpDir, 'test.mind'));
|
||||
store = new CronStore(db);
|
||||
const rows = store.listPendingActions('held');
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].id).toBe('pa-1');
|
||||
});
|
||||
|
||||
it('expireStalePendingActions flips past-due held rows to expired', () => {
|
||||
store.savePendingAction(held({ expiresAt: '2000-01-01T00:00:00Z' })); // long past
|
||||
store.savePendingAction(held({ id: 'pa-2', expiresAt: '2999-01-01T00:00:00Z' })); // future
|
||||
expect(store.expireStalePendingActions()).toBe(1);
|
||||
expect(store.listPendingActions('held').map(r => r.id)).toEqual(['pa-2']);
|
||||
expect(store.listPendingActions('expired').map(r => r.id)).toEqual(['pa-1']);
|
||||
});
|
||||
});
|
||||
});
|
||||
267
packages/core/tests/embedding-provider-quota.test.ts
Normal file
267
packages/core/tests/embedding-provider-quota.test.ts
Normal file
@@ -0,0 +1,267 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import Database from 'better-sqlite3';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { createEmbeddingProvider, EmbeddingQuotaExceededError, getMinimumTierForProvider } from '@waggle/hive-mind-core';
|
||||
import { TierError, TIER_CAPABILITIES } from '@waggle/shared';
|
||||
|
||||
describe('Embedding Provider — Tier & Quota Enforcement', () => {
|
||||
let tmpDir: string;
|
||||
let db: InstanceType<typeof Database>;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-embed-test-'));
|
||||
db = new Database(path.join(tmpDir, 'quota.db'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('Tier enforcement on provider selection', () => {
|
||||
it('FREE user requesting litellm throws TierError (litellm stays Team-only)', async () => {
|
||||
// Solo (FREE) unlocks BYO cloud embeddings (voyage/openai) but NOT the
|
||||
// managed litellm router — that stays a paid-tier provider.
|
||||
await expect(
|
||||
createEmbeddingProvider({
|
||||
provider: 'litellm',
|
||||
userTier: 'FREE',
|
||||
quotaDb: db,
|
||||
})
|
||||
).rejects.toThrow(TierError);
|
||||
});
|
||||
|
||||
it('FREE user requesting inprocess succeeds', async () => {
|
||||
// inprocess may fail to load ONNX in test env, but should NOT throw TierError
|
||||
try {
|
||||
await createEmbeddingProvider({
|
||||
provider: 'inprocess',
|
||||
userTier: 'FREE',
|
||||
quotaDb: db,
|
||||
});
|
||||
} catch (err) {
|
||||
// If it fails, it should NOT be a TierError — it should be a probe failure
|
||||
expect(err).not.toBeInstanceOf(TierError);
|
||||
}
|
||||
});
|
||||
|
||||
it('FREE user requesting voyage does not throw TierError (Solo unlocks BYO cloud embeddings)', async () => {
|
||||
// voyage will fail to connect (no real API), but should NOT throw TierError —
|
||||
// Solo now allows the voyage/openai providers directly.
|
||||
const provider = await createEmbeddingProvider({
|
||||
provider: 'auto',
|
||||
userTier: 'FREE',
|
||||
quotaDb: db,
|
||||
});
|
||||
// Should fall back to mock (no real providers in test), but no TierError
|
||||
expect(provider.getActiveProvider()).toBeDefined();
|
||||
});
|
||||
|
||||
it('auto mode surfaces only key-backed providers for FREE tier', async () => {
|
||||
const provider = await createEmbeddingProvider({
|
||||
provider: 'auto',
|
||||
userTier: 'FREE',
|
||||
quotaDb: db,
|
||||
});
|
||||
// FREE allows inprocess, mock, ollama + cloud (voyage/openai); cloud is
|
||||
// skipped here without API keys, so only local providers + mock surface.
|
||||
const status = provider.getStatus();
|
||||
for (const p of status.availableProviders) {
|
||||
expect(['inprocess', 'mock', 'ollama']).toContain(p);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Quota enforcement', () => {
|
||||
// All current tiers have unlimited quotas (-1), so we patch FREE
|
||||
// to a finite quota for these tests to exercise the quota mechanism.
|
||||
const originalQuota = TIER_CAPABILITIES.FREE.embeddingQuotaPerMonth;
|
||||
|
||||
beforeEach(() => {
|
||||
(TIER_CAPABILITIES.FREE as { embeddingQuotaPerMonth: number }).embeddingQuotaPerMonth = 500;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
(TIER_CAPABILITIES.FREE as { embeddingQuotaPerMonth: number }).embeddingQuotaPerMonth = originalQuota;
|
||||
});
|
||||
|
||||
it('FREE user at 499 embeddings succeeds', async () => {
|
||||
const provider = await createEmbeddingProvider({
|
||||
provider: 'mock',
|
||||
userTier: 'FREE',
|
||||
userId: 'test-user',
|
||||
quotaDb: db,
|
||||
});
|
||||
|
||||
// Pre-fill 499 embeddings
|
||||
const ym = `${new Date().getFullYear()}-${String(new Date().getMonth() + 1).padStart(2, '0')}`;
|
||||
db.prepare('INSERT INTO embedding_usage (user_id, year_month, count, updated_at) VALUES (?, ?, 499, ?)').run('test-user', ym, Date.now());
|
||||
|
||||
// 500th should succeed (499 + 1 = 500 = quota)
|
||||
const result = await provider.embed('test text');
|
||||
expect(result).toBeInstanceOf(Float32Array);
|
||||
});
|
||||
|
||||
it('FREE user at 500 embeddings throws EmbeddingQuotaExceededError', async () => {
|
||||
const provider = await createEmbeddingProvider({
|
||||
provider: 'mock',
|
||||
userTier: 'FREE',
|
||||
userId: 'test-user',
|
||||
quotaDb: db,
|
||||
});
|
||||
|
||||
// Pre-fill to quota limit
|
||||
const ym = `${new Date().getFullYear()}-${String(new Date().getMonth() + 1).padStart(2, '0')}`;
|
||||
db.prepare('INSERT INTO embedding_usage (user_id, year_month, count, updated_at) VALUES (?, ?, 500, ?)').run('test-user', ym, Date.now());
|
||||
|
||||
await expect(provider.embed('test text')).rejects.toThrow(EmbeddingQuotaExceededError);
|
||||
});
|
||||
|
||||
it('TEAMS user with unlimited quota never throws quota error', async () => {
|
||||
const provider = await createEmbeddingProvider({
|
||||
provider: 'mock',
|
||||
userTier: 'TEAMS',
|
||||
userId: 'test-user',
|
||||
quotaDb: db,
|
||||
});
|
||||
|
||||
// Even with high usage, should succeed (TEAMS has -1 = unlimited)
|
||||
const ym = `${new Date().getFullYear()}-${String(new Date().getMonth() + 1).padStart(2, '0')}`;
|
||||
db.prepare('INSERT INTO embedding_usage (user_id, year_month, count, updated_at) VALUES (?, ?, 999999, ?)').run('test-user', ym, Date.now());
|
||||
|
||||
const result = await provider.embed('test text');
|
||||
expect(result).toBeInstanceOf(Float32Array);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getQuotaStatus', () => {
|
||||
const originalQuota = TIER_CAPABILITIES.FREE.embeddingQuotaPerMonth;
|
||||
|
||||
beforeEach(() => {
|
||||
(TIER_CAPABILITIES.FREE as { embeddingQuotaPerMonth: number }).embeddingQuotaPerMonth = 500;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
(TIER_CAPABILITIES.FREE as { embeddingQuotaPerMonth: number }).embeddingQuotaPerMonth = originalQuota;
|
||||
});
|
||||
|
||||
it('returns correct percentage for FREE user', async () => {
|
||||
const provider = await createEmbeddingProvider({
|
||||
provider: 'mock',
|
||||
userTier: 'FREE',
|
||||
userId: 'test-user',
|
||||
quotaDb: db,
|
||||
});
|
||||
|
||||
// Use 250 of 500 quota
|
||||
const ym = `${new Date().getFullYear()}-${String(new Date().getMonth() + 1).padStart(2, '0')}`;
|
||||
db.prepare('INSERT INTO embedding_usage (user_id, year_month, count, updated_at) VALUES (?, ?, 250, ?)').run('test-user', ym, Date.now());
|
||||
|
||||
const status = provider.getQuotaStatus();
|
||||
expect(status.tier).toBe('FREE');
|
||||
expect(status.quota).toBe(500);
|
||||
expect(status.used).toBe(250);
|
||||
expect(status.remaining).toBe(250);
|
||||
expect(status.percentage).toBe(50);
|
||||
expect(status.resetsAt).toBeTruthy();
|
||||
});
|
||||
|
||||
it('returns unlimited for TEAMS tier', async () => {
|
||||
const provider = await createEmbeddingProvider({
|
||||
provider: 'mock',
|
||||
userTier: 'TEAMS',
|
||||
userId: 'test-user',
|
||||
quotaDb: db,
|
||||
});
|
||||
|
||||
const status = provider.getQuotaStatus();
|
||||
expect(status.tier).toBe('TEAMS');
|
||||
expect(status.quota).toBe(-1);
|
||||
expect(status.remaining).toBe(-1);
|
||||
expect(status.percentage).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('WAGGLE_EVAL_MODE tier bypass (PA v5 §11.3)', () => {
|
||||
// Defensive cleanup: never leak the flag across tests. If a prior run
|
||||
// crashed mid-test, this block restores a known-clean baseline.
|
||||
beforeEach(() => { delete process.env.WAGGLE_EVAL_MODE; });
|
||||
afterEach(() => { delete process.env.WAGGLE_EVAL_MODE; });
|
||||
|
||||
it('without WAGGLE_EVAL_MODE: FREE + litellm still throws TierError (control)', async () => {
|
||||
// Explicit sanity check that the normal gate is still live — baseline
|
||||
// for the bypass test below. litellm is the provider FREE still lacks.
|
||||
expect(process.env.WAGGLE_EVAL_MODE).toBeUndefined();
|
||||
await expect(
|
||||
createEmbeddingProvider({
|
||||
provider: 'litellm',
|
||||
userTier: 'FREE',
|
||||
quotaDb: db,
|
||||
})
|
||||
).rejects.toThrow(TierError);
|
||||
});
|
||||
|
||||
it('with WAGGLE_EVAL_MODE=1: FREE + litellm no longer throws TierError', async () => {
|
||||
process.env.WAGGLE_EVAL_MODE = '1';
|
||||
try {
|
||||
await createEmbeddingProvider({
|
||||
provider: 'litellm',
|
||||
userTier: 'FREE',
|
||||
quotaDb: db,
|
||||
});
|
||||
} catch (err) {
|
||||
// Probe failure (no real voyage backend in tests) is fine — just not
|
||||
// a TierError. The point: the tier gate is bypassed.
|
||||
expect(err).not.toBeInstanceOf(TierError);
|
||||
}
|
||||
});
|
||||
|
||||
it('with WAGGLE_EVAL_MODE=1 and no userTier set: behaves as if unenforced', async () => {
|
||||
process.env.WAGGLE_EVAL_MODE = '1';
|
||||
const provider = await createEmbeddingProvider({
|
||||
provider: 'auto',
|
||||
quotaDb: db,
|
||||
});
|
||||
// Falls back to mock with no tier skip/probe; provider is constructed.
|
||||
expect(provider.getActiveProvider()).toBeDefined();
|
||||
});
|
||||
|
||||
it('only activates when env value is exactly "1" (defensive literal match)', async () => {
|
||||
// Guard against accidental truthy-but-not-"1" values. Harness must use
|
||||
// "1" exactly per §11.3.
|
||||
for (const bad of ['true', 'yes', '0', '']) {
|
||||
process.env.WAGGLE_EVAL_MODE = bad;
|
||||
await expect(
|
||||
createEmbeddingProvider({
|
||||
provider: 'litellm',
|
||||
userTier: 'FREE',
|
||||
quotaDb: db,
|
||||
})
|
||||
).rejects.toThrow(TierError);
|
||||
delete process.env.WAGGLE_EVAL_MODE;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('getMinimumTierForProvider', () => {
|
||||
it('inprocess requires TRIAL (first tier that allows it)', () => {
|
||||
expect(getMinimumTierForProvider('inprocess')).toBe('TRIAL');
|
||||
});
|
||||
|
||||
it('voyage requires TRIAL (TRIAL unlocks all providers)', () => {
|
||||
// TRIAL is the first tier in TIERS array and has all providers
|
||||
expect(getMinimumTierForProvider('voyage')).toBe('TRIAL');
|
||||
});
|
||||
|
||||
it('litellm requires TRIAL', () => {
|
||||
// TRIAL is the first tier with litellm (all unlocked)
|
||||
expect(getMinimumTierForProvider('litellm')).toBe('TRIAL');
|
||||
});
|
||||
|
||||
it('mock requires TRIAL', () => {
|
||||
expect(getMinimumTierForProvider('mock')).toBe('TRIAL');
|
||||
});
|
||||
});
|
||||
});
|
||||
267
packages/core/tests/file-indexer.test.ts
Normal file
267
packages/core/tests/file-indexer.test.ts
Normal file
@@ -0,0 +1,267 @@
|
||||
/**
|
||||
* FileIndexer unit tests (L-20)
|
||||
*
|
||||
* Covers: format gate, indexing, overwrite semantics, move, remove, truncation,
|
||||
* shared-content dedup safety, and the underlying file_index table.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import { MindDB } from '@waggle/hive-mind-core';
|
||||
import { FrameStore } from '@waggle/hive-mind-core';
|
||||
import { FileIndexer, MAX_CONTENT_BYTES } from '../src/file-indexer.js';
|
||||
|
||||
describe('FileIndexer', () => {
|
||||
let db: MindDB;
|
||||
let indexer: FileIndexer;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
indexer = new FileIndexer(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
describe('shouldIndex', () => {
|
||||
it('accepts markdown + text', () => {
|
||||
expect(FileIndexer.shouldIndex('/notes/a.md')).toBe(true);
|
||||
expect(FileIndexer.shouldIndex('/notes/a.markdown')).toBe(true);
|
||||
expect(FileIndexer.shouldIndex('/notes/a.txt')).toBe(true);
|
||||
});
|
||||
|
||||
it('ignores case differences in the extension', () => {
|
||||
expect(FileIndexer.shouldIndex('/a.MD')).toBe(true);
|
||||
expect(FileIndexer.shouldIndex('/a.TXT')).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects formats deferred to Bucket 2', () => {
|
||||
expect(FileIndexer.shouldIndex('/a.pdf')).toBe(false);
|
||||
expect(FileIndexer.shouldIndex('/a.docx')).toBe(false);
|
||||
expect(FileIndexer.shouldIndex('/a.xlsx')).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects files with no extension', () => {
|
||||
expect(FileIndexer.shouldIndex('/a')).toBe(false);
|
||||
expect(FileIndexer.shouldIndex('/README')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('indexFile', () => {
|
||||
it('returns unsupported_format for non-indexable extensions', () => {
|
||||
const result = indexer.indexFile('/a.pdf', Buffer.from('PDF content'));
|
||||
expect(result.skipped).toBe(true);
|
||||
if (result.skipped) expect(result.reason).toBe('unsupported_format');
|
||||
});
|
||||
|
||||
it('indexes a markdown file and creates a backing frame', () => {
|
||||
const content = Buffer.from('# Hello\n\nThis is a note.');
|
||||
const result = indexer.indexFile('/notes/hello.md', content, 'text/markdown');
|
||||
|
||||
expect(result.skipped).toBe(false);
|
||||
if (!result.skipped) {
|
||||
expect(result.frameId).toBeGreaterThan(0);
|
||||
expect(result.truncated).toBe(false);
|
||||
|
||||
const frames = new FrameStore(db);
|
||||
const frame = frames.getById(result.frameId);
|
||||
expect(frame).toBeTruthy();
|
||||
expect(frame!.content).toContain('# Hello');
|
||||
expect(frame!.content).toContain('[FILE: /notes/hello.md');
|
||||
expect(frame!.content).toContain('text/markdown');
|
||||
expect(frame!.source).toBe('system');
|
||||
}
|
||||
});
|
||||
|
||||
it('records the index row with hash + size + mime', () => {
|
||||
const content = Buffer.from('hello world');
|
||||
indexer.indexFile('/a.txt', content, 'text/plain');
|
||||
const row = indexer.getRow('/a.txt');
|
||||
expect(row).toBeTruthy();
|
||||
expect(row!.sizeBytes).toBe(content.length);
|
||||
expect(row!.mimeType).toBe('text/plain');
|
||||
expect(row!.contentHash).toMatch(/^[a-f0-9]{64}$/);
|
||||
expect(row!.indexedAt).toBeTruthy();
|
||||
});
|
||||
|
||||
it('returns unchanged when re-indexed with identical content', () => {
|
||||
const content = Buffer.from('same bytes');
|
||||
const first = indexer.indexFile('/a.md', content);
|
||||
expect(first.skipped).toBe(false);
|
||||
|
||||
const second = indexer.indexFile('/a.md', content);
|
||||
expect(second.skipped).toBe(true);
|
||||
if (second.skipped) expect(second.reason).toBe('unchanged');
|
||||
});
|
||||
|
||||
it('swaps the frame on content change (overwrite path)', () => {
|
||||
const first = indexer.indexFile('/a.md', Buffer.from('original'));
|
||||
const second = indexer.indexFile('/a.md', Buffer.from('updated'));
|
||||
expect(first.skipped).toBe(false);
|
||||
expect(second.skipped).toBe(false);
|
||||
if (!first.skipped && !second.skipped) {
|
||||
expect(second.frameId).not.toBe(first.frameId);
|
||||
|
||||
// Old frame is gone.
|
||||
const frames = new FrameStore(db);
|
||||
expect(frames.getById(first.frameId)).toBeUndefined();
|
||||
// New frame exists and row points at it.
|
||||
expect(frames.getById(second.frameId)).toBeTruthy();
|
||||
const row = indexer.getRow('/a.md');
|
||||
expect(row!.frameId).toBe(second.frameId);
|
||||
}
|
||||
});
|
||||
|
||||
it('truncates content over MAX_CONTENT_BYTES and sets the truncated flag', () => {
|
||||
const giant = Buffer.alloc(MAX_CONTENT_BYTES + 5000, 0x41); // lots of 'A'
|
||||
const result = indexer.indexFile('/big.md', giant);
|
||||
expect(result.skipped).toBe(false);
|
||||
if (!result.skipped) {
|
||||
expect(result.truncated).toBe(true);
|
||||
const frames = new FrameStore(db);
|
||||
const frame = frames.getById(result.frameId);
|
||||
expect(frame!.content).toContain('[…truncated');
|
||||
}
|
||||
});
|
||||
|
||||
it('treats an empty file as remove-if-present', () => {
|
||||
indexer.indexFile('/a.md', Buffer.from('something'));
|
||||
expect(indexer.getRow('/a.md')).toBeTruthy();
|
||||
|
||||
const emptyResult = indexer.indexFile('/a.md', Buffer.from(''));
|
||||
expect(emptyResult.skipped).toBe(true);
|
||||
if (emptyResult.skipped) expect(emptyResult.reason).toBe('empty');
|
||||
expect(indexer.getRow('/a.md')).toBeNull();
|
||||
});
|
||||
|
||||
it('rolls back atomically when a mutation throws mid-overwrite (L-20 BLOCKER-1)', () => {
|
||||
// Index a file, then simulate a crash during the overwrite path by
|
||||
// making frames.delete throw. The whole transaction (new frame +
|
||||
// old-frame delete + file_index UPDATE) must roll back together. Table
|
||||
// state after the throw must match state before the throw.
|
||||
const first = indexer.indexFile('/a.md', Buffer.from('original'));
|
||||
expect(first.skipped).toBe(false);
|
||||
if (first.skipped) return;
|
||||
const originalFrameId = first.frameId;
|
||||
|
||||
const raw = db.getDatabase();
|
||||
const framesBefore = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
|
||||
const indexBefore = (raw.prepare('SELECT COUNT(*) as c FROM file_index').get() as { c: number }).c;
|
||||
|
||||
// Inject crash mid-transaction: the old-frame delete step throws.
|
||||
const framesProp = (indexer as unknown as { frames: FrameStore }).frames;
|
||||
const deleteSpy = vi.spyOn(framesProp, 'delete').mockImplementation(() => {
|
||||
throw new Error('simulated crash mid-overwrite');
|
||||
});
|
||||
|
||||
try {
|
||||
expect(() => indexer.indexFile('/a.md', Buffer.from('updated'))).toThrow('simulated crash mid-overwrite');
|
||||
} finally {
|
||||
deleteSpy.mockRestore();
|
||||
}
|
||||
|
||||
// Rollback invariants:
|
||||
// 1. Frame-table row count unchanged (new frame not committed).
|
||||
// 2. Old frame still present (delete was rolled back).
|
||||
// 3. file_index row count unchanged + row still points at original frame.
|
||||
const framesAfter = (raw.prepare('SELECT COUNT(*) as c FROM memory_frames').get() as { c: number }).c;
|
||||
const indexAfter = (raw.prepare('SELECT COUNT(*) as c FROM file_index').get() as { c: number }).c;
|
||||
expect(framesAfter).toBe(framesBefore);
|
||||
expect(indexAfter).toBe(indexBefore);
|
||||
|
||||
const framesStore = new FrameStore(db);
|
||||
expect(framesStore.getById(originalFrameId)).toBeTruthy();
|
||||
expect(indexer.getRow('/a.md')!.frameId).toBe(originalFrameId);
|
||||
});
|
||||
});
|
||||
|
||||
describe('removeFile', () => {
|
||||
it('returns false when the path is not indexed', () => {
|
||||
expect(indexer.removeFile('/not-indexed.md')).toBe(false);
|
||||
});
|
||||
|
||||
it('deletes the frame + row when indexed', () => {
|
||||
const result = indexer.indexFile('/a.md', Buffer.from('goodbye'));
|
||||
expect(result.skipped).toBe(false);
|
||||
if (!result.skipped) {
|
||||
expect(indexer.removeFile('/a.md')).toBe(true);
|
||||
expect(indexer.getRow('/a.md')).toBeNull();
|
||||
const frames = new FrameStore(db);
|
||||
expect(frames.getById(result.frameId)).toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
||||
it('two paths with identical bodies get distinct frames (path is in the header)', () => {
|
||||
// The header `[FILE: <path>]` is part of the frame content, so two files
|
||||
// with the same body but different paths produce different frame hashes.
|
||||
// This is intentional: a file is a file, not just its bytes.
|
||||
const body = Buffer.from('shared body');
|
||||
const first = indexer.indexFile('/dir-a/shared.md', body);
|
||||
const second = indexer.indexFile('/dir-b/shared.md', body);
|
||||
expect(first.skipped).toBe(false);
|
||||
expect(second.skipped).toBe(false);
|
||||
if (!first.skipped && !second.skipped) {
|
||||
expect(second.frameId).not.toBe(first.frameId);
|
||||
|
||||
// Removing one file removes its own frame + row without touching the other.
|
||||
indexer.removeFile('/dir-a/shared.md');
|
||||
const frames = new FrameStore(db);
|
||||
expect(frames.getById(first.frameId)).toBeUndefined();
|
||||
expect(frames.getById(second.frameId)).toBeTruthy();
|
||||
expect(indexer.getRow('/dir-b/shared.md')).toBeTruthy();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('moveFile', () => {
|
||||
it('updates the recorded path + keeps the frame', () => {
|
||||
const res = indexer.indexFile('/old.md', Buffer.from('body'));
|
||||
expect(res.skipped).toBe(false);
|
||||
if (!res.skipped) {
|
||||
expect(indexer.moveFile('/old.md', '/new.md')).toBe(true);
|
||||
expect(indexer.getRow('/old.md')).toBeNull();
|
||||
const row = indexer.getRow('/new.md');
|
||||
expect(row!.frameId).toBe(res.frameId);
|
||||
|
||||
const frames = new FrameStore(db);
|
||||
expect(frames.getById(res.frameId)).toBeTruthy();
|
||||
}
|
||||
});
|
||||
|
||||
it('returns false when the source is not indexed', () => {
|
||||
expect(indexer.moveFile('/missing.md', '/somewhere.md')).toBe(false);
|
||||
});
|
||||
|
||||
it('overwrites a pre-existing destination index row', () => {
|
||||
indexer.indexFile('/src.md', Buffer.from('src-body'));
|
||||
indexer.indexFile('/dst.md', Buffer.from('dst-body'));
|
||||
expect(indexer.moveFile('/src.md', '/dst.md')).toBe(true);
|
||||
expect(indexer.getRow('/src.md')).toBeNull();
|
||||
expect(indexer.getRow('/dst.md')).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe('listAll', () => {
|
||||
it('returns an empty array with no index rows', () => {
|
||||
expect(indexer.listAll()).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns rows newest-indexed first', async () => {
|
||||
indexer.indexFile('/a.md', Buffer.from('a'));
|
||||
await new Promise(resolve => setTimeout(resolve, 1100));
|
||||
indexer.indexFile('/b.md', Buffer.from('b'));
|
||||
const all = indexer.listAll();
|
||||
expect(all).toHaveLength(2);
|
||||
expect(all[0].filePath).toBe('/b.md');
|
||||
expect(all[1].filePath).toBe('/a.md');
|
||||
});
|
||||
});
|
||||
|
||||
describe('constructor side effects', () => {
|
||||
it('ensureTable is idempotent (second instance on same DB is OK)', () => {
|
||||
indexer.indexFile('/a.md', Buffer.from('x'));
|
||||
const indexer2 = new FileIndexer(db);
|
||||
expect(indexer2.getRow('/a.md')).toBeTruthy();
|
||||
});
|
||||
});
|
||||
});
|
||||
175
packages/core/tests/file-store-path-safety.test.ts
Normal file
175
packages/core/tests/file-store-path-safety.test.ts
Normal file
@@ -0,0 +1,175 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { LocalFileStore, LinkedDirStore, isSensitiveFilePath } from '../src/file-store.js';
|
||||
|
||||
let tmp: string;
|
||||
beforeEach(() => { tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-fsguard-')); });
|
||||
afterEach(() => { fs.rmSync(tmp, { recursive: true, force: true }); });
|
||||
|
||||
describe('isSensitiveFilePath', () => {
|
||||
it('flags SSH/GPG/cloud secret directories and key basenames', () => {
|
||||
for (const p of ['.ssh/id_rsa', '.ssh/config', '.aws/credentials', '.gnupg/secring.gpg',
|
||||
'id_rsa', 'id_ed25519', '.netrc', '.pgpass', '.git-credentials', 'credentials.json',
|
||||
'project/nested/.ssh/known_hosts']) {
|
||||
expect(isSensitiveFilePath(p), p).toBe(true);
|
||||
}
|
||||
});
|
||||
it('flags dotenv files but NOT their checked-in templates', () => {
|
||||
expect(isSensitiveFilePath('.env')).toBe(true);
|
||||
expect(isSensitiveFilePath('config/.env.production')).toBe(true);
|
||||
expect(isSensitiveFilePath('.env.example')).toBe(false);
|
||||
expect(isSensitiveFilePath('.env.template')).toBe(false);
|
||||
});
|
||||
it('is case-insensitive and path-separator agnostic', () => {
|
||||
expect(isSensitiveFilePath('.SSH/ID_RSA')).toBe(true);
|
||||
expect(isSensitiveFilePath('.ssh\\id_rsa')).toBe(true);
|
||||
});
|
||||
it('does NOT flag ordinary files (incl. public keys)', () => {
|
||||
for (const p of ['', 'readme.md', 'src/index.ts', 'config.json', 'id_rsa.pub', '.environment', 'data/credentials-form.tsx']) {
|
||||
expect(isSensitiveFilePath(p), p).toBe(false);
|
||||
}
|
||||
});
|
||||
it('defeats Windows ADS + trailing dot/space normalization tricks', () => {
|
||||
for (const p of ['id_rsa::$DATA', '.env::$DATA', 'id_rsa.', '.env ', '.NPMRC ']) {
|
||||
expect(isSensitiveFilePath(p), p).toBe(true);
|
||||
}
|
||||
});
|
||||
it('flags backup copies of secrets but not ordinary backups', () => {
|
||||
expect(isSensitiveFilePath('credentials.bak')).toBe(true);
|
||||
expect(isSensitiveFilePath('id_rsa.old')).toBe(true);
|
||||
expect(isSensitiveFilePath('.npmrc.backup')).toBe(true);
|
||||
expect(isSensitiveFilePath('readme.bak')).toBe(false);
|
||||
});
|
||||
it('flags extended secret classes (pem keys, authorized_keys, cloud, terraform)', () => {
|
||||
for (const p of ['deploy/secret.pem', 'authorized_keys', 'known_hosts', '.azure/accessTokens.json',
|
||||
'.terraform/x', 'service-account.json', 'terraform.tfstate', 'infra/terraform.tfstate.backup']) {
|
||||
expect(isSensitiveFilePath(p), p).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveSafe containment (via LocalFileStore)', () => {
|
||||
const store = () => new LocalFileStore(tmp, 'ws1'); // root = tmp/workspaces/ws1/files
|
||||
|
||||
it('reads/writes a normal in-root path', async () => {
|
||||
const s = store();
|
||||
await s.writeFile('notes/todo.txt', 'hi');
|
||||
expect((await s.readFile('notes/todo.txt')).toString()).toBe('hi');
|
||||
});
|
||||
|
||||
it('denies a classic ../ escape', async () => {
|
||||
await expect(store().readFile('../../../etc/passwd')).rejects.toThrow(/traversal denied/i);
|
||||
});
|
||||
|
||||
it('denies a SIBLING-prefix escape (the startsWith bug)', async () => {
|
||||
// root is .../ws1/files; this resolves to a sibling .../ws1/files-evil which a
|
||||
// bare startsWith(root) check WRONGLY admitted. Segment-boundary check rejects it.
|
||||
await expect(store().writeFile('../files-evil/loot.txt', 'x')).rejects.toThrow(/traversal denied/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe('LinkedDirStore sensitive-file deny (external folder)', () => {
|
||||
function seedLinked(): string {
|
||||
const dir = path.join(tmp, 'project');
|
||||
fs.mkdirSync(path.join(dir, '.ssh'), { recursive: true });
|
||||
fs.mkdirSync(path.join(dir, '.aws'), { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, '.ssh', 'id_rsa'), 'PRIVATE KEY');
|
||||
fs.writeFileSync(path.join(dir, '.aws', 'credentials'), '[default]\naws_secret=xxx');
|
||||
fs.writeFileSync(path.join(dir, '.env'), 'SECRET=1');
|
||||
fs.writeFileSync(path.join(dir, '.env.example'), 'SECRET=');
|
||||
fs.writeFileSync(path.join(dir, 'README.md'), 'hello');
|
||||
fs.writeFileSync(path.join(dir, 'credentials.json'), '{"token":"x"}');
|
||||
fs.writeFileSync(path.join(dir, 'data.json'), '{"ok":true}');
|
||||
return dir;
|
||||
}
|
||||
|
||||
it('denies reading SSH keys, cloud creds, and .env', async () => {
|
||||
const s = new LinkedDirStore(seedLinked());
|
||||
await expect(s.readFile('.ssh/id_rsa')).rejects.toThrow(/sensitive file denied/i);
|
||||
await expect(s.readFile('.aws/credentials')).rejects.toThrow(/sensitive file denied/i);
|
||||
await expect(s.readFile('.env')).rejects.toThrow(/sensitive file denied/i);
|
||||
});
|
||||
|
||||
it('allows a normal file and the .env template', async () => {
|
||||
const s = new LinkedDirStore(seedLinked());
|
||||
expect((await s.readFile('README.md')).toString()).toBe('hello');
|
||||
expect((await s.readFile('.env.example')).toString()).toBe('SECRET=');
|
||||
});
|
||||
|
||||
it('denies writing/clobbering and moving a secret file', async () => {
|
||||
const s = new LinkedDirStore(seedLinked());
|
||||
await expect(s.writeFile('.ssh/authorized_keys', 'attacker-key')).rejects.toThrow(/sensitive file denied/i);
|
||||
await expect(s.moveFile('README.md', '.env')).rejects.toThrow(/sensitive file denied/i);
|
||||
});
|
||||
|
||||
it('searchFiles never discloses a non-dot secret (credentials.json)', async () => {
|
||||
const s = new LinkedDirStore(seedLinked());
|
||||
const names = (await s.searchFiles('*.json')).map(f => f.name);
|
||||
expect(names).toContain('data.json');
|
||||
expect(names).not.toContain('credentials.json');
|
||||
});
|
||||
|
||||
it('listFiles hides non-dot secrets (credentials.json) but keeps normal files', async () => {
|
||||
const s = new LinkedDirStore(seedLinked());
|
||||
const names = (await s.listFiles()).map(f => f.name);
|
||||
expect(names).toContain('README.md');
|
||||
expect(names).not.toContain('credentials.json');
|
||||
});
|
||||
|
||||
it('searchFiles cannot escape the root via a ../ glob pattern', async () => {
|
||||
const outside = path.join(tmp, 'outside'); fs.mkdirSync(outside, { recursive: true });
|
||||
fs.writeFileSync(path.join(outside, 'loot.txt'), 'x');
|
||||
const s = new LinkedDirStore(seedLinked());
|
||||
const results = await s.searchFiles('../**/*');
|
||||
expect(results.every(f => !f.path.includes('..'))).toBe(true);
|
||||
expect(results.map(f => f.name)).not.toContain('loot.txt');
|
||||
});
|
||||
|
||||
it('still denies a ../ escape out of the linked root', async () => {
|
||||
const s = new LinkedDirStore(seedLinked());
|
||||
await expect(s.readFile('../../secret.txt')).rejects.toThrow(/traversal denied/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe('LinkedDirStore symlink containment (the CRITICAL escape)', () => {
|
||||
// Symlink creation can fail without privilege (esp. Windows file symlinks) — the
|
||||
// fix still applies; these tests self-skip when the env cannot create the link.
|
||||
function trySymlink(target: string, link: string, type: 'junction' | 'file' | 'dir'): boolean {
|
||||
try { fs.symlinkSync(target, link, type); return true; } catch { return false; }
|
||||
}
|
||||
|
||||
it('denies reading through a junction/symlink that escapes the root', async () => {
|
||||
const outside = path.join(tmp, 'outside'); fs.mkdirSync(outside, { recursive: true });
|
||||
fs.writeFileSync(path.join(outside, 'secret.txt'), 'TOPSECRET');
|
||||
const root = path.join(tmp, 'proj'); fs.mkdirSync(root, { recursive: true });
|
||||
if (!trySymlink(outside, path.join(root, 'escape'), 'junction')) return;
|
||||
const s = new LinkedDirStore(root);
|
||||
await expect(s.readFile('escape/secret.txt')).rejects.toThrow(/traversal denied/i);
|
||||
});
|
||||
|
||||
it('still allows a symlink that stays inside the root (monorepo-style link)', async () => {
|
||||
const root = path.join(tmp, 'proj2'); fs.mkdirSync(path.join(root, 'real'), { recursive: true });
|
||||
fs.writeFileSync(path.join(root, 'real', 'data.txt'), 'OK');
|
||||
if (!trySymlink(path.join(root, 'real'), path.join(root, 'alias'), 'junction')) return;
|
||||
const s = new LinkedDirStore(root);
|
||||
expect((await s.readFile('alias/data.txt')).toString()).toBe('OK');
|
||||
});
|
||||
|
||||
it('denies a symlink that launders an in-root secret past a benign name', async () => {
|
||||
const root = path.join(tmp, 'proj3'); fs.mkdirSync(root, { recursive: true });
|
||||
fs.writeFileSync(path.join(root, '.env'), 'SECRET=1');
|
||||
if (!trySymlink(path.join(root, '.env'), path.join(root, 'notes'), 'file')) return;
|
||||
const s = new LinkedDirStore(root);
|
||||
await expect(s.readFile('notes')).rejects.toThrow(/sensitive file denied/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe('LocalFileStore is sandboxed — no sensitive-file deny', () => {
|
||||
it('allows a .env in the virtual workspace (the agent\'s own scratch)', async () => {
|
||||
const s = new LocalFileStore(tmp, 'ws2');
|
||||
await s.writeFile('.env', 'LOCAL=1');
|
||||
expect((await s.readFile('.env')).toString()).toBe('LOCAL=1');
|
||||
});
|
||||
});
|
||||
266
packages/core/tests/file-store-s3.test.ts
Normal file
266
packages/core/tests/file-store-s3.test.ts
Normal file
@@ -0,0 +1,266 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { S3FileStore, type S3Config } from '../src/file-store.js';
|
||||
|
||||
// ── Mock @aws-sdk/client-s3 ────────────────────────────────────────
|
||||
|
||||
const mockSend = vi.fn();
|
||||
|
||||
type CommandInput = Record<string, unknown>;
|
||||
|
||||
class MockS3Client {
|
||||
send = mockSend;
|
||||
constructor(_config: unknown) {}
|
||||
}
|
||||
|
||||
vi.mock('@aws-sdk/client-s3', () => {
|
||||
return {
|
||||
S3Client: MockS3Client,
|
||||
GetObjectCommand: class { [k: string]: unknown; _type = 'GetObject'; constructor(input: CommandInput) { Object.assign(this, input); } },
|
||||
PutObjectCommand: class { [k: string]: unknown; _type = 'PutObject'; constructor(input: CommandInput) { Object.assign(this, input); } },
|
||||
DeleteObjectCommand: class { [k: string]: unknown; _type = 'DeleteObject'; constructor(input: CommandInput) { Object.assign(this, input); } },
|
||||
ListObjectsV2Command: class { [k: string]: unknown; _type = 'ListObjects'; constructor(input: CommandInput) { Object.assign(this, input); } },
|
||||
CopyObjectCommand: class { [k: string]: unknown; _type = 'CopyObject'; constructor(input: CommandInput) { Object.assign(this, input); } },
|
||||
};
|
||||
});
|
||||
|
||||
// ── Test setup ──────────────────────────────────────────────────────
|
||||
|
||||
const testConfig: S3Config = {
|
||||
endpoint: 'minio:9000',
|
||||
bucket: 'waggle-files',
|
||||
accessKey: 'waggle',
|
||||
secretKey: 'waggle_s3_prod',
|
||||
prefix: 'workspaces/ws-123/',
|
||||
};
|
||||
|
||||
describe('S3FileStore', () => {
|
||||
let store: S3FileStore;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
store = new S3FileStore(testConfig);
|
||||
});
|
||||
|
||||
// ── Meta ────────────────────────────────────────────────────────
|
||||
|
||||
it('getStorageType returns virtual', () => {
|
||||
expect(store.getStorageType()).toBe('virtual');
|
||||
});
|
||||
|
||||
it('getRootPath returns s3:// URL', () => {
|
||||
expect(store.getRootPath()).toBe('s3://waggle-files/workspaces/ws-123/');
|
||||
});
|
||||
|
||||
// ── writeFile ───────────────────────────────────────────────────
|
||||
|
||||
it('writeFile sends PutObjectCommand with correct bucket and key', async () => {
|
||||
mockSend.mockResolvedValueOnce({});
|
||||
|
||||
await store.writeFile('docs/notes.md', 'hello world');
|
||||
|
||||
expect(mockSend).toHaveBeenCalledTimes(1);
|
||||
const cmd = mockSend.mock.calls[0][0];
|
||||
expect(cmd._type).toBe('PutObject');
|
||||
expect(cmd.Bucket).toBe('waggle-files');
|
||||
expect(cmd.Key).toBe('workspaces/ws-123/docs/notes.md');
|
||||
expect(cmd.Body).toEqual(Buffer.from('hello world'));
|
||||
});
|
||||
|
||||
it('writeFile accepts Buffer content', async () => {
|
||||
mockSend.mockResolvedValueOnce({});
|
||||
const buf = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
|
||||
|
||||
await store.writeFile('image.png', buf);
|
||||
|
||||
const cmd = mockSend.mock.calls[0][0];
|
||||
expect(cmd.Body).toBe(buf);
|
||||
});
|
||||
|
||||
// ── readFile ────────────────────────────────────────────────────
|
||||
|
||||
it('readFile sends GetObjectCommand and returns Buffer', async () => {
|
||||
const chunks = [Buffer.from('chunk1'), Buffer.from('chunk2')];
|
||||
const asyncIterable = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
for (const c of chunks) yield c;
|
||||
},
|
||||
};
|
||||
mockSend.mockResolvedValueOnce({ Body: asyncIterable });
|
||||
|
||||
const result = await store.readFile('docs/notes.md');
|
||||
|
||||
expect(mockSend).toHaveBeenCalledTimes(1);
|
||||
const cmd = mockSend.mock.calls[0][0];
|
||||
expect(cmd._type).toBe('GetObject');
|
||||
expect(cmd.Bucket).toBe('waggle-files');
|
||||
expect(cmd.Key).toBe('workspaces/ws-123/docs/notes.md');
|
||||
expect(Buffer.isBuffer(result)).toBe(true);
|
||||
expect(result.toString()).toBe('chunk1chunk2');
|
||||
});
|
||||
|
||||
// ── deleteFile ──────────────────────────────────────────────────
|
||||
|
||||
it('deleteFile sends DeleteObjectCommand', async () => {
|
||||
mockSend.mockResolvedValueOnce({});
|
||||
|
||||
await store.deleteFile('old-file.txt');
|
||||
|
||||
expect(mockSend).toHaveBeenCalledTimes(1);
|
||||
const cmd = mockSend.mock.calls[0][0];
|
||||
expect(cmd._type).toBe('DeleteObject');
|
||||
expect(cmd.Bucket).toBe('waggle-files');
|
||||
expect(cmd.Key).toBe('workspaces/ws-123/old-file.txt');
|
||||
});
|
||||
|
||||
// ── listFiles ───────────────────────────────────────────────────
|
||||
|
||||
it('listFiles sends ListObjectsV2Command and returns FileEntry[]', async () => {
|
||||
mockSend.mockResolvedValueOnce({
|
||||
Contents: [
|
||||
{ Key: 'workspaces/ws-123/readme.md', Size: 1024, LastModified: new Date('2025-06-01T00:00:00Z') },
|
||||
{ Key: 'workspaces/ws-123/src/index.ts', Size: 512, LastModified: new Date('2025-06-02T00:00:00Z') },
|
||||
],
|
||||
CommonPrefixes: [
|
||||
{ Prefix: 'workspaces/ws-123/docs/' },
|
||||
],
|
||||
});
|
||||
|
||||
const entries = await store.listFiles();
|
||||
|
||||
expect(mockSend).toHaveBeenCalledTimes(1);
|
||||
const cmd = mockSend.mock.calls[0][0];
|
||||
expect(cmd._type).toBe('ListObjects');
|
||||
expect(cmd.Bucket).toBe('waggle-files');
|
||||
expect(cmd.Prefix).toBe('workspaces/ws-123/');
|
||||
expect(cmd.Delimiter).toBe('/');
|
||||
|
||||
expect(entries).toHaveLength(3);
|
||||
// File entries
|
||||
expect(entries[0]).toEqual({
|
||||
name: 'readme.md',
|
||||
path: 'readme.md',
|
||||
size: 1024,
|
||||
modified: '2025-06-01T00:00:00.000Z',
|
||||
isDirectory: false,
|
||||
});
|
||||
expect(entries[1]).toEqual({
|
||||
name: 'index.ts',
|
||||
path: 'src/index.ts',
|
||||
size: 512,
|
||||
modified: '2025-06-02T00:00:00.000Z',
|
||||
isDirectory: false,
|
||||
});
|
||||
// Directory entry
|
||||
expect(entries[2]).toEqual({
|
||||
name: 'docs',
|
||||
path: 'docs',
|
||||
size: 0,
|
||||
modified: '',
|
||||
isDirectory: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('listFiles with directory argument appends to prefix', async () => {
|
||||
mockSend.mockResolvedValueOnce({ Contents: [], CommonPrefixes: [] });
|
||||
|
||||
await store.listFiles('src');
|
||||
|
||||
const cmd = mockSend.mock.calls[0][0];
|
||||
expect(cmd.Prefix).toBe('workspaces/ws-123/src/');
|
||||
});
|
||||
|
||||
it('listFiles handles empty response', async () => {
|
||||
mockSend.mockResolvedValueOnce({});
|
||||
|
||||
const entries = await store.listFiles();
|
||||
expect(entries).toEqual([]);
|
||||
});
|
||||
|
||||
// ── moveFile ────────────────────────────────────────────────────
|
||||
|
||||
it('moveFile copies then deletes', async () => {
|
||||
mockSend.mockResolvedValueOnce({}); // CopyObject
|
||||
mockSend.mockResolvedValueOnce({}); // DeleteObject
|
||||
|
||||
await store.moveFile('old.txt', 'new.txt');
|
||||
|
||||
expect(mockSend).toHaveBeenCalledTimes(2);
|
||||
const copyCmd = mockSend.mock.calls[0][0];
|
||||
expect(copyCmd._type).toBe('CopyObject');
|
||||
expect(copyCmd.CopySource).toBe('waggle-files/workspaces/ws-123/old.txt');
|
||||
expect(copyCmd.Key).toBe('workspaces/ws-123/new.txt');
|
||||
|
||||
const deleteCmd = mockSend.mock.calls[1][0];
|
||||
expect(deleteCmd._type).toBe('DeleteObject');
|
||||
expect(deleteCmd.Key).toBe('workspaces/ws-123/old.txt');
|
||||
});
|
||||
|
||||
// ── searchFiles ─────────────────────────────────────────────────
|
||||
|
||||
it('searchFiles filters listFiles results by pattern', async () => {
|
||||
mockSend.mockResolvedValueOnce({
|
||||
Contents: [
|
||||
{ Key: 'workspaces/ws-123/readme.md', Size: 100, LastModified: new Date() },
|
||||
{ Key: 'workspaces/ws-123/notes.txt', Size: 200, LastModified: new Date() },
|
||||
{ Key: 'workspaces/ws-123/data.md', Size: 50, LastModified: new Date() },
|
||||
],
|
||||
CommonPrefixes: [],
|
||||
});
|
||||
|
||||
const results = await store.searchFiles('*.md');
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results[0].name).toBe('readme.md');
|
||||
expect(results[1].name).toBe('data.md');
|
||||
});
|
||||
|
||||
it('searchFiles is ReDoS-safe with a hostile regex pattern', async () => {
|
||||
mockSend.mockResolvedValueOnce({
|
||||
Contents: [{ Key: 'workspaces/ws-123/' + 'a'.repeat(40) + 'X', Size: 1, LastModified: new Date() }],
|
||||
CommonPrefixes: [],
|
||||
});
|
||||
const start = Date.now();
|
||||
// Under the old `new RegExp(pattern…)` this built /(a+)+$/i and catastrophically
|
||||
// backtracked on the key. Now the metachars are escaped → a literal match → instant.
|
||||
const results = await store.searchFiles('(a+)+$');
|
||||
expect(Date.now() - start).toBeLessThan(1000);
|
||||
expect(results).toHaveLength(0); // matched literally; no key contains "(a+)+$"
|
||||
});
|
||||
|
||||
// ── key traversal protection ────────────────────────────────────
|
||||
|
||||
it('rejects ../ traversal in every op BEFORE any S3 call', async () => {
|
||||
await expect(store.readFile('../ws-456/secret')).rejects.toThrow(/traversal denied/i);
|
||||
await expect(store.writeFile('../ws-456/x', 'y')).rejects.toThrow(/traversal denied/i);
|
||||
await expect(store.deleteFile('../../etc/passwd')).rejects.toThrow(/traversal denied/i);
|
||||
await expect(store.moveFile('ok', '../ws-456/b')).rejects.toThrow(/traversal denied/i);
|
||||
await expect(store.listFiles('../ws-456')).rejects.toThrow(/traversal denied/i);
|
||||
expect(mockSend).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('allows a normal nested key', async () => {
|
||||
mockSend.mockResolvedValueOnce({});
|
||||
await store.writeFile('docs/sub/file.md', 'x');
|
||||
expect(mockSend.mock.calls[0][0].Key).toBe('workspaces/ws-123/docs/sub/file.md');
|
||||
});
|
||||
|
||||
// ── getStorageInfo ──────────────────────────────────────────────
|
||||
|
||||
it('getStorageInfo sums sizes from S3 listing', async () => {
|
||||
mockSend.mockResolvedValueOnce({
|
||||
Contents: [
|
||||
{ Key: 'workspaces/ws-123/a.txt', Size: 100 },
|
||||
{ Key: 'workspaces/ws-123/b.txt', Size: 250 },
|
||||
{ Key: 'workspaces/ws-123/c.txt', Size: 50 },
|
||||
],
|
||||
});
|
||||
|
||||
const info = await store.getStorageInfo();
|
||||
|
||||
expect(info).toEqual({
|
||||
usedBytes: 400,
|
||||
fileCount: 3,
|
||||
storageType: 'virtual',
|
||||
});
|
||||
});
|
||||
});
|
||||
44
packages/core/tests/install-audit-check-parity.test.ts
Normal file
44
packages/core/tests/install-audit-check-parity.test.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* P7/D15 A3 — drift-lock for the install_audit CHECK constraints.
|
||||
*
|
||||
* The table is declared in TWO places: install-audit.ts (core, generated from
|
||||
* the canonical @waggle/shared arrays) and hive-mind-core/src/mind/schema.ts
|
||||
* (the OSS substrate, a standalone literal). They MUST produce identical CHECK
|
||||
* lists or auditStore.record() crashes on one path. This test pins both to the
|
||||
* single canonical source, so a drift in either fails CI instead of production.
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
sqlInList, RISK_LEVELS, APPROVAL_CLASSES, AUDIT_ACTIONS,
|
||||
AUDIT_CAPABILITY_TYPES, AUDIT_INITIATORS, TRUST_SOURCES,
|
||||
} from '@waggle/shared';
|
||||
import { SCHEMA_SQL } from '@waggle/hive-mind-core';
|
||||
import { INSTALL_AUDIT_TABLE_SQL } from '../src/install-audit.js';
|
||||
|
||||
const COLUMNS = [
|
||||
{ col: 'capability_type', values: AUDIT_CAPABILITY_TYPES },
|
||||
{ col: 'risk_level', values: RISK_LEVELS },
|
||||
{ col: 'trust_source', values: TRUST_SOURCES },
|
||||
{ col: 'approval_class', values: APPROVAL_CLASSES },
|
||||
{ col: 'action', values: AUDIT_ACTIONS },
|
||||
{ col: 'initiator', values: AUDIT_INITIATORS },
|
||||
] as const;
|
||||
|
||||
describe('install_audit CHECK parity (A3)', () => {
|
||||
for (const { col, values } of COLUMNS) {
|
||||
const expected = `${col} IN (${sqlInList(values)})`;
|
||||
|
||||
it(`core install-audit DDL pins ${col} to the canonical list`, () => {
|
||||
expect(INSTALL_AUDIT_TABLE_SQL).toContain(expected);
|
||||
});
|
||||
|
||||
it(`OSS substrate schema.ts pins ${col} to the canonical list`, () => {
|
||||
expect(SCHEMA_SQL).toContain(expected);
|
||||
});
|
||||
}
|
||||
|
||||
it('both DDLs accept the P5/D4 uninstalled action', () => {
|
||||
expect(INSTALL_AUDIT_TABLE_SQL).toContain("'uninstalled'");
|
||||
expect(SCHEMA_SQL).toContain("'uninstalled'");
|
||||
});
|
||||
});
|
||||
479
packages/core/tests/install-audit.test.ts
Normal file
479
packages/core/tests/install-audit.test.ts
Normal file
@@ -0,0 +1,479 @@
|
||||
/**
|
||||
* InstallAuditStore tests.
|
||||
*
|
||||
* Relocated here from packages/hive-mind-core/tests/mind/ — commit 05c9ec3
|
||||
* ("relocate substrate tests") moved this test to hive-mind-core, but the
|
||||
* source (`install-audit.ts`) stayed in @waggle/core and imports MindDB FROM
|
||||
* @waggle/hive-mind-core. hive-mind-core cannot depend back on @waggle/core
|
||||
* (dependency inversion + breaks the OSS parity model), so the relocated test
|
||||
* imported a non-existent `../../src/install-audit.js` and the entire suite
|
||||
* silently failed at collection — masking the FIX-3 CHECK-drift regression.
|
||||
* Its correct home is alongside the source, in @waggle/core.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { MindDB } from '@waggle/hive-mind-core';
|
||||
import { InstallAuditStore, type RecordAuditInput } from '../src/install-audit.js';
|
||||
|
||||
describe('InstallAuditStore', () => {
|
||||
let tmpDir: string;
|
||||
let db: MindDB;
|
||||
let store: InstallAuditStore;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-audit-'));
|
||||
db = new MindDB(path.join(tmpDir, 'test.mind'));
|
||||
store = new InstallAuditStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function makeInput(overrides?: Partial<RecordAuditInput>): RecordAuditInput {
|
||||
return {
|
||||
capabilityName: 'risk-assessment',
|
||||
capabilityType: 'skill',
|
||||
source: 'starter-pack',
|
||||
riskLevel: 'low',
|
||||
trustSource: 'starter_pack',
|
||||
approvalClass: 'standard',
|
||||
action: 'installed',
|
||||
initiator: 'agent',
|
||||
detail: 'Installed successfully',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
it("records an 'uninstalled' action (P5/D4 — capability removal trail)", () => {
|
||||
const entry = store.record(makeInput({ action: 'uninstalled', initiator: 'agent', detail: 'deleted by agent' }));
|
||||
expect(entry.action).toBe('uninstalled');
|
||||
expect(store.getByAction('uninstalled')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("migrates a legacy-CHECK table to accept 'uninstalled' (P5/D4)", () => {
|
||||
const tmp2 = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-audit-legacy-'));
|
||||
const legacyDb = new MindDB(path.join(tmp2, 'legacy.mind'));
|
||||
const raw = legacyDb.getDatabase();
|
||||
// Simulate a pre-P5 install: drop the migrated table and recreate it with
|
||||
// the OLD narrower action CHECK (no 'uninstalled'), seeding one row.
|
||||
raw.exec('DROP TABLE IF EXISTS install_audit');
|
||||
raw.exec(`
|
||||
CREATE TABLE install_audit (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
capability_name TEXT NOT NULL,
|
||||
capability_type TEXT NOT NULL CHECK (capability_type IN ('native','skill','plugin','mcp','connector','marketplace')),
|
||||
source TEXT NOT NULL,
|
||||
version TEXT,
|
||||
risk_level TEXT NOT NULL CHECK (risk_level IN ('low','medium','high','critical')),
|
||||
trust_source TEXT NOT NULL,
|
||||
approval_class TEXT NOT NULL CHECK (approval_class IN ('standard','elevated','critical','blocked')),
|
||||
action TEXT NOT NULL CHECK (action IN ('proposed','approved','installed','rejected','failed','blocked')),
|
||||
initiator TEXT NOT NULL CHECK (initiator IN ('agent','user','system')),
|
||||
detail TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
`);
|
||||
raw.prepare(`INSERT INTO install_audit
|
||||
(capability_name, capability_type, source, risk_level, trust_source, approval_class, action, initiator, detail)
|
||||
VALUES ('legacy-skill','skill','starter-pack','low','starter_pack','standard','installed','user','pre-migration row')`).run();
|
||||
|
||||
// Constructing the store triggers ensureTable() → rebuild migration.
|
||||
const migrated = new InstallAuditStore(legacyDb);
|
||||
// Pre-existing row survives the rebuild.
|
||||
expect(migrated.getByCapability('legacy-skill')).toHaveLength(1);
|
||||
// The widened CHECK now accepts 'uninstalled' (would throw on a stale table).
|
||||
const entry = migrated.record(makeInput({ capabilityName: 'legacy-skill', action: 'uninstalled', detail: 'removed' }));
|
||||
expect(entry.action).toBe('uninstalled');
|
||||
// Review #1: the rebuild must NOT drop the declared indexes (SQLite RENAME
|
||||
// carries index names to the legacy table; without an explicit DROP INDEX
|
||||
// they get destroyed with it, leaving the audit table index-less).
|
||||
const idx = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='install_audit' AND name LIKE 'idx_audit_%'",
|
||||
).all() as Array<{ name: string }>;
|
||||
expect(idx.map(i => i.name).sort()).toEqual(['idx_audit_capability', 'idx_audit_timestamp']);
|
||||
|
||||
// #15: the rebuild also added the trust_source CHECK (the legacy table had
|
||||
// trust_source unconstrained). The migrated DDL now carries it, and the
|
||||
// pre-migration row (trust_source 'starter_pack') survived because every
|
||||
// historical value is in the canonical 7-set.
|
||||
const ddl = raw.prepare(
|
||||
"SELECT sql FROM sqlite_master WHERE type='table' AND name='install_audit'",
|
||||
).get() as { sql: string };
|
||||
expect(ddl.sql).toContain('CHECK (trust_source IN');
|
||||
// A bogus trust_source is now rejected at the DB (was previously accepted).
|
||||
expect(() => raw.prepare(
|
||||
`INSERT INTO install_audit
|
||||
(capability_name, capability_type, source, risk_level, trust_source, approval_class, action, initiator, detail)
|
||||
VALUES ('x','skill','s','low','BOGUS_SOURCE','standard','installed','user','')`,
|
||||
).run()).toThrow();
|
||||
|
||||
legacyDb.close();
|
||||
fs.rmSync(tmp2, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('records and retrieves an audit entry', () => {
|
||||
const entry = store.record(makeInput());
|
||||
expect(entry.id).toBeGreaterThan(0);
|
||||
expect(entry.capability_name).toBe('risk-assessment');
|
||||
expect(entry.action).toBe('installed');
|
||||
expect(entry.risk_level).toBe('low');
|
||||
expect(entry.trust_source).toBe('starter_pack');
|
||||
expect(entry.timestamp).toBeTruthy();
|
||||
});
|
||||
|
||||
it('records multiple events for same capability', () => {
|
||||
store.record(makeInput({ action: 'proposed' }));
|
||||
store.record(makeInput({ action: 'approved' }));
|
||||
store.record(makeInput({ action: 'installed' }));
|
||||
|
||||
const history = store.getByCapability('risk-assessment');
|
||||
expect(history).toHaveLength(3);
|
||||
expect(history[0].action).toBe('installed');
|
||||
expect(history[2].action).toBe('proposed');
|
||||
});
|
||||
|
||||
it('queries by action type', () => {
|
||||
store.record(makeInput({ capabilityName: 'draft-memo', action: 'installed' }));
|
||||
store.record(makeInput({ capabilityName: 'code-review', action: 'proposed' }));
|
||||
store.record(makeInput({ capabilityName: 'brainstorm', action: 'installed' }));
|
||||
|
||||
const installed = store.getByAction('installed');
|
||||
expect(installed).toHaveLength(2);
|
||||
expect(installed.map(e => e.capability_name).sort()).toEqual(['brainstorm', 'draft-memo']);
|
||||
});
|
||||
|
||||
it('retrieves recent entries in descending order', () => {
|
||||
store.record(makeInput({ capabilityName: 'first', action: 'proposed' }));
|
||||
store.record(makeInput({ capabilityName: 'second', action: 'installed' }));
|
||||
store.record(makeInput({ capabilityName: 'third', action: 'failed' }));
|
||||
|
||||
const recent = store.getRecent(2);
|
||||
expect(recent).toHaveLength(2);
|
||||
expect(recent[0].capability_name).toBe('third');
|
||||
expect(recent[1].capability_name).toBe('second');
|
||||
});
|
||||
|
||||
it('preserves all fields round-trip', () => {
|
||||
const entry = store.record(makeInput({
|
||||
capabilityName: 'test-skill',
|
||||
capabilityType: 'plugin',
|
||||
source: 'third-party',
|
||||
version: '1.2.3',
|
||||
riskLevel: 'high',
|
||||
trustSource: 'third_party_unverified',
|
||||
approvalClass: 'critical',
|
||||
action: 'failed',
|
||||
initiator: 'user',
|
||||
detail: 'Permission denied by user',
|
||||
}));
|
||||
|
||||
expect(entry.capability_name).toBe('test-skill');
|
||||
expect(entry.capability_type).toBe('plugin');
|
||||
expect(entry.source).toBe('third-party');
|
||||
expect(entry.version).toBe('1.2.3');
|
||||
expect(entry.risk_level).toBe('high');
|
||||
expect(entry.trust_source).toBe('third_party_unverified');
|
||||
expect(entry.approval_class).toBe('critical');
|
||||
expect(entry.action).toBe('failed');
|
||||
expect(entry.initiator).toBe('user');
|
||||
expect(entry.detail).toBe('Permission denied by user');
|
||||
});
|
||||
|
||||
it('handles null version', () => {
|
||||
const entry = store.record(makeInput({ version: null }));
|
||||
expect(entry.version).toBeNull();
|
||||
});
|
||||
|
||||
it('handles empty detail', () => {
|
||||
const entry = store.record(makeInput({ detail: undefined }));
|
||||
expect(entry.detail).toBe('');
|
||||
});
|
||||
|
||||
it('getAll returns entries in insertion order', () => {
|
||||
store.record(makeInput({ capabilityName: 'a' }));
|
||||
store.record(makeInput({ capabilityName: 'b' }));
|
||||
store.record(makeInput({ capabilityName: 'c' }));
|
||||
|
||||
const all = store.getAll();
|
||||
expect(all).toHaveLength(3);
|
||||
expect(all[0].capability_name).toBe('a');
|
||||
expect(all[2].capability_name).toBe('c');
|
||||
});
|
||||
|
||||
it('clear removes all entries', () => {
|
||||
store.record(makeInput({ capabilityName: 'a' }));
|
||||
store.record(makeInput({ capabilityName: 'b' }));
|
||||
expect(store.getAll()).toHaveLength(2);
|
||||
|
||||
store.clear();
|
||||
expect(store.getAll()).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('creates table lazily on pre-existing databases', () => {
|
||||
const store2 = new InstallAuditStore(db);
|
||||
const entry = store2.record(makeInput({ capabilityName: 'from-second-store' }));
|
||||
expect(entry.capability_name).toBe('from-second-store');
|
||||
});
|
||||
|
||||
it('records failed validation as audit event', () => {
|
||||
const entry = store.record(makeInput({
|
||||
action: 'failed',
|
||||
riskLevel: 'low',
|
||||
trustSource: 'unknown',
|
||||
detail: 'Skill "nonexistent" not found in the starter pack.',
|
||||
}));
|
||||
expect(entry.action).toBe('failed');
|
||||
expect(entry.detail).toContain('not found');
|
||||
});
|
||||
|
||||
// P0-005/FIX-3: install_audit CHECK constraints drifted behind their TS
|
||||
// type unions. Once marketplace FTS search returned candidates,
|
||||
// acquire_capability recommended a `type:'marketplace'` capability and
|
||||
// auditStore.record() crashed with "CHECK constraint failed: capability_type
|
||||
// IN ('native','skill','plugin','mcp')", throwing the whole tool and
|
||||
// dead-ending the agent. These lock the CHECK <-> type-union alignment.
|
||||
it('records a marketplace-type capability (capability_type CHECK widened)', () => {
|
||||
const entry = store.record(makeInput({ capabilityType: 'marketplace', source: 'marketplace' }));
|
||||
expect(entry.capability_type).toBe('marketplace');
|
||||
});
|
||||
|
||||
it('records a connector-type capability', () => {
|
||||
const entry = store.record(makeInput({ capabilityType: 'connector', source: 'connector' }));
|
||||
expect(entry.capability_type).toBe('connector');
|
||||
});
|
||||
|
||||
it("records the 'blocked' action and 'blocked' approval_class (sibling CHECK drift)", () => {
|
||||
const entry = store.record(makeInput({ action: 'blocked', approvalClass: 'blocked' }));
|
||||
expect(entry.action).toBe('blocked');
|
||||
expect(entry.approval_class).toBe('blocked');
|
||||
});
|
||||
|
||||
// M2 (UX-Refactor Phase 4 / C15): risk_level CHECK lacked 'critical' while
|
||||
// the TS union had it — marketplace.ts's CRITICAL-block audit write was
|
||||
// silently rejected (throw swallowed by `catch {}`). This locks the widened
|
||||
// CHECK on fresh databases.
|
||||
it("records riskLevel 'critical' (M2: risk_level CHECK widened)", () => {
|
||||
const entry = store.record(makeInput({
|
||||
riskLevel: 'critical',
|
||||
approvalClass: 'blocked',
|
||||
action: 'blocked',
|
||||
trustSource: 'security-gate',
|
||||
detail: 'SecurityGate blocked: CRITICAL findings',
|
||||
}));
|
||||
expect(entry.risk_level).toBe('critical');
|
||||
expect(entry.action).toBe('blocked');
|
||||
});
|
||||
|
||||
// C18: type-filtered read backing GET /api/extend/audit?type=
|
||||
it('getRecentByType filters by capability_type, most recent first', () => {
|
||||
store.record(makeInput({ capabilityName: 'a-skill', capabilityType: 'skill' }));
|
||||
store.record(makeInput({ capabilityName: 'a-server', capabilityType: 'mcp' }));
|
||||
store.record(makeInput({ capabilityName: 'b-server', capabilityType: 'mcp' }));
|
||||
store.record(makeInput({ capabilityName: 'a-conn', capabilityType: 'connector' }));
|
||||
|
||||
const mcps = store.getRecentByType('mcp');
|
||||
expect(mcps).toHaveLength(2);
|
||||
expect(mcps[0].capability_name).toBe('b-server');
|
||||
expect(mcps[1].capability_name).toBe('a-server');
|
||||
expect(store.getRecentByType('mcp', 1)).toHaveLength(1);
|
||||
expect(store.getRecentByType('native')).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('InstallAuditStore — legacy CHECK migration', () => {
|
||||
let tmpDir: string;
|
||||
let dbPath: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-audit-mig-'));
|
||||
dbPath = path.join(tmpDir, 'legacy.mind');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const OLD_DDL = `CREATE TABLE install_audit (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
capability_name TEXT NOT NULL,
|
||||
capability_type TEXT NOT NULL CHECK (capability_type IN ('native', 'skill', 'plugin', 'mcp')),
|
||||
source TEXT NOT NULL,
|
||||
version TEXT,
|
||||
risk_level TEXT NOT NULL CHECK (risk_level IN ('low', 'medium', 'high')),
|
||||
trust_source TEXT NOT NULL,
|
||||
approval_class TEXT NOT NULL CHECK (approval_class IN ('standard', 'elevated', 'critical')),
|
||||
action TEXT NOT NULL CHECK (action IN ('proposed', 'approved', 'installed', 'rejected', 'failed')),
|
||||
initiator TEXT NOT NULL CHECK (initiator IN ('agent', 'user', 'system')),
|
||||
detail TEXT NOT NULL DEFAULT ''
|
||||
)`;
|
||||
|
||||
it('rebuilds a legacy install_audit table (narrow CHECK) and preserves rows', () => {
|
||||
// 1. Create the mind DB, then forcibly downgrade install_audit to the
|
||||
// historical 4-value CHECK with one legacy row — simulating a real
|
||||
// user .mind created before connector/marketplace existed.
|
||||
{
|
||||
const seed = new MindDB(dbPath);
|
||||
const raw = seed.getDatabase();
|
||||
raw.prepare('DROP TABLE IF EXISTS install_audit').run();
|
||||
raw.prepare(OLD_DDL).run();
|
||||
raw.prepare(`INSERT INTO install_audit
|
||||
(capability_name, capability_type, source, risk_level, trust_source, approval_class, action, initiator, detail)
|
||||
VALUES ('legacy-skill','skill','starter-pack','low','starter_pack','standard','installed','agent','pre-migration row')`).run();
|
||||
seed.close();
|
||||
}
|
||||
|
||||
// 2. Reopen — runMigrations() must rebuild install_audit with the widened CHECK.
|
||||
const db = new MindDB(dbPath);
|
||||
const store = new InstallAuditStore(db);
|
||||
|
||||
const legacy = store.getByCapability('legacy-skill');
|
||||
expect(legacy).toHaveLength(1);
|
||||
expect(legacy[0].detail).toBe('pre-migration row');
|
||||
|
||||
expect(() => store.record({
|
||||
capabilityName: 'filesystem', capabilityType: 'marketplace', source: 'marketplace',
|
||||
riskLevel: 'medium', trustSource: 'unknown', approvalClass: 'standard',
|
||||
action: 'proposed', initiator: 'agent', detail: 'Proposed for need: read external files',
|
||||
})).not.toThrow();
|
||||
const fs2 = store.getByCapability('filesystem');
|
||||
expect(fs2[0].capability_type).toBe('marketplace');
|
||||
|
||||
db.close();
|
||||
});
|
||||
|
||||
// M2 (UX-Refactor Phase 4 / C15): the FIX-3-era DDL had every list widened
|
||||
// EXCEPT risk_level — the exact shape real .minds created between FIX-3
|
||||
// (2026-05-17) and Phase 4 are in. Only the new "'low', 'medium', 'high',
|
||||
// 'critical'" sentinel triggers this rebuild ('critical' alone appears in
|
||||
// approval_class, so it is NOT the key).
|
||||
const FIX3_ERA_DDL = `CREATE TABLE install_audit (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
capability_name TEXT NOT NULL,
|
||||
capability_type TEXT NOT NULL CHECK (capability_type IN ('native', 'skill', 'plugin', 'mcp', 'connector', 'marketplace')),
|
||||
source TEXT NOT NULL,
|
||||
version TEXT,
|
||||
risk_level TEXT NOT NULL CHECK (risk_level IN ('low', 'medium', 'high')),
|
||||
trust_source TEXT NOT NULL,
|
||||
approval_class TEXT NOT NULL CHECK (approval_class IN ('standard', 'elevated', 'critical', 'blocked')),
|
||||
action TEXT NOT NULL CHECK (action IN ('proposed', 'approved', 'installed', 'rejected', 'failed', 'blocked')),
|
||||
initiator TEXT NOT NULL CHECK (initiator IN ('agent', 'user', 'system')),
|
||||
detail TEXT NOT NULL DEFAULT ''
|
||||
)`;
|
||||
|
||||
it("M2: rebuilds a FIX-3-era table (risk_level missing 'critical') and preserves rows", () => {
|
||||
{
|
||||
const seed = new MindDB(dbPath);
|
||||
const raw = seed.getDatabase();
|
||||
raw.prepare('DROP TABLE IF EXISTS install_audit').run();
|
||||
raw.prepare(FIX3_ERA_DDL).run();
|
||||
raw.prepare(`INSERT INTO install_audit
|
||||
(capability_name, capability_type, source, risk_level, trust_source, approval_class, action, initiator, detail)
|
||||
VALUES ('legacy-mcp','mcp','marketplace','high','security-gate','blocked','blocked','system','pre-M2 row')`).run();
|
||||
// Sanity: the legacy CHECK really rejects 'critical' (the live bug)
|
||||
expect(() => raw.prepare(`INSERT INTO install_audit
|
||||
(capability_name, capability_type, source, risk_level, trust_source, approval_class, action, initiator, detail)
|
||||
VALUES ('x','mcp','marketplace','critical','security-gate','blocked','blocked','system','')`).run()
|
||||
).toThrow(/CHECK/);
|
||||
seed.close();
|
||||
}
|
||||
|
||||
// Reopen — runMigrations() must rebuild keyed on the M2 sentinel.
|
||||
const db = new MindDB(dbPath);
|
||||
const store = new InstallAuditStore(db);
|
||||
|
||||
const legacy = store.getByCapability('legacy-mcp');
|
||||
expect(legacy).toHaveLength(1);
|
||||
expect(legacy[0].detail).toBe('pre-M2 row');
|
||||
expect(legacy[0].risk_level).toBe('high');
|
||||
|
||||
const critical = store.record({
|
||||
capabilityName: 'evil-pkg', capabilityType: 'marketplace', source: 'marketplace',
|
||||
riskLevel: 'critical', trustSource: 'security-gate', approvalClass: 'blocked',
|
||||
action: 'blocked', initiator: 'system', detail: 'SecurityGate blocked: CRITICAL',
|
||||
});
|
||||
expect(critical.risk_level).toBe('critical');
|
||||
db.close();
|
||||
|
||||
// Idempotence: a second reopen must NOT rebuild again (rows + ids stable).
|
||||
const db2 = new MindDB(dbPath);
|
||||
const store2 = new InstallAuditStore(db2);
|
||||
const all = store2.getAll();
|
||||
expect(all).toHaveLength(2);
|
||||
expect(all.map((e) => e.capability_name)).toEqual(['legacy-mcp', 'evil-pkg']);
|
||||
// The rebuilt DDL carries the sentinel, so a third store write still works.
|
||||
expect(() => store2.record({
|
||||
capabilityName: 'again', capabilityType: 'mcp', source: 'mcp',
|
||||
riskLevel: 'critical', trustSource: 'security-gate', approvalClass: 'blocked',
|
||||
action: 'blocked', initiator: 'system',
|
||||
})).not.toThrow();
|
||||
db2.close();
|
||||
});
|
||||
|
||||
// The rebuild now runs in ONE transaction, so a crash mid-rebuild rolls
|
||||
// back — but DBs damaged by a PRE-transactional crashed rebuild exist in the
|
||||
// wild with all rows stranded in install_audit__mig_old. These lock the
|
||||
// recovery the FIX-3 comment always promised but never performed.
|
||||
const LEGACY_ROW_INSERT = (table: string) => `INSERT INTO ${table}
|
||||
(capability_name, capability_type, source, risk_level, trust_source, approval_class, action, initiator, detail)
|
||||
VALUES ('stranded','mcp','marketplace','high','security-gate','blocked','blocked','system','pre-crash row')`;
|
||||
|
||||
it('recovers rows stranded by a crash between RENAME and recreate (install_audit missing)', () => {
|
||||
{
|
||||
const seed = new MindDB(dbPath);
|
||||
const raw = seed.getDatabase();
|
||||
raw.prepare('DROP TABLE IF EXISTS install_audit').run();
|
||||
raw.prepare(FIX3_ERA_DDL).run();
|
||||
raw.prepare(LEGACY_ROW_INSERT('install_audit')).run();
|
||||
// Simulate the pre-transactional crash: renamed aside, then process died
|
||||
// before SCHEMA_SQL recreated install_audit.
|
||||
raw.prepare('ALTER TABLE install_audit RENAME TO install_audit__mig_old').run();
|
||||
seed.close();
|
||||
}
|
||||
|
||||
const db = new MindDB(dbPath);
|
||||
const store = new InstallAuditStore(db);
|
||||
// Rows restored AND the rebuild completed (the restored table had the
|
||||
// FIX-3-era DDL, so the M2 sentinel re-triggered the rebuild)
|
||||
const rows = store.getByCapability('stranded');
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].detail).toBe('pre-crash row');
|
||||
expect(() => store.record({
|
||||
capabilityName: 'post-recovery', capabilityType: 'mcp', source: 'mcp',
|
||||
riskLevel: 'critical', trustSource: 'security-gate', approvalClass: 'blocked',
|
||||
action: 'blocked', initiator: 'system',
|
||||
})).not.toThrow();
|
||||
// No stale __mig_old left to be destroyed by a future rebuild
|
||||
const leftover = db.getDatabase().prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='install_audit__mig_old'",
|
||||
).get();
|
||||
expect(leftover).toBeUndefined();
|
||||
db.close();
|
||||
});
|
||||
|
||||
it('recovers rows stranded by a crash between recreate and copy-back (fresh empty install_audit)', () => {
|
||||
{
|
||||
const seed = new MindDB(dbPath); // creates the CURRENT empty install_audit
|
||||
const raw = seed.getDatabase();
|
||||
raw.prepare(FIX3_ERA_DDL.replace('CREATE TABLE install_audit', 'CREATE TABLE install_audit__mig_old')).run();
|
||||
raw.prepare(LEGACY_ROW_INSERT('install_audit__mig_old')).run();
|
||||
seed.close();
|
||||
}
|
||||
|
||||
const db = new MindDB(dbPath);
|
||||
const store = new InstallAuditStore(db);
|
||||
const rows = store.getByCapability('stranded');
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].detail).toBe('pre-crash row');
|
||||
const leftover = db.getDatabase().prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='install_audit__mig_old'",
|
||||
).get();
|
||||
expect(leftover).toBeUndefined();
|
||||
db.close();
|
||||
});
|
||||
});
|
||||
162
packages/core/tests/litellm-embedder.test.ts
Normal file
162
packages/core/tests/litellm-embedder.test.ts
Normal file
@@ -0,0 +1,162 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { createLiteLLMEmbedder } from '@waggle/hive-mind-core';
|
||||
|
||||
describe('createLiteLLMEmbedder', () => {
|
||||
const baseConfig = {
|
||||
litellmUrl: 'http://localhost:4000/v1',
|
||||
litellmApiKey: 'sk-test',
|
||||
model: 'text-embedding',
|
||||
dimensions: 8,
|
||||
};
|
||||
|
||||
function mockFetchOk(data: unknown) {
|
||||
return vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => data,
|
||||
} as unknown as Response);
|
||||
}
|
||||
|
||||
it('exposes the configured dimensions', () => {
|
||||
const embedder = createLiteLLMEmbedder({ ...baseConfig, dimensions: 256, fetch: mockFetchOk({}) });
|
||||
expect(embedder.dimensions).toBe(256);
|
||||
});
|
||||
|
||||
it('defaults dimensions to 1024', () => {
|
||||
const embedder = createLiteLLMEmbedder({
|
||||
litellmUrl: 'http://localhost:4000/v1',
|
||||
fetch: mockFetchOk({}),
|
||||
});
|
||||
expect(embedder.dimensions).toBe(1024);
|
||||
});
|
||||
|
||||
it('embed() calls the correct endpoint with Bearer auth', async () => {
|
||||
const fakeFetch = mockFetchOk({
|
||||
data: [{ embedding: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] }],
|
||||
});
|
||||
|
||||
const embedder = createLiteLLMEmbedder({ ...baseConfig, fetch: fakeFetch });
|
||||
const result = await embedder.embed('hello world');
|
||||
|
||||
expect(fakeFetch).toHaveBeenCalledOnce();
|
||||
const [url, options] = fakeFetch.mock.calls[0];
|
||||
expect(url).toBe('http://localhost:4000/v1/embeddings');
|
||||
expect(options.method).toBe('POST');
|
||||
expect(options.headers['Authorization']).toBe('Bearer sk-test');
|
||||
expect(options.headers['Content-Type']).toBe('application/json');
|
||||
|
||||
const body = JSON.parse(options.body);
|
||||
expect(body.model).toBe('text-embedding');
|
||||
expect(body.input).toBe('hello world');
|
||||
|
||||
expect(result).toBeInstanceOf(Float32Array);
|
||||
expect(result.length).toBe(8);
|
||||
expect(result[0]).toBeCloseTo(0.1);
|
||||
});
|
||||
|
||||
it('embed() strips trailing /v1 to avoid double path', async () => {
|
||||
const fakeFetch = mockFetchOk({
|
||||
data: [{ embedding: [1, 2, 3, 4, 5, 6, 7, 8] }],
|
||||
});
|
||||
|
||||
const embedder = createLiteLLMEmbedder({
|
||||
...baseConfig,
|
||||
litellmUrl: 'http://localhost:4000/v1',
|
||||
fetch: fakeFetch,
|
||||
});
|
||||
await embedder.embed('test');
|
||||
|
||||
const [url] = fakeFetch.mock.calls[0];
|
||||
expect(url).toBe('http://localhost:4000/v1/embeddings');
|
||||
});
|
||||
|
||||
it('embedBatch() returns multiple Float32Arrays', async () => {
|
||||
const fakeFetch = mockFetchOk({
|
||||
data: [
|
||||
{ embedding: [1, 2, 3, 4, 5, 6, 7, 8] },
|
||||
{ embedding: [8, 7, 6, 5, 4, 3, 2, 1] },
|
||||
],
|
||||
});
|
||||
|
||||
const embedder = createLiteLLMEmbedder({ ...baseConfig, fetch: fakeFetch });
|
||||
const results = await embedder.embedBatch(['hello', 'world']);
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results[0]).toBeInstanceOf(Float32Array);
|
||||
expect(results[1]).toBeInstanceOf(Float32Array);
|
||||
expect(results[0][0]).toBe(1);
|
||||
expect(results[1][0]).toBe(8);
|
||||
|
||||
// Should send array as input
|
||||
const body = JSON.parse(fakeFetch.mock.calls[0][1].body);
|
||||
expect(body.input).toEqual(['hello', 'world']);
|
||||
});
|
||||
|
||||
it('embedBatch() returns empty array for empty input', async () => {
|
||||
const fakeFetch = vi.fn();
|
||||
const embedder = createLiteLLMEmbedder({ ...baseConfig, fetch: fakeFetch });
|
||||
const results = await embedder.embedBatch([]);
|
||||
|
||||
expect(results).toEqual([]);
|
||||
expect(fakeFetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('throws on API error when fallbackToMock is false', async () => {
|
||||
const fakeFetch = vi.fn().mockResolvedValue({
|
||||
ok: false,
|
||||
status: 500,
|
||||
text: async () => 'Internal Server Error',
|
||||
} as unknown as Response);
|
||||
|
||||
const embedder = createLiteLLMEmbedder({ ...baseConfig, fetch: fakeFetch, fallbackToMock: false });
|
||||
await expect(embedder.embed('test')).rejects.toThrow('LiteLLM embeddings error (500)');
|
||||
});
|
||||
|
||||
it('falls back to mock on API error when fallbackToMock is true', async () => {
|
||||
const fakeFetch = vi.fn().mockResolvedValue({
|
||||
ok: false,
|
||||
status: 500,
|
||||
text: async () => 'error',
|
||||
} as unknown as Response);
|
||||
|
||||
const embedder = createLiteLLMEmbedder({ ...baseConfig, fetch: fakeFetch, fallbackToMock: true });
|
||||
const result = await embedder.embed('hello');
|
||||
|
||||
expect(result).toBeInstanceOf(Float32Array);
|
||||
expect(result.length).toBe(8);
|
||||
// Verify it's the deterministic mock: 'h' = 104, (104 - 128) / 128 = -0.1875
|
||||
expect(result[0]).toBeCloseTo(-0.1875);
|
||||
});
|
||||
|
||||
it('falls back to mock on network error when fallbackToMock is true', async () => {
|
||||
const fakeFetch = vi.fn().mockRejectedValue(new Error('ECONNREFUSED'));
|
||||
|
||||
const embedder = createLiteLLMEmbedder({ ...baseConfig, fetch: fakeFetch, fallbackToMock: true });
|
||||
const result = await embedder.embed('hi');
|
||||
|
||||
expect(result).toBeInstanceOf(Float32Array);
|
||||
expect(result.length).toBe(8);
|
||||
});
|
||||
|
||||
it('throws on network error when fallbackToMock is false', async () => {
|
||||
const fakeFetch = vi.fn().mockRejectedValue(new Error('ECONNREFUSED'));
|
||||
|
||||
const embedder = createLiteLLMEmbedder({ ...baseConfig, fetch: fakeFetch });
|
||||
await expect(embedder.embed('test')).rejects.toThrow('ECONNREFUSED');
|
||||
});
|
||||
|
||||
it('omits Authorization header when no API key is provided', async () => {
|
||||
const fakeFetch = mockFetchOk({
|
||||
data: [{ embedding: [1, 2, 3, 4, 5, 6, 7, 8] }],
|
||||
});
|
||||
|
||||
const embedder = createLiteLLMEmbedder({
|
||||
litellmUrl: 'http://localhost:4000/v1',
|
||||
dimensions: 8,
|
||||
fetch: fakeFetch,
|
||||
});
|
||||
await embedder.embed('test');
|
||||
|
||||
const [, options] = fakeFetch.mock.calls[0];
|
||||
expect(options.headers['Authorization']).toBeUndefined();
|
||||
});
|
||||
});
|
||||
340
packages/core/tests/memory-import.test.ts
Normal file
340
packages/core/tests/memory-import.test.ts
Normal file
@@ -0,0 +1,340 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { parseChatGPTExport, parseClaudeExport, extractKnowledge, processImport } from '../src/memory-import';
|
||||
|
||||
describe('ChatGPT Export Parser', () => {
|
||||
it('parses conversations with mapping structure', () => {
|
||||
const data = [
|
||||
{
|
||||
title: 'Test Chat',
|
||||
create_time: 1709000000,
|
||||
mapping: {
|
||||
'node1': {
|
||||
message: {
|
||||
author: { role: 'user' },
|
||||
content: { parts: ['Hello, I need help with React'] },
|
||||
create_time: 1709000001,
|
||||
},
|
||||
},
|
||||
'node2': {
|
||||
message: {
|
||||
author: { role: 'assistant' },
|
||||
content: { parts: ['Sure, I can help with React!'] },
|
||||
create_time: 1709000002,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const result = parseChatGPTExport(data);
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].title).toBe('Test Chat');
|
||||
expect(result[0].messages).toHaveLength(2);
|
||||
expect(result[0].messages[0].role).toBe('user');
|
||||
expect(result[0].messages[0].text).toBe('Hello, I need help with React');
|
||||
expect(result[0].source).toBe('chatgpt');
|
||||
});
|
||||
|
||||
it('skips system messages', () => {
|
||||
const data = [
|
||||
{
|
||||
title: 'Test',
|
||||
mapping: {
|
||||
'sys': { message: { author: { role: 'system' }, content: { parts: ['System prompt'] } } },
|
||||
'user': { message: { author: { role: 'user' }, content: { parts: ['Hi'] }, create_time: 1 } },
|
||||
},
|
||||
},
|
||||
];
|
||||
const result = parseChatGPTExport(data);
|
||||
expect(result[0].messages).toHaveLength(1);
|
||||
expect(result[0].messages[0].role).toBe('user');
|
||||
});
|
||||
|
||||
it('handles empty/missing conversations gracefully', () => {
|
||||
expect(parseChatGPTExport(null)).toEqual([]);
|
||||
expect(parseChatGPTExport({})).toEqual([]);
|
||||
expect(parseChatGPTExport([])).toEqual([]);
|
||||
});
|
||||
|
||||
it('sorts messages chronologically by create_time', () => {
|
||||
const data = [
|
||||
{
|
||||
title: 'Order Test',
|
||||
mapping: {
|
||||
'late': {
|
||||
message: {
|
||||
author: { role: 'user' },
|
||||
content: { parts: ['Second message'] },
|
||||
create_time: 200,
|
||||
},
|
||||
},
|
||||
'early': {
|
||||
message: {
|
||||
author: { role: 'user' },
|
||||
content: { parts: ['First message'] },
|
||||
create_time: 100,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
const result = parseChatGPTExport(data);
|
||||
expect(result[0].messages[0].text).toBe('First message');
|
||||
expect(result[0].messages[1].text).toBe('Second message');
|
||||
});
|
||||
|
||||
it('filters out conversations with no messages', () => {
|
||||
const data = [
|
||||
{ title: 'Empty', mapping: {} },
|
||||
{
|
||||
title: 'Has Messages',
|
||||
mapping: {
|
||||
'n1': { message: { author: { role: 'user' }, content: { parts: ['Hello'] }, create_time: 1 } },
|
||||
},
|
||||
},
|
||||
];
|
||||
const result = parseChatGPTExport(data);
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].title).toBe('Has Messages');
|
||||
});
|
||||
|
||||
it('handles wrapped format with conversations key', () => {
|
||||
const data = {
|
||||
conversations: [
|
||||
{
|
||||
title: 'Wrapped',
|
||||
mapping: {
|
||||
'n1': { message: { author: { role: 'user' }, content: { parts: ['Test'] }, create_time: 1 } },
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const result = parseChatGPTExport(data);
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].title).toBe('Wrapped');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Claude Export Parser', () => {
|
||||
it('parses conversations with chat_messages', () => {
|
||||
const data = [
|
||||
{
|
||||
name: 'Claude Chat',
|
||||
created_at: '2024-03-01T10:00:00Z',
|
||||
chat_messages: [
|
||||
{ sender: 'human', text: 'What is TypeScript?', created_at: '2024-03-01T10:00:01Z' },
|
||||
{ sender: 'assistant', text: 'TypeScript is a superset of JavaScript.', created_at: '2024-03-01T10:00:05Z' },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const result = parseClaudeExport(data);
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].title).toBe('Claude Chat');
|
||||
expect(result[0].messages).toHaveLength(2);
|
||||
expect(result[0].messages[0].role).toBe('user');
|
||||
expect(result[0].source).toBe('claude');
|
||||
});
|
||||
|
||||
it('handles empty exports gracefully', () => {
|
||||
expect(parseClaudeExport(null)).toEqual([]);
|
||||
expect(parseClaudeExport([])).toEqual([]);
|
||||
});
|
||||
|
||||
it('handles alternative field names (title, messages, role, content)', () => {
|
||||
const data = [
|
||||
{
|
||||
title: 'Alt Format',
|
||||
messages: [
|
||||
{ role: 'user', content: 'Hello from alt format', timestamp: '2024-03-01T10:00:00Z' },
|
||||
],
|
||||
},
|
||||
];
|
||||
const result = parseClaudeExport(data);
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].title).toBe('Alt Format');
|
||||
expect(result[0].messages[0].text).toBe('Hello from alt format');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Knowledge Extraction', () => {
|
||||
it('extracts decisions from user messages', () => {
|
||||
const convs = [{
|
||||
title: 'Tech Discussion',
|
||||
messages: [
|
||||
{ role: 'user' as const, text: 'I decided to use React for the frontend and Node.js for the backend' },
|
||||
{ role: 'assistant' as const, text: 'Great choice!' },
|
||||
],
|
||||
source: 'chatgpt' as const,
|
||||
}];
|
||||
|
||||
const knowledge = extractKnowledge(convs);
|
||||
const decisions = knowledge.filter(k => k.type === 'decision');
|
||||
expect(decisions.length).toBeGreaterThan(0);
|
||||
expect(decisions[0].content).toContain('decided');
|
||||
expect(decisions[0].importance).toBe('important');
|
||||
});
|
||||
|
||||
it('extracts preferences from user messages', () => {
|
||||
const convs = [{
|
||||
title: 'Preferences',
|
||||
messages: [
|
||||
{ role: 'user' as const, text: 'I prefer bullet-point summaries over long paragraphs' },
|
||||
],
|
||||
source: 'claude' as const,
|
||||
}];
|
||||
|
||||
const knowledge = extractKnowledge(convs);
|
||||
const prefs = knowledge.filter(k => k.type === 'preference');
|
||||
expect(prefs.length).toBeGreaterThan(0);
|
||||
expect(prefs[0].content).toContain('prefer');
|
||||
expect(prefs[0].importance).toBe('important');
|
||||
});
|
||||
|
||||
it('extracts facts about the user', () => {
|
||||
const convs = [{
|
||||
title: 'About me',
|
||||
messages: [
|
||||
{ role: 'user' as const, text: 'I work at Egzakta Advisory as a partner and consultant' },
|
||||
],
|
||||
source: 'chatgpt' as const,
|
||||
}];
|
||||
|
||||
const knowledge = extractKnowledge(convs);
|
||||
const facts = knowledge.filter(k => k.type === 'fact');
|
||||
expect(facts.length).toBeGreaterThan(0);
|
||||
expect(facts[0].content).toContain('Egzakta');
|
||||
});
|
||||
|
||||
it('extracts conversation topics', () => {
|
||||
const convs = [{
|
||||
title: 'Building a SaaS Platform',
|
||||
messages: [
|
||||
{ role: 'user' as const, text: 'Help me plan the architecture' },
|
||||
],
|
||||
source: 'chatgpt' as const,
|
||||
}];
|
||||
|
||||
const knowledge = extractKnowledge(convs);
|
||||
const topics = knowledge.filter(k => k.type === 'topic');
|
||||
expect(topics.length).toBeGreaterThan(0);
|
||||
expect(topics[0].content).toContain('Building a SaaS Platform');
|
||||
});
|
||||
|
||||
it('caps extraction at 100 items', () => {
|
||||
const convs = Array.from({ length: 200 }, (_, i) => ({
|
||||
title: `Conversation ${i}`,
|
||||
messages: [
|
||||
{ role: 'user' as const, text: `I decided to use approach ${i} for the implementation` },
|
||||
],
|
||||
source: 'chatgpt' as const,
|
||||
}));
|
||||
|
||||
const knowledge = extractKnowledge(convs);
|
||||
expect(knowledge.length).toBeLessThanOrEqual(100);
|
||||
});
|
||||
|
||||
it('deduplicates similar content', () => {
|
||||
const convs = [
|
||||
{
|
||||
title: 'Chat 1',
|
||||
messages: [{ role: 'user' as const, text: 'I prefer TypeScript over JavaScript for type safety' }],
|
||||
source: 'chatgpt' as const,
|
||||
},
|
||||
{
|
||||
title: 'Chat 2',
|
||||
messages: [{ role: 'user' as const, text: 'I prefer TypeScript over JavaScript for type safety' }],
|
||||
source: 'chatgpt' as const,
|
||||
},
|
||||
];
|
||||
|
||||
const knowledge = extractKnowledge(convs);
|
||||
const prefs = knowledge.filter(k => k.type === 'preference');
|
||||
expect(prefs.length).toBe(1); // Deduped
|
||||
});
|
||||
|
||||
it('skips very short and very long messages', () => {
|
||||
const convs = [{
|
||||
title: 'Test',
|
||||
messages: [
|
||||
{ role: 'user' as const, text: 'Hi' }, // Too short
|
||||
{ role: 'user' as const, text: 'x'.repeat(600) }, // Too long
|
||||
{ role: 'user' as const, text: 'I decided to use Python for data analysis tasks' }, // Just right
|
||||
],
|
||||
source: 'chatgpt' as const,
|
||||
}];
|
||||
|
||||
const knowledge = extractKnowledge(convs);
|
||||
const decisions = knowledge.filter(k => k.type === 'decision');
|
||||
expect(decisions.length).toBe(1);
|
||||
});
|
||||
|
||||
it('skips assistant messages', () => {
|
||||
const convs = [{
|
||||
title: 'Test',
|
||||
messages: [
|
||||
{ role: 'assistant' as const, text: 'I decided to use a different approach for this solution' },
|
||||
],
|
||||
source: 'chatgpt' as const,
|
||||
}];
|
||||
|
||||
const knowledge = extractKnowledge(convs);
|
||||
const decisions = knowledge.filter(k => k.type === 'decision');
|
||||
expect(decisions.length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('processImport (end-to-end)', () => {
|
||||
it('processes a ChatGPT export end-to-end', () => {
|
||||
const data = [
|
||||
{
|
||||
title: 'Project Planning',
|
||||
create_time: 1709000000,
|
||||
mapping: {
|
||||
'n1': { message: { author: { role: 'user' }, content: { parts: ['I decided to use React with TypeScript for the frontend'] }, create_time: 1 } },
|
||||
'n2': { message: { author: { role: 'assistant' }, content: { parts: ['Great choice!'] }, create_time: 2 } },
|
||||
'n3': { message: { author: { role: 'user' }, content: { parts: ['I prefer concise responses without filler words'] }, create_time: 3 } },
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const result = processImport(data, 'chatgpt');
|
||||
expect(result.source).toBe('chatgpt');
|
||||
expect(result.conversationsFound).toBe(1);
|
||||
expect(result.conversationsParsed).toBe(1);
|
||||
expect(result.knowledgeExtracted.length).toBeGreaterThan(0);
|
||||
expect(result.errors).toHaveLength(0);
|
||||
|
||||
const decisions = result.knowledgeExtracted.filter(k => k.type === 'decision');
|
||||
expect(decisions.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('processes a Claude export end-to-end', () => {
|
||||
const data = [
|
||||
{
|
||||
name: 'Architecture Discussion',
|
||||
created_at: '2024-03-01T10:00:00Z',
|
||||
chat_messages: [
|
||||
{ sender: 'human', text: 'I work at Acme Corp as a senior engineer', created_at: '2024-03-01T10:00:01Z' },
|
||||
{ sender: 'assistant', text: 'Nice to meet you!', created_at: '2024-03-01T10:00:05Z' },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const result = processImport(data, 'claude');
|
||||
expect(result.source).toBe('claude');
|
||||
expect(result.conversationsFound).toBe(1);
|
||||
expect(result.knowledgeExtracted.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('returns errors for invalid data', () => {
|
||||
const result = processImport('not json', 'chatgpt');
|
||||
expect(result.conversationsFound).toBe(0);
|
||||
expect(result.knowledgeExtracted).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('reports when no conversations found', () => {
|
||||
const result = processImport([], 'chatgpt');
|
||||
expect(result.errors).toContain('No conversations found in export');
|
||||
});
|
||||
});
|
||||
125
packages/core/tests/migration.test.ts
Normal file
125
packages/core/tests/migration.test.ts
Normal file
@@ -0,0 +1,125 @@
|
||||
import { describe, it, expect, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { MindDB } from '@waggle/hive-mind-core';
|
||||
import { FrameStore } from '@waggle/hive-mind-core';
|
||||
import { needsMigration, migrateToMultiMind } from '../src/migration.js';
|
||||
|
||||
describe('Migration: default.mind → personal.mind', () => {
|
||||
let tmpDir: string;
|
||||
|
||||
function makeTmpDir(): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-migration-'));
|
||||
return dir;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
if (tmpDir && fs.existsSync(tmpDir)) {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
describe('needsMigration', () => {
|
||||
it('returns true when default.mind exists and personal.mind does not', () => {
|
||||
tmpDir = makeTmpDir();
|
||||
// Create a real MindDB so it's a valid SQLite file
|
||||
const db = new MindDB(path.join(tmpDir, 'default.mind'));
|
||||
db.close();
|
||||
|
||||
expect(needsMigration(tmpDir)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns false when personal.mind already exists', () => {
|
||||
tmpDir = makeTmpDir();
|
||||
// Both exist
|
||||
const db1 = new MindDB(path.join(tmpDir, 'default.mind'));
|
||||
db1.close();
|
||||
const db2 = new MindDB(path.join(tmpDir, 'personal.mind'));
|
||||
db2.close();
|
||||
|
||||
expect(needsMigration(tmpDir)).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false on fresh install (nothing exists)', () => {
|
||||
tmpDir = makeTmpDir();
|
||||
expect(needsMigration(tmpDir)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('migrateToMultiMind', () => {
|
||||
it('migrates default.mind to personal.mind with data preserved', () => {
|
||||
tmpDir = makeTmpDir();
|
||||
const defaultPath = path.join(tmpDir, 'default.mind');
|
||||
|
||||
// Create a MindDB with real data
|
||||
const db = new MindDB(defaultPath);
|
||||
// Create a session (foreign key requirement)
|
||||
db.getDatabase().prepare(
|
||||
"INSERT OR IGNORE INTO sessions (gop_id, status) VALUES (?, 'active')"
|
||||
).run('test-gop');
|
||||
const frames = new FrameStore(db);
|
||||
frames.createIFrame('test-gop', 'migration test content', 'normal');
|
||||
db.close();
|
||||
|
||||
const result = migrateToMultiMind(tmpDir);
|
||||
|
||||
expect(result.migrated).toBe(true);
|
||||
expect(result.message).toBe('Migrated default.mind to personal.mind');
|
||||
|
||||
// Verify personal.mind has the data
|
||||
const personalPath = path.join(tmpDir, 'personal.mind');
|
||||
expect(fs.existsSync(personalPath)).toBe(true);
|
||||
|
||||
const personalDb = new MindDB(personalPath);
|
||||
const personalFrames = new FrameStore(personalDb);
|
||||
const state = personalFrames.reconstructState('test-gop');
|
||||
expect(state.iframe).not.toBeNull();
|
||||
expect(state.iframe!.content).toBe('migration test content');
|
||||
personalDb.close();
|
||||
});
|
||||
|
||||
it('keeps default.mind as backup (.bak) after migration', () => {
|
||||
tmpDir = makeTmpDir();
|
||||
const db = new MindDB(path.join(tmpDir, 'default.mind'));
|
||||
db.close();
|
||||
|
||||
migrateToMultiMind(tmpDir);
|
||||
|
||||
expect(fs.existsSync(path.join(tmpDir, 'default.mind.bak'))).toBe(true);
|
||||
expect(fs.existsSync(path.join(tmpDir, 'default.mind'))).toBe(false);
|
||||
});
|
||||
|
||||
it('creates workspaces directory', () => {
|
||||
tmpDir = makeTmpDir();
|
||||
const db = new MindDB(path.join(tmpDir, 'default.mind'));
|
||||
db.close();
|
||||
|
||||
migrateToMultiMind(tmpDir);
|
||||
|
||||
const wsDir = path.join(tmpDir, 'workspaces');
|
||||
expect(fs.existsSync(wsDir)).toBe(true);
|
||||
expect(fs.statSync(wsDir).isDirectory()).toBe(true);
|
||||
});
|
||||
|
||||
it('does not migrate twice (idempotent)', () => {
|
||||
tmpDir = makeTmpDir();
|
||||
const db = new MindDB(path.join(tmpDir, 'default.mind'));
|
||||
db.close();
|
||||
|
||||
const first = migrateToMultiMind(tmpDir);
|
||||
expect(first.migrated).toBe(true);
|
||||
|
||||
const second = migrateToMultiMind(tmpDir);
|
||||
expect(second.migrated).toBe(false);
|
||||
expect(second.message).toBe('No migration needed');
|
||||
});
|
||||
|
||||
it('returns no migration needed when nothing exists', () => {
|
||||
tmpDir = makeTmpDir();
|
||||
const result = migrateToMultiMind(tmpDir);
|
||||
expect(result.migrated).toBe(false);
|
||||
expect(result.message).toBe('No migration needed');
|
||||
});
|
||||
});
|
||||
});
|
||||
115
packages/core/tests/skill-hashes.test.ts
Normal file
115
packages/core/tests/skill-hashes.test.ts
Normal file
@@ -0,0 +1,115 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { MindDB } from '@waggle/hive-mind-core';
|
||||
import { SkillHashStore, computeSkillHash } from '../src/skill-hashes.js';
|
||||
|
||||
describe('SkillHashStore', () => {
|
||||
let tmpDir: string;
|
||||
let db: MindDB;
|
||||
let store: SkillHashStore;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-skill-hash-'));
|
||||
db = new MindDB(path.join(tmpDir, 'test.mind'));
|
||||
store = new SkillHashStore(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('setHash + getHash round-trips correctly', () => {
|
||||
const hash = computeSkillHash('# My Skill\nDo something useful');
|
||||
store.setHash('my-skill', hash);
|
||||
|
||||
const stored = store.getHash('my-skill');
|
||||
expect(stored).toBeDefined();
|
||||
expect(stored!.name).toBe('my-skill');
|
||||
expect(stored!.hash).toBe(hash);
|
||||
expect(stored!.verified_at).toBeTruthy();
|
||||
});
|
||||
|
||||
it('computeSkillHash produces consistent SHA-256', () => {
|
||||
const content = '# Draft Memo\nHelp the user draft professional memos.';
|
||||
const hash1 = computeSkillHash(content);
|
||||
const hash2 = computeSkillHash(content);
|
||||
|
||||
expect(hash1).toBe(hash2);
|
||||
// SHA-256 hex is 64 characters
|
||||
expect(hash1).toHaveLength(64);
|
||||
// Should be hex string
|
||||
expect(hash1).toMatch(/^[0-9a-f]{64}$/);
|
||||
});
|
||||
|
||||
it('checkAll detects changed skill', () => {
|
||||
const originalContent = '# Skill v1\nOriginal content';
|
||||
store.verify('changed-skill', originalContent);
|
||||
|
||||
const result = store.checkAll([
|
||||
{ name: 'changed-skill', content: '# Skill v2\nModified content' },
|
||||
]);
|
||||
|
||||
expect(result.changed).toEqual(['changed-skill']);
|
||||
expect(result.added).toEqual([]);
|
||||
expect(result.removed).toEqual([]);
|
||||
});
|
||||
|
||||
it('checkAll detects new (added) skill', () => {
|
||||
const result = store.checkAll([
|
||||
{ name: 'brand-new-skill', content: '# New Skill\nFresh content' },
|
||||
]);
|
||||
|
||||
expect(result.added).toEqual(['brand-new-skill']);
|
||||
expect(result.changed).toEqual([]);
|
||||
expect(result.removed).toEqual([]);
|
||||
});
|
||||
|
||||
it('checkAll detects removed skill', () => {
|
||||
store.verify('old-skill', '# Old Skill\nGone now');
|
||||
|
||||
const result = store.checkAll([]);
|
||||
|
||||
expect(result.removed).toEqual(['old-skill']);
|
||||
expect(result.changed).toEqual([]);
|
||||
expect(result.added).toEqual([]);
|
||||
});
|
||||
|
||||
it('checkAll returns empty when nothing changed', () => {
|
||||
const content = '# Stable Skill\nNothing changed here';
|
||||
store.verify('stable-skill', content);
|
||||
|
||||
const result = store.checkAll([
|
||||
{ name: 'stable-skill', content },
|
||||
]);
|
||||
|
||||
expect(result.changed).toEqual([]);
|
||||
expect(result.added).toEqual([]);
|
||||
expect(result.removed).toEqual([]);
|
||||
});
|
||||
|
||||
it('verify updates hash to current content', () => {
|
||||
const v1 = '# Skill v1';
|
||||
const v2 = '# Skill v2';
|
||||
store.verify('evolving-skill', v1);
|
||||
|
||||
const hashBefore = store.getHash('evolving-skill')!.hash;
|
||||
expect(hashBefore).toBe(computeSkillHash(v1));
|
||||
|
||||
store.verify('evolving-skill', v2);
|
||||
|
||||
const hashAfter = store.getHash('evolving-skill')!.hash;
|
||||
expect(hashAfter).toBe(computeSkillHash(v2));
|
||||
expect(hashAfter).not.toBe(hashBefore);
|
||||
});
|
||||
|
||||
it('removeHash cleans up', () => {
|
||||
store.verify('doomed-skill', '# Doomed');
|
||||
expect(store.getHash('doomed-skill')).toBeDefined();
|
||||
|
||||
store.removeHash('doomed-skill');
|
||||
expect(store.getHash('doomed-skill')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
99
packages/core/tests/structured-tasks.test.ts
Normal file
99
packages/core/tests/structured-tasks.test.ts
Normal file
@@ -0,0 +1,99 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { MindDB } from '@waggle/hive-mind-core';
|
||||
import { AwarenessLayer } from '@waggle/hive-mind-core';
|
||||
|
||||
describe('Structured Task Model', () => {
|
||||
let db: MindDB;
|
||||
let awareness: AwarenessLayer;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new MindDB(':memory:');
|
||||
awareness = new AwarenessLayer(db);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
db.close();
|
||||
});
|
||||
|
||||
it('stores task with metadata', () => {
|
||||
const item = awareness.add('task', 'Fix the login bug', 0, undefined, {
|
||||
status: 'pending',
|
||||
context: 'auth module',
|
||||
priority: 'high',
|
||||
});
|
||||
expect(item).toBeDefined();
|
||||
expect(item.content).toBe('Fix the login bug');
|
||||
const meta = awareness.parseMetadata(item);
|
||||
expect(meta.status).toBe('pending');
|
||||
expect(meta.context).toBe('auth module');
|
||||
expect(meta.priority).toBe('high');
|
||||
});
|
||||
|
||||
it('stores task without metadata (defaults to empty object)', () => {
|
||||
const item = awareness.add('task', 'Simple task');
|
||||
expect(item.metadata).toBe('{}');
|
||||
const meta = awareness.parseMetadata(item);
|
||||
expect(meta).toEqual({});
|
||||
});
|
||||
|
||||
it('updates metadata on existing task', () => {
|
||||
const item = awareness.add('task', 'Deploy v2.0', 0, undefined, { status: 'pending' });
|
||||
awareness.updateMetadata(item.id, { status: 'in_progress', result: 'deploying...' });
|
||||
const updated = awareness.get(item.id);
|
||||
expect(updated).toBeDefined();
|
||||
const meta = awareness.parseMetadata(updated!);
|
||||
expect(meta.status).toBe('in_progress');
|
||||
expect(meta.result).toBe('deploying...');
|
||||
});
|
||||
|
||||
it('merges metadata without losing existing fields', () => {
|
||||
const item = awareness.add('task', 'Multi-step task', 0, undefined, {
|
||||
status: 'pending',
|
||||
context: 'deployment',
|
||||
});
|
||||
awareness.updateMetadata(item.id, { status: 'in_progress' });
|
||||
const updated = awareness.get(item.id);
|
||||
const meta = awareness.parseMetadata(updated!);
|
||||
expect(meta.status).toBe('in_progress');
|
||||
expect(meta.context).toBe('deployment'); // preserved
|
||||
});
|
||||
|
||||
it('throws when updating metadata for nonexistent item', () => {
|
||||
expect(() => awareness.updateMetadata(999, { status: 'done' })).toThrow('Awareness item 999 not found');
|
||||
});
|
||||
|
||||
it('retrieves tasks by status', () => {
|
||||
awareness.add('task', 'Task A', 0, undefined, { status: 'pending' });
|
||||
awareness.add('task', 'Task B', 0, undefined, { status: 'done' });
|
||||
awareness.add('task', 'Task C', 0, undefined, { status: 'pending' });
|
||||
const pending = awareness.getByStatus('pending');
|
||||
expect(pending.length).toBe(2);
|
||||
expect(pending.every(i => awareness.parseMetadata(i).status === 'pending')).toBe(true);
|
||||
});
|
||||
|
||||
it('getByStatus returns empty array when no matches', () => {
|
||||
awareness.add('task', 'Task A', 0, undefined, { status: 'pending' });
|
||||
const inProgress = awareness.getByStatus('in_progress');
|
||||
expect(inProgress).toEqual([]);
|
||||
});
|
||||
|
||||
it('getByStatus ignores items without metadata status', () => {
|
||||
awareness.add('task', 'No metadata task');
|
||||
awareness.add('task', 'Has status', 0, undefined, { status: 'pending' });
|
||||
const pending = awareness.getByStatus('pending');
|
||||
expect(pending.length).toBe(1);
|
||||
});
|
||||
|
||||
it('get() retrieves a single item by id', () => {
|
||||
const item = awareness.add('task', 'Find me', 5);
|
||||
const found = awareness.get(item.id);
|
||||
expect(found).toBeDefined();
|
||||
expect(found!.content).toBe('Find me');
|
||||
expect(found!.priority).toBe(5);
|
||||
});
|
||||
|
||||
it('get() returns undefined for nonexistent id', () => {
|
||||
const found = awareness.get(999);
|
||||
expect(found).toBeUndefined();
|
||||
});
|
||||
});
|
||||
297
packages/core/tests/team-sync.test.ts
Normal file
297
packages/core/tests/team-sync.test.ts
Normal file
@@ -0,0 +1,297 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
import { frameToEntity, entityToSyncedFrame, TeamSync, type TeamSyncConfig } from '../src/team-sync.js';
|
||||
import type { MemoryFrame } from '@waggle/hive-mind-core';
|
||||
|
||||
describe('frameToEntity', () => {
|
||||
it('converts a MemoryFrame to team entity format', () => {
|
||||
const frame: MemoryFrame = {
|
||||
id: 42,
|
||||
frame_type: 'I',
|
||||
gop_id: 'project-context',
|
||||
t: 3,
|
||||
base_frame_id: null,
|
||||
content: 'The project uses React and TypeScript',
|
||||
importance: 'important',
|
||||
access_count: 5,
|
||||
created_at: '2026-03-12T10:00:00.000Z',
|
||||
last_accessed: '2026-03-12T12:00:00.000Z',
|
||||
};
|
||||
|
||||
const entity = frameToEntity(frame, 'user-abc', 'Marko');
|
||||
|
||||
expect(entity.entityType).toBe('memory_frame');
|
||||
expect(entity.name).toBe('project-context');
|
||||
expect(entity.properties.frameType).toBe('I');
|
||||
expect(entity.properties.t).toBe(3);
|
||||
expect(entity.properties.baseFrameId).toBeNull();
|
||||
expect(entity.properties.content).toBe('The project uses React and TypeScript');
|
||||
expect(entity.properties.importance).toBe('important');
|
||||
expect(entity.properties.authorId).toBe('user-abc');
|
||||
expect(entity.properties.authorName).toBe('Marko');
|
||||
expect(entity.properties.localId).toBe(42);
|
||||
});
|
||||
|
||||
it('preserves P-frame base_frame_id', () => {
|
||||
const frame: MemoryFrame = {
|
||||
id: 43,
|
||||
frame_type: 'P',
|
||||
gop_id: 'project-context',
|
||||
t: 4,
|
||||
base_frame_id: 42,
|
||||
content: 'Updated: now also using Tailwind',
|
||||
importance: 'normal',
|
||||
access_count: 0,
|
||||
created_at: '2026-03-12T11:00:00.000Z',
|
||||
last_accessed: '2026-03-12T11:00:00.000Z',
|
||||
};
|
||||
|
||||
const entity = frameToEntity(frame, 'user-xyz', 'Ana');
|
||||
|
||||
expect(entity.properties.frameType).toBe('P');
|
||||
expect(entity.properties.baseFrameId).toBe(42);
|
||||
expect(entity.properties.authorName).toBe('Ana');
|
||||
});
|
||||
});
|
||||
|
||||
describe('entityToSyncedFrame', () => {
|
||||
it('converts a team entity back to SyncedFrame', () => {
|
||||
const entity = {
|
||||
id: 'uuid-remote-1',
|
||||
name: 'project-context',
|
||||
properties: {
|
||||
frameType: 'I',
|
||||
t: 3,
|
||||
baseFrameId: null,
|
||||
content: 'The project uses React and TypeScript',
|
||||
importance: 'important',
|
||||
authorId: 'user-abc',
|
||||
authorName: 'Marko',
|
||||
localId: 42,
|
||||
},
|
||||
createdAt: '2026-03-12T10:00:00.000Z',
|
||||
};
|
||||
|
||||
const frame = entityToSyncedFrame(entity);
|
||||
|
||||
expect(frame.remoteId).toBe('uuid-remote-1');
|
||||
expect(frame.gopId).toBe('project-context');
|
||||
expect(frame.t).toBe(3);
|
||||
expect(frame.frameType).toBe('I');
|
||||
expect(frame.content).toBe('The project uses React and TypeScript');
|
||||
expect(frame.importance).toBe('important');
|
||||
expect(frame.authorId).toBe('user-abc');
|
||||
expect(frame.authorName).toBe('Marko');
|
||||
expect(frame.createdAt).toBe('2026-03-12T10:00:00.000Z');
|
||||
});
|
||||
|
||||
it('handles missing properties gracefully', () => {
|
||||
const entity = {
|
||||
id: 'uuid-remote-2',
|
||||
name: 'some-gop',
|
||||
properties: {},
|
||||
createdAt: '2026-03-12T10:00:00.000Z',
|
||||
};
|
||||
|
||||
const frame = entityToSyncedFrame(entity);
|
||||
|
||||
expect(frame.remoteId).toBe('uuid-remote-2');
|
||||
expect(frame.gopId).toBe('some-gop');
|
||||
expect(frame.t).toBe(0);
|
||||
expect(frame.frameType).toBe('I');
|
||||
expect(frame.content).toBe('');
|
||||
expect(frame.importance).toBe('normal');
|
||||
expect(frame.authorId).toBe('');
|
||||
expect(frame.authorName).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('TeamSync', () => {
|
||||
const mockConfig: TeamSyncConfig = {
|
||||
teamServerUrl: 'https://team.waggle.dev',
|
||||
teamSlug: 'test-team',
|
||||
authToken: 'test-jwt-token',
|
||||
userId: 'user-abc',
|
||||
displayName: 'Marko',
|
||||
};
|
||||
|
||||
let fetchSpy: ReturnType<typeof vi.fn>;
|
||||
|
||||
beforeEach(() => {
|
||||
fetchSpy = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchSpy);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('pushFrame', () => {
|
||||
it('sends frame to team server entities endpoint', async () => {
|
||||
fetchSpy.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ id: 'remote-uuid-1' }),
|
||||
});
|
||||
|
||||
const sync = new TeamSync(mockConfig);
|
||||
const frame: MemoryFrame = {
|
||||
id: 1,
|
||||
frame_type: 'I',
|
||||
gop_id: 'test-gop',
|
||||
t: 0,
|
||||
base_frame_id: null,
|
||||
content: 'Test content',
|
||||
importance: 'normal',
|
||||
access_count: 0,
|
||||
created_at: '2026-03-12T10:00:00.000Z',
|
||||
last_accessed: '2026-03-12T10:00:00.000Z',
|
||||
};
|
||||
|
||||
const result = await sync.pushFrame(frame);
|
||||
|
||||
expect(result).toEqual({ remoteId: 'remote-uuid-1' });
|
||||
expect(fetchSpy).toHaveBeenCalledOnce();
|
||||
|
||||
const [url, opts] = fetchSpy.mock.calls[0];
|
||||
expect(url).toBe('https://team.waggle.dev/api/teams/test-team/entities');
|
||||
expect(opts.method).toBe('POST');
|
||||
expect(opts.headers['Authorization']).toBe('Bearer test-jwt-token');
|
||||
|
||||
const body = JSON.parse(opts.body);
|
||||
expect(body.entityType).toBe('memory_frame');
|
||||
expect(body.name).toBe('test-gop');
|
||||
expect(body.properties.content).toBe('Test content');
|
||||
expect(body.properties.authorId).toBe('user-abc');
|
||||
expect(body.properties.authorName).toBe('Marko');
|
||||
});
|
||||
|
||||
it('returns null on server error', async () => {
|
||||
fetchSpy.mockResolvedValue({
|
||||
ok: false,
|
||||
status: 500,
|
||||
statusText: 'Internal Server Error',
|
||||
});
|
||||
|
||||
const sync = new TeamSync(mockConfig);
|
||||
const frame: MemoryFrame = {
|
||||
id: 1, frame_type: 'I', gop_id: 'test', t: 0, base_frame_id: null,
|
||||
content: 'x', importance: 'normal', access_count: 0,
|
||||
created_at: '2026-03-12T10:00:00.000Z', last_accessed: '2026-03-12T10:00:00.000Z',
|
||||
};
|
||||
|
||||
const result = await sync.pushFrame(frame);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null on network error', async () => {
|
||||
fetchSpy.mockRejectedValue(new Error('Network unreachable'));
|
||||
|
||||
const sync = new TeamSync(mockConfig);
|
||||
const frame: MemoryFrame = {
|
||||
id: 1, frame_type: 'I', gop_id: 'test', t: 0, base_frame_id: null,
|
||||
content: 'x', importance: 'normal', access_count: 0,
|
||||
created_at: '2026-03-12T10:00:00.000Z', last_accessed: '2026-03-12T10:00:00.000Z',
|
||||
};
|
||||
|
||||
const result = await sync.pushFrame(frame);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('pullFrames', () => {
|
||||
it('fetches frames from team server', async () => {
|
||||
fetchSpy.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ([
|
||||
{
|
||||
id: 'uuid-1',
|
||||
name: 'gop-a',
|
||||
properties: { frameType: 'I', t: 0, content: 'First', importance: 'important', authorId: 'u1', authorName: 'Marko' },
|
||||
createdAt: '2026-03-12T10:00:00.000Z',
|
||||
},
|
||||
{
|
||||
id: 'uuid-2',
|
||||
name: 'gop-b',
|
||||
properties: { frameType: 'P', t: 1, content: 'Second', importance: 'normal', authorId: 'u2', authorName: 'Ana' },
|
||||
createdAt: '2026-03-12T11:00:00.000Z',
|
||||
},
|
||||
]),
|
||||
});
|
||||
|
||||
const sync = new TeamSync(mockConfig);
|
||||
const frames = await sync.pullFrames();
|
||||
|
||||
expect(frames).toHaveLength(2);
|
||||
expect(frames[0].gopId).toBe('gop-a');
|
||||
expect(frames[0].authorName).toBe('Marko');
|
||||
expect(frames[1].gopId).toBe('gop-b');
|
||||
expect(frames[1].authorName).toBe('Ana');
|
||||
|
||||
const [url, opts] = fetchSpy.mock.calls[0];
|
||||
expect(url).toContain('/api/teams/test-team/entities?type=memory_frame');
|
||||
expect(opts.headers['Authorization']).toBe('Bearer test-jwt-token');
|
||||
});
|
||||
|
||||
it('filters by since timestamp when provided', async () => {
|
||||
fetchSpy.mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ([
|
||||
{
|
||||
id: 'uuid-1', name: 'gop-a',
|
||||
properties: { frameType: 'I', t: 0, content: 'Old', authorId: 'u1', authorName: 'M' },
|
||||
createdAt: '2026-03-11T10:00:00.000Z',
|
||||
},
|
||||
{
|
||||
id: 'uuid-2', name: 'gop-b',
|
||||
properties: { frameType: 'I', t: 0, content: 'New', authorId: 'u2', authorName: 'A' },
|
||||
createdAt: '2026-03-12T15:00:00.000Z',
|
||||
},
|
||||
]),
|
||||
});
|
||||
|
||||
const sync = new TeamSync(mockConfig);
|
||||
const frames = await sync.pullFrames('2026-03-12T00:00:00.000Z');
|
||||
|
||||
// Only the frame after the since timestamp
|
||||
expect(frames).toHaveLength(1);
|
||||
expect(frames[0].content).toBe('New');
|
||||
});
|
||||
|
||||
it('returns empty array on server error', async () => {
|
||||
fetchSpy.mockResolvedValue({ ok: false, status: 500, statusText: 'Error' });
|
||||
|
||||
const sync = new TeamSync(mockConfig);
|
||||
const frames = await sync.pullFrames();
|
||||
expect(frames).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns empty array on network error', async () => {
|
||||
fetchSpy.mockRejectedValue(new Error('Offline'));
|
||||
|
||||
const sync = new TeamSync(mockConfig);
|
||||
const frames = await sync.pullFrames();
|
||||
expect(frames).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sync timestamp tracking', () => {
|
||||
it('starts with null timestamp', () => {
|
||||
const sync = new TeamSync(mockConfig);
|
||||
expect(sync.getLastSyncTimestamp()).toBeNull();
|
||||
});
|
||||
|
||||
it('updates timestamp after successful pull', async () => {
|
||||
fetchSpy.mockResolvedValue({ ok: true, json: async () => ([]) });
|
||||
|
||||
const sync = new TeamSync(mockConfig);
|
||||
await sync.pullFrames();
|
||||
|
||||
expect(sync.getLastSyncTimestamp()).toBeTruthy();
|
||||
});
|
||||
|
||||
it('allows manual timestamp setting', () => {
|
||||
const sync = new TeamSync(mockConfig);
|
||||
sync.setLastSyncTimestamp('2026-03-12T10:00:00.000Z');
|
||||
expect(sync.getLastSyncTimestamp()).toBe('2026-03-12T10:00:00.000Z');
|
||||
});
|
||||
});
|
||||
});
|
||||
112
packages/core/tests/telemetry.test.ts
Normal file
112
packages/core/tests/telemetry.test.ts
Normal file
@@ -0,0 +1,112 @@
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { TelemetryCollector } from '../src/telemetry.js';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
|
||||
describe('TelemetryCollector', () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = path.join(os.tmpdir(), `waggle-telemetry-test-${Date.now()}`);
|
||||
fs.mkdirSync(tmpDir, { recursive: true });
|
||||
});
|
||||
|
||||
it('records events when enabled', () => {
|
||||
const collector = new TelemetryCollector(tmpDir, true);
|
||||
collector.recordToolUse('web_search');
|
||||
collector.recordToolUse('web_search');
|
||||
collector.recordToolUse('save_memory');
|
||||
|
||||
const report = collector.getReport();
|
||||
expect(report.totalEvents).toBe(3);
|
||||
const webSearch = report.events.find(e => e.name === 'web_search');
|
||||
expect(webSearch?.count).toBe(2);
|
||||
});
|
||||
|
||||
it('silently drops events when disabled', () => {
|
||||
const collector = new TelemetryCollector(tmpDir, false);
|
||||
collector.recordToolUse('web_search');
|
||||
collector.recordCommand('/research');
|
||||
collector.recordError('timeout');
|
||||
|
||||
const report = collector.getReport();
|
||||
expect(report.totalEvents).toBe(0);
|
||||
});
|
||||
|
||||
it('no PII in collected data — tool names only', () => {
|
||||
const collector = new TelemetryCollector(tmpDir, true);
|
||||
collector.recordToolUse('connector_github_create_issue');
|
||||
collector.recordError('api_timeout');
|
||||
collector.recordCapabilityGap('email sending');
|
||||
|
||||
const report = collector.getReport();
|
||||
for (const event of report.events) {
|
||||
// Only category, name, count, date — no message content or file paths
|
||||
expect(Object.keys(event).sort()).toEqual(['category', 'count', 'date', 'name']);
|
||||
}
|
||||
});
|
||||
|
||||
it('daily aggregation: same-day events merge counts', () => {
|
||||
const collector = new TelemetryCollector(tmpDir, true);
|
||||
collector.recordToolUse('bash');
|
||||
collector.recordToolUse('bash');
|
||||
collector.recordToolUse('bash');
|
||||
|
||||
const report = collector.getReport();
|
||||
const bashEvents = report.events.filter(e => e.name === 'bash');
|
||||
expect(bashEvents).toHaveLength(1);
|
||||
expect(bashEvents[0].count).toBe(3);
|
||||
});
|
||||
|
||||
it('flush writes to telemetry.json', () => {
|
||||
const collector = new TelemetryCollector(tmpDir, true);
|
||||
collector.recordToolUse('search_memory');
|
||||
collector.flush();
|
||||
|
||||
const filePath = path.join(tmpDir, 'telemetry.json');
|
||||
expect(fs.existsSync(filePath)).toBe(true);
|
||||
|
||||
const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
|
||||
expect(data).toHaveLength(1);
|
||||
expect(data[0].name).toBe('search_memory');
|
||||
|
||||
collector.close();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('report covers last N days', () => {
|
||||
const collector = new TelemetryCollector(tmpDir, true);
|
||||
collector.recordToolUse('today_tool');
|
||||
|
||||
const report = collector.getReport(7);
|
||||
expect(report.events.length).toBeGreaterThan(0);
|
||||
expect(report.dateRange.from).toBeTruthy();
|
||||
});
|
||||
|
||||
it('setEnabled toggles collection', () => {
|
||||
const collector = new TelemetryCollector(tmpDir, false);
|
||||
expect(collector.isEnabled()).toBe(false);
|
||||
|
||||
collector.setEnabled(true);
|
||||
expect(collector.isEnabled()).toBe(true);
|
||||
|
||||
collector.recordToolUse('test_tool');
|
||||
expect(collector.getReport().totalEvents).toBe(1);
|
||||
});
|
||||
|
||||
it('recordSession aggregates duration and interaction count', () => {
|
||||
const collector = new TelemetryCollector(tmpDir, true);
|
||||
collector.recordSession(30000, 15);
|
||||
collector.recordSession(20000, 10);
|
||||
|
||||
const report = collector.getReport();
|
||||
const duration = report.events.find(e => e.name === 'duration_total_ms');
|
||||
const interactions = report.events.find(e => e.name === 'interaction_count');
|
||||
const sessions = report.events.find(e => e.name === 'session_count');
|
||||
|
||||
expect(duration?.count).toBe(50000);
|
||||
expect(interactions?.count).toBe(25);
|
||||
expect(sessions?.count).toBe(2);
|
||||
});
|
||||
});
|
||||
136
packages/core/tests/vault-concurrency.test.ts
Normal file
136
packages/core/tests/vault-concurrency.test.ts
Normal file
@@ -0,0 +1,136 @@
|
||||
import { describe, it, expect, afterEach } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { VaultStore } from '../src/vault.js';
|
||||
|
||||
describe('Vault Concurrency & Atomic Writes (11B-8)', () => {
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
function makeTempDir(): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-vault-conc-'));
|
||||
tempDirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of tempDirs) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
tempDirs.length = 0;
|
||||
});
|
||||
|
||||
it('5 concurrent setAsync calls — all 5 keys exist after', async () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
// Fire 5 concurrent writes
|
||||
await Promise.all([
|
||||
vault.setAsync('key-1', 'value-1'),
|
||||
vault.setAsync('key-2', 'value-2'),
|
||||
vault.setAsync('key-3', 'value-3'),
|
||||
vault.setAsync('key-4', 'value-4'),
|
||||
vault.setAsync('key-5', 'value-5'),
|
||||
]);
|
||||
|
||||
// Verify all 5 keys exist and have correct values
|
||||
for (let i = 1; i <= 5; i++) {
|
||||
const entry = vault.get(`key-${i}`);
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry!.value).toBe(`value-${i}`);
|
||||
}
|
||||
|
||||
// Verify via list that all 5 are present
|
||||
const list = vault.list();
|
||||
expect(list).toHaveLength(5);
|
||||
});
|
||||
|
||||
it('concurrent setAsync and deleteAsync do not corrupt vault', async () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
// Set some initial keys
|
||||
vault.set('keep-1', 'v1');
|
||||
vault.set('remove-1', 'v2');
|
||||
vault.set('keep-2', 'v3');
|
||||
|
||||
// Concurrently set new keys and delete existing ones
|
||||
await Promise.all([
|
||||
vault.setAsync('new-1', 'new-v1'),
|
||||
vault.deleteAsync('remove-1'),
|
||||
vault.setAsync('new-2', 'new-v2'),
|
||||
]);
|
||||
|
||||
expect(vault.get('keep-1')?.value).toBe('v1');
|
||||
expect(vault.get('keep-2')?.value).toBe('v3');
|
||||
expect(vault.get('new-1')?.value).toBe('new-v1');
|
||||
expect(vault.get('new-2')?.value).toBe('new-v2');
|
||||
expect(vault.get('remove-1')).toBeNull();
|
||||
});
|
||||
|
||||
it('atomic write — no vault.json.tmp left behind after write', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
vault.set('test', 'value');
|
||||
|
||||
const tmpPath = path.join(dir, 'vault.json.tmp');
|
||||
expect(fs.existsSync(tmpPath)).toBe(false);
|
||||
expect(fs.existsSync(path.join(dir, 'vault.json'))).toBe(true);
|
||||
});
|
||||
|
||||
it('deleteAsync returns correct existed flag', async () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
vault.set('exists', 'val');
|
||||
|
||||
const existed = await vault.deleteAsync('exists');
|
||||
expect(existed).toBe(true);
|
||||
|
||||
const notExisted = await vault.deleteAsync('never-was');
|
||||
expect(notExisted).toBe(false);
|
||||
});
|
||||
|
||||
it('setAsync overwrites existing key', async () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
vault.set('overwrite-me', 'old-value');
|
||||
await vault.setAsync('overwrite-me', 'new-value');
|
||||
|
||||
const entry = vault.get('overwrite-me');
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry!.value).toBe('new-value');
|
||||
});
|
||||
|
||||
it('10 concurrent setAsync calls on same key — last one wins', async () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
// Chain them — since writeLock serializes, the last .then() should win
|
||||
const promises: Promise<void>[] = [];
|
||||
for (let i = 0; i < 10; i++) {
|
||||
promises.push(vault.setAsync('contested', `value-${i}`));
|
||||
}
|
||||
await Promise.all(promises);
|
||||
|
||||
const entry = vault.get('contested');
|
||||
expect(entry).not.toBeNull();
|
||||
// The value should be the last one written (value-9)
|
||||
// because the write lock serializes them in order
|
||||
expect(entry!.value).toBe('value-9');
|
||||
});
|
||||
|
||||
it('sync set still works correctly (backward compatibility)', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
vault.set('sync-key', 'sync-value', { tag: 'test' });
|
||||
const entry = vault.get('sync-key');
|
||||
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry!.value).toBe('sync-value');
|
||||
expect(entry!.metadata).toEqual({ tag: 'test' });
|
||||
});
|
||||
});
|
||||
261
packages/core/tests/vault-edge-cases.test.ts
Normal file
261
packages/core/tests/vault-edge-cases.test.ts
Normal file
@@ -0,0 +1,261 @@
|
||||
/**
|
||||
* Vault edge case tests — corrupted files, missing files,
|
||||
* concurrent read/write safety, and large value storage.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, afterEach } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { VaultStore } from '../src/vault.js';
|
||||
|
||||
describe('VaultStore edge cases', () => {
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
function makeTempDir(): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-vault-edge-'));
|
||||
tempDirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of tempDirs) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
tempDirs.length = 0;
|
||||
});
|
||||
|
||||
// ── Corrupted vault.json ────────────────────────────────────────────
|
||||
|
||||
describe('corrupted vault.json', () => {
|
||||
it('handles invalid JSON gracefully — returns null for get', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
// Store a valid secret first
|
||||
vault.set('test-key', 'test-value');
|
||||
expect(vault.get('test-key')!.value).toBe('test-value');
|
||||
|
||||
// Corrupt the vault.json file with invalid JSON
|
||||
const vaultPath = path.join(dir, 'vault.json');
|
||||
fs.writeFileSync(vaultPath, '{ this is not valid JSON !!!', 'utf-8');
|
||||
|
||||
// get should return null (not crash)
|
||||
expect(vault.get('test-key')).toBeNull();
|
||||
});
|
||||
|
||||
it('handles invalid JSON gracefully — list returns empty array', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
// Corrupt the vault.json file
|
||||
const vaultPath = path.join(dir, 'vault.json');
|
||||
fs.writeFileSync(vaultPath, '<xml>not json</xml>', 'utf-8');
|
||||
|
||||
// list should return empty (not crash)
|
||||
expect(vault.list()).toEqual([]);
|
||||
});
|
||||
|
||||
it('handles invalid JSON gracefully — has returns false', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
// Corrupt the vault.json file
|
||||
const vaultPath = path.join(dir, 'vault.json');
|
||||
fs.writeFileSync(vaultPath, '}}broken{{', 'utf-8');
|
||||
|
||||
// has should return false (not crash)
|
||||
expect(vault.has('anything')).toBe(false);
|
||||
});
|
||||
|
||||
it('can write new secrets after corruption — overwrites corrupted file', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
// Corrupt the vault.json file
|
||||
const vaultPath = path.join(dir, 'vault.json');
|
||||
fs.writeFileSync(vaultPath, 'CORRUPT!', 'utf-8');
|
||||
|
||||
// set should overwrite corrupted file with valid data
|
||||
vault.set('recovery-key', 'recovered-value');
|
||||
|
||||
// Should be able to read back the new value
|
||||
const entry = vault.get('recovery-key');
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry!.value).toBe('recovered-value');
|
||||
|
||||
// Verify the file is now valid JSON
|
||||
const raw = fs.readFileSync(vaultPath, 'utf-8');
|
||||
expect(() => JSON.parse(raw)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Missing vault.json ──────────────────────────────────────────────
|
||||
|
||||
describe('missing vault.json', () => {
|
||||
it('creates a new vault.json on next write when file is missing', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
const vaultPath = path.join(dir, 'vault.json');
|
||||
|
||||
// Ensure no vault.json exists initially
|
||||
if (fs.existsSync(vaultPath)) {
|
||||
fs.unlinkSync(vaultPath);
|
||||
}
|
||||
expect(fs.existsSync(vaultPath)).toBe(false);
|
||||
|
||||
// Write a secret — should create vault.json
|
||||
vault.set('new-key', 'new-value');
|
||||
|
||||
expect(fs.existsSync(vaultPath)).toBe(true);
|
||||
const entry = vault.get('new-key');
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry!.value).toBe('new-value');
|
||||
});
|
||||
|
||||
it('get returns null when vault.json does not exist', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
const vaultPath = path.join(dir, 'vault.json');
|
||||
|
||||
// Ensure no vault.json exists
|
||||
if (fs.existsSync(vaultPath)) {
|
||||
fs.unlinkSync(vaultPath);
|
||||
}
|
||||
|
||||
expect(vault.get('missing')).toBeNull();
|
||||
});
|
||||
|
||||
it('list returns empty array when vault.json does not exist', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
const vaultPath = path.join(dir, 'vault.json');
|
||||
|
||||
// Ensure no vault.json exists
|
||||
if (fs.existsSync(vaultPath)) {
|
||||
fs.unlinkSync(vaultPath);
|
||||
}
|
||||
|
||||
expect(vault.list()).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Concurrent reads during write ──────────────────────────────────
|
||||
|
||||
describe('concurrent reads during write', () => {
|
||||
it('read during setAsync does not corrupt data', async () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
// Pre-populate with a known value
|
||||
vault.set('existing', 'original-value');
|
||||
|
||||
// Start an async write, then immediately read
|
||||
const writePromise = vault.setAsync('new-key', 'new-value');
|
||||
const readResult = vault.get('existing');
|
||||
|
||||
await writePromise;
|
||||
|
||||
// The original key should still be readable (not corrupted)
|
||||
expect(readResult).not.toBeNull();
|
||||
expect(readResult!.value).toBe('original-value');
|
||||
|
||||
// The new key should also be present
|
||||
const newEntry = vault.get('new-key');
|
||||
expect(newEntry).not.toBeNull();
|
||||
expect(newEntry!.value).toBe('new-value');
|
||||
|
||||
// The existing key should still be intact
|
||||
const existingAfter = vault.get('existing');
|
||||
expect(existingAfter).not.toBeNull();
|
||||
expect(existingAfter!.value).toBe('original-value');
|
||||
});
|
||||
|
||||
it('multiple rapid reads interleaved with writes produce consistent results', async () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
// Write initial data
|
||||
vault.set('stable', 'stable-value');
|
||||
|
||||
// Fire off writes and reads in rapid succession
|
||||
const writePromises = [];
|
||||
for (let i = 0; i < 5; i++) {
|
||||
writePromises.push(vault.setAsync(`rapid-${i}`, `value-${i}`));
|
||||
}
|
||||
|
||||
// Interleave reads
|
||||
const readResults: (string | null)[] = [];
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const entry = vault.get('stable');
|
||||
readResults.push(entry?.value ?? null);
|
||||
}
|
||||
|
||||
await Promise.all(writePromises);
|
||||
|
||||
// All reads of the stable key should return the correct value
|
||||
for (const val of readResults) {
|
||||
expect(val).toBe('stable-value');
|
||||
}
|
||||
|
||||
// All written keys should be present
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const entry = vault.get(`rapid-${i}`);
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry!.value).toBe(`value-${i}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── Large value storage ────────────────────────────────────────────
|
||||
|
||||
describe('large value storage', () => {
|
||||
it('stores and retrieves a 1MB string correctly', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
// Generate a 1MB string (1,048,576 characters)
|
||||
const largeValue = 'A'.repeat(1024 * 1024);
|
||||
expect(largeValue.length).toBe(1024 * 1024);
|
||||
|
||||
vault.set('large-secret', largeValue);
|
||||
|
||||
const entry = vault.get('large-secret');
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry!.value).toBe(largeValue);
|
||||
expect(entry!.value.length).toBe(1024 * 1024);
|
||||
});
|
||||
|
||||
it('large value is encrypted in vault.json — plaintext not present', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
// Use a distinctive pattern that would be easy to find if unencrypted
|
||||
const largeValue = 'SECRET_MARKER_'.repeat(10000);
|
||||
vault.set('large-encrypted', largeValue);
|
||||
|
||||
const rawContent = fs.readFileSync(path.join(dir, 'vault.json'), 'utf-8');
|
||||
expect(rawContent).not.toContain('SECRET_MARKER_');
|
||||
|
||||
// Verify the encrypted field has the correct format
|
||||
const parsed = JSON.parse(rawContent);
|
||||
expect(parsed['large-encrypted'].encrypted).toMatch(/^[0-9a-f]+:[0-9a-f]+:[0-9a-f]+$/);
|
||||
});
|
||||
|
||||
it('large value survives round-trip through different VaultStore instances', () => {
|
||||
const dir = makeTempDir();
|
||||
|
||||
// Write with one instance
|
||||
const vault1 = new VaultStore(dir);
|
||||
const largeValue = 'B'.repeat(1024 * 1024);
|
||||
vault1.set('large-roundtrip', largeValue);
|
||||
|
||||
// Read with a different instance (same key file)
|
||||
const vault2 = new VaultStore(dir);
|
||||
const entry = vault2.get('large-roundtrip');
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry!.value).toBe(largeValue);
|
||||
expect(entry!.value.length).toBe(1024 * 1024);
|
||||
});
|
||||
});
|
||||
});
|
||||
391
packages/core/tests/vault.test.ts
Normal file
391
packages/core/tests/vault.test.ts
Normal file
@@ -0,0 +1,391 @@
|
||||
import { describe, it, expect, afterEach, vi } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
// Hoisted mock for node:child_process so static imports in vault.ts are intercepted.
|
||||
// Defaults to the real implementation; individual tests override via mockImplementation.
|
||||
const mockExecFileSync = vi.hoisted(() => vi.fn());
|
||||
vi.mock('node:child_process', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('node:child_process')>();
|
||||
return { ...actual, execFileSync: mockExecFileSync };
|
||||
});
|
||||
|
||||
import { VaultStore, type VaultEntry } from '../src/vault.js';
|
||||
|
||||
describe('VaultStore', () => {
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
function makeTempDir(): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-vault-test-'));
|
||||
tempDirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of tempDirs) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
tempDirs.length = 0;
|
||||
});
|
||||
|
||||
it('set and get — store a secret, retrieve it, value matches', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
vault.set('anthropic', 'sk-ant-secret-key-123');
|
||||
|
||||
const entry = vault.get('anthropic');
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry!.name).toBe('anthropic');
|
||||
expect(entry!.value).toBe('sk-ant-secret-key-123');
|
||||
expect(entry!.updatedAt).toBeTruthy();
|
||||
});
|
||||
|
||||
it('set overwrites — set same name twice, get returns latest', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
vault.set('openai', 'sk-old-key');
|
||||
vault.set('openai', 'sk-new-key');
|
||||
|
||||
const entry = vault.get('openai');
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry!.value).toBe('sk-new-key');
|
||||
});
|
||||
|
||||
it('get nonexistent — returns null', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
expect(vault.get('doesnotexist')).toBeNull();
|
||||
});
|
||||
|
||||
it('delete — removes secret, get returns null after', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
vault.set('anthropic', 'sk-ant-key');
|
||||
expect(vault.delete('anthropic')).toBe(true);
|
||||
expect(vault.get('anthropic')).toBeNull();
|
||||
});
|
||||
|
||||
it('delete nonexistent — returns false', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
expect(vault.delete('nope')).toBe(false);
|
||||
});
|
||||
|
||||
it('list — shows names + metadata without values', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
vault.set('anthropic', 'sk-ant-key', { models: ['claude-sonnet-4-6'] });
|
||||
vault.set('openai', 'sk-openai-key', { models: ['gpt-4o'], baseUrl: 'https://api.openai.com' });
|
||||
|
||||
const entries = vault.list();
|
||||
expect(entries).toHaveLength(2);
|
||||
|
||||
const names = entries.map(e => e.name);
|
||||
expect(names).toContain('anthropic');
|
||||
expect(names).toContain('openai');
|
||||
|
||||
// list must NOT contain secret values
|
||||
for (const entry of entries) {
|
||||
expect(entry).not.toHaveProperty('value');
|
||||
expect(entry.updatedAt).toBeTruthy();
|
||||
}
|
||||
|
||||
const anthropicEntry = entries.find(e => e.name === 'anthropic')!;
|
||||
expect(anthropicEntry.metadata).toEqual({ models: ['claude-sonnet-4-6'] });
|
||||
});
|
||||
|
||||
it('has — returns true for existing, false for missing', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
vault.set('anthropic', 'sk-ant-key');
|
||||
|
||||
expect(vault.has('anthropic')).toBe(true);
|
||||
expect(vault.has('missing')).toBe(false);
|
||||
});
|
||||
|
||||
it('encryption is real — vault.json does NOT contain plaintext value', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
const secret = 'sk-ant-super-secret-api-key-12345';
|
||||
|
||||
vault.set('anthropic', secret);
|
||||
|
||||
const rawContent = fs.readFileSync(path.join(dir, 'vault.json'), 'utf-8');
|
||||
expect(rawContent).not.toContain(secret);
|
||||
|
||||
// The encrypted field should exist and contain hex data with colons (iv:tag:ciphertext)
|
||||
const parsed = JSON.parse(rawContent);
|
||||
expect(parsed.anthropic.encrypted).toMatch(/^[0-9a-f]+:[0-9a-f]+:[0-9a-f]+$/);
|
||||
});
|
||||
|
||||
it('different VaultStore instances with same key can decrypt', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault1 = new VaultStore(dir);
|
||||
vault1.set('anthropic', 'sk-ant-shared-secret');
|
||||
|
||||
// Create a second instance pointing at the same directory (same key file)
|
||||
const vault2 = new VaultStore(dir);
|
||||
const entry = vault2.get('anthropic');
|
||||
|
||||
expect(entry).not.toBeNull();
|
||||
expect(entry!.value).toBe('sk-ant-shared-secret');
|
||||
});
|
||||
|
||||
it('migration from config — providers migrated to vault', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
const config = {
|
||||
providers: {
|
||||
anthropic: { apiKey: 'sk-ant-key-1', models: ['claude-sonnet-4-6'], baseUrl: undefined },
|
||||
openai: { apiKey: 'sk-openai-key-1', models: ['gpt-4o'] },
|
||||
},
|
||||
};
|
||||
|
||||
const migrated = vault.migrateFromConfig(config);
|
||||
expect(migrated).toBe(2);
|
||||
|
||||
const anthropic = vault.get('anthropic');
|
||||
expect(anthropic).not.toBeNull();
|
||||
expect(anthropic!.value).toBe('sk-ant-key-1');
|
||||
expect(anthropic!.metadata?.models).toEqual(['claude-sonnet-4-6']);
|
||||
|
||||
const openai = vault.get('openai');
|
||||
expect(openai).not.toBeNull();
|
||||
expect(openai!.value).toBe('sk-openai-key-1');
|
||||
});
|
||||
|
||||
it('migration skips existing — migrate twice, count stays same', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
const config = {
|
||||
providers: {
|
||||
anthropic: { apiKey: 'sk-ant-key-1', models: ['claude-sonnet-4-6'] },
|
||||
},
|
||||
};
|
||||
|
||||
const first = vault.migrateFromConfig(config);
|
||||
expect(first).toBe(1);
|
||||
|
||||
const second = vault.migrateFromConfig(config);
|
||||
expect(second).toBe(0);
|
||||
});
|
||||
|
||||
it('key file is generated on first use and reused', () => {
|
||||
const dir = makeTempDir();
|
||||
new VaultStore(dir);
|
||||
|
||||
const keyPath = path.join(dir, '.vault-key');
|
||||
expect(fs.existsSync(keyPath)).toBe(true);
|
||||
|
||||
// Key should be 64 hex chars (32 bytes)
|
||||
const keyHex = fs.readFileSync(keyPath, 'utf-8').trim();
|
||||
expect(keyHex).toMatch(/^[0-9a-f]{64}$/);
|
||||
|
||||
// Second instance should use the same key (not overwrite)
|
||||
new VaultStore(dir);
|
||||
const keyHex2 = fs.readFileSync(keyPath, 'utf-8').trim();
|
||||
expect(keyHex2).toBe(keyHex);
|
||||
});
|
||||
|
||||
it('setConnectorCredential — refresh token is NOT stored as plaintext in vault.json', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
const refreshToken = 'rt-super-secret-refresh-token-xyz';
|
||||
|
||||
vault.setConnectorCredential('github', {
|
||||
type: 'oauth2',
|
||||
value: 'gho_access_token_123',
|
||||
refreshToken,
|
||||
expiresAt: '2099-01-01T00:00:00Z',
|
||||
scopes: ['repo', 'user'],
|
||||
});
|
||||
|
||||
// Read raw vault.json and verify the refresh token is NOT in plaintext
|
||||
const rawContent = fs.readFileSync(path.join(dir, 'vault.json'), 'utf-8');
|
||||
expect(rawContent).not.toContain(refreshToken);
|
||||
|
||||
// The metadata for the main connector entry must NOT contain refreshToken
|
||||
const parsed = JSON.parse(rawContent);
|
||||
const mainEntry = parsed['connector:github'];
|
||||
expect(mainEntry).toBeDefined();
|
||||
expect(mainEntry.metadata).not.toHaveProperty('refreshToken');
|
||||
|
||||
// The refresh token should be stored as a separate encrypted entry
|
||||
const refreshEntry = parsed['connector:github:refresh'];
|
||||
expect(refreshEntry).toBeDefined();
|
||||
expect(refreshEntry.encrypted).toMatch(/^[0-9a-f]+:[0-9a-f]+:[0-9a-f]+$/);
|
||||
});
|
||||
|
||||
it('getConnectorCredential — returns decrypted refresh token correctly', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
const accessToken = 'gho_access_token_456';
|
||||
const refreshToken = 'rt-secret-refresh-token-abc';
|
||||
|
||||
vault.setConnectorCredential('github', {
|
||||
type: 'oauth2',
|
||||
value: accessToken,
|
||||
refreshToken,
|
||||
expiresAt: '2099-01-01T00:00:00Z',
|
||||
scopes: ['repo'],
|
||||
});
|
||||
|
||||
const cred = vault.getConnectorCredential('github');
|
||||
expect(cred).not.toBeNull();
|
||||
expect(cred!.value).toBe(accessToken);
|
||||
expect(cred!.refreshToken).toBe(refreshToken);
|
||||
expect(cred!.type).toBe('oauth2');
|
||||
expect(cred!.expiresAt).toBe('2099-01-01T00:00:00Z');
|
||||
expect(cred!.scopes).toEqual(['repo']);
|
||||
expect(cred!.isExpired).toBe(false);
|
||||
});
|
||||
|
||||
it('setConnectorCredential — without refresh token does not create refresh entry', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
vault.setConnectorCredential('slack', {
|
||||
type: 'bearer',
|
||||
value: 'xoxb-token-123',
|
||||
});
|
||||
|
||||
const rawContent = fs.readFileSync(path.join(dir, 'vault.json'), 'utf-8');
|
||||
const parsed = JSON.parse(rawContent);
|
||||
expect(parsed['connector:slack']).toBeDefined();
|
||||
expect(parsed['connector:slack:refresh']).toBeUndefined();
|
||||
|
||||
const cred = vault.getConnectorCredential('slack');
|
||||
expect(cred).not.toBeNull();
|
||||
expect(cred!.value).toBe('xoxb-token-123');
|
||||
expect(cred!.refreshToken).toBeUndefined();
|
||||
});
|
||||
|
||||
it('setConnectorCredential — clears refresh token when re-set without one', () => {
|
||||
const dir = makeTempDir();
|
||||
const vault = new VaultStore(dir);
|
||||
|
||||
// First set with refresh token
|
||||
vault.setConnectorCredential('github', {
|
||||
type: 'oauth2',
|
||||
value: 'gho_token_1',
|
||||
refreshToken: 'rt-old-refresh',
|
||||
});
|
||||
expect(vault.getConnectorCredential('github')!.refreshToken).toBe('rt-old-refresh');
|
||||
|
||||
// Re-set without refresh token
|
||||
vault.setConnectorCredential('github', {
|
||||
type: 'oauth2',
|
||||
value: 'gho_token_2',
|
||||
});
|
||||
const cred = vault.getConnectorCredential('github');
|
||||
expect(cred!.value).toBe('gho_token_2');
|
||||
expect(cred!.refreshToken).toBeUndefined();
|
||||
expect(vault.has('connector:github:refresh')).toBe(false);
|
||||
});
|
||||
|
||||
it('corrupted key file — truncated hex throws clear error', () => {
|
||||
const dir = makeTempDir();
|
||||
const keyPath = path.join(dir, '.vault-key');
|
||||
|
||||
// Write a truncated key (only 10 hex chars = 5 bytes instead of 32)
|
||||
fs.writeFileSync(keyPath, 'abcdef0123', { mode: 0o600 });
|
||||
|
||||
expect(() => new VaultStore(dir)).toThrowError(
|
||||
/Vault key file is corrupted — expected 32 bytes, got 5/
|
||||
);
|
||||
});
|
||||
|
||||
it('corrupted key file — non-hex content throws clear error', () => {
|
||||
const dir = makeTempDir();
|
||||
const keyPath = path.join(dir, '.vault-key');
|
||||
|
||||
// Non-hex content: Buffer.from('not-hex-at-all', 'hex') silently produces a short buffer
|
||||
fs.writeFileSync(keyPath, 'not-hex-at-all-garbage-content', { mode: 0o600 });
|
||||
|
||||
expect(() => new VaultStore(dir)).toThrowError(
|
||||
/Vault key file is corrupted — expected 32 bytes/
|
||||
);
|
||||
});
|
||||
|
||||
it('corrupted key file — empty file throws clear error', () => {
|
||||
const dir = makeTempDir();
|
||||
const keyPath = path.join(dir, '.vault-key');
|
||||
|
||||
fs.writeFileSync(keyPath, '', { mode: 0o600 });
|
||||
|
||||
expect(() => new VaultStore(dir)).toThrowError(
|
||||
/Vault key file is corrupted — expected 32 bytes, got 0/
|
||||
);
|
||||
});
|
||||
|
||||
it('Windows key protection — icacls is attempted on win32', () => {
|
||||
const dir = makeTempDir();
|
||||
const keyPath = path.join(dir, '.vault-key');
|
||||
|
||||
// Ensure no key file exists so ensureKey() will generate one
|
||||
if (fs.existsSync(keyPath)) fs.unlinkSync(keyPath);
|
||||
|
||||
// Mock process.platform to 'win32'
|
||||
const originalPlatform = Object.getOwnPropertyDescriptor(process, 'platform')!;
|
||||
Object.defineProperty(process, 'platform', { value: 'win32', configurable: true });
|
||||
|
||||
// Use the hoisted mock: first call = whoami, second call = icacls
|
||||
mockExecFileSync.mockImplementation((cmd: string) => {
|
||||
if (cmd === 'whoami') return 'DOMAIN\\testuser';
|
||||
return undefined;
|
||||
});
|
||||
|
||||
try {
|
||||
new VaultStore(dir);
|
||||
|
||||
// Verify whoami was called first, then icacls with the resolved user
|
||||
expect(mockExecFileSync).toHaveBeenCalledWith('whoami', expect.objectContaining({ encoding: 'utf-8' }));
|
||||
expect(mockExecFileSync).toHaveBeenCalledWith(
|
||||
'icacls',
|
||||
expect.arrayContaining([keyPath, '/inheritance:r', '/grant:r']),
|
||||
expect.objectContaining({ stdio: 'ignore' })
|
||||
);
|
||||
} finally {
|
||||
Object.defineProperty(process, 'platform', originalPlatform);
|
||||
mockExecFileSync.mockReset();
|
||||
}
|
||||
});
|
||||
|
||||
it('Windows key protection — icacls failure does not prevent vault creation', () => {
|
||||
const dir = makeTempDir();
|
||||
const keyPath = path.join(dir, '.vault-key');
|
||||
if (fs.existsSync(keyPath)) fs.unlinkSync(keyPath);
|
||||
|
||||
const originalPlatform = Object.getOwnPropertyDescriptor(process, 'platform')!;
|
||||
Object.defineProperty(process, 'platform', { value: 'win32', configurable: true });
|
||||
|
||||
// Mock: whoami succeeds but icacls throws
|
||||
mockExecFileSync.mockImplementation((cmd: string) => {
|
||||
if (cmd === 'whoami') return 'DOMAIN\\testuser';
|
||||
throw new Error('icacls not found');
|
||||
});
|
||||
|
||||
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
|
||||
try {
|
||||
const vault = new VaultStore(dir);
|
||||
// Vault should still work despite icacls failure
|
||||
vault.set('test', 'value');
|
||||
expect(vault.get('test')!.value).toBe('value');
|
||||
} finally {
|
||||
Object.defineProperty(process, 'platform', originalPlatform);
|
||||
mockExecFileSync.mockReset();
|
||||
warnSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
21
packages/core/tsconfig.json
Normal file
21
packages/core/tsconfig.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"lib": ["ES2022"],
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"composite": true
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["node_modules", "dist", "tests"]
|
||||
}
|
||||
9
packages/core/vitest.config.ts
Normal file
9
packages/core/vitest.config.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: true,
|
||||
testTimeout: 30_000,
|
||||
include: ['tests/**/*.test.ts'],
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user