497 lines
19 KiB
TypeScript
497 lines
19 KiB
TypeScript
import { randomUUID } from 'node:crypto';
|
|
import type { FastifyInstance } from 'fastify';
|
|
import { FrameStore, SessionStore } from '@waggle/core';
|
|
import {
|
|
TraceRecorder,
|
|
detectTaskShape,
|
|
filterAvailableTools,
|
|
isEnabled,
|
|
listPersonas,
|
|
runAgentLoop,
|
|
selectAgentRunBudget,
|
|
type AgentResponse,
|
|
} from '@waggle/agent';
|
|
import type {
|
|
CollaborationRunMemoryRefs,
|
|
CollaborationWorkerRun,
|
|
GoalAncestry,
|
|
WaggleMessage,
|
|
} from '@waggle/shared';
|
|
import { applyPersonaToolFilter, selectToolsForTurn } from './persona-tool-filter.js';
|
|
import { resolveWorkspaceExecutionRoot } from './workspace-execution-root.js';
|
|
import { persistMessage } from './routes/chat-persistence.js';
|
|
import { emitWaggleSignal } from './routes/waggle-signals.js';
|
|
import type { AgentRunner } from './routes/chat.js';
|
|
import {
|
|
listOllamaChatModelIds,
|
|
OllamaModelNotLocalError,
|
|
resolveUsableModel,
|
|
} from './model-availability.js';
|
|
import type { WorkspaceTurnScope } from './workspace-turn-coordinator.js';
|
|
import { isOfflineOllamaModelReference } from './routes/chat-helpers.js';
|
|
import {
|
|
bindModelSpendBudget,
|
|
createModelSpendMeter,
|
|
type ModelSpendMeter,
|
|
} from './model-spend-meter.js';
|
|
|
|
const ACTIVE = new Set(['queued', 'starting', 'running', 'waiting_for_approval', 'paused', 'cancelling']);
|
|
|
|
async function resolveExplicitFleetModel(
|
|
server: FastifyInstance,
|
|
selectedModel: string,
|
|
): Promise<string | null> {
|
|
const model = selectedModel.trim();
|
|
const separator = model.indexOf('/');
|
|
if (separator <= 0) return null;
|
|
const provider = model.slice(0, separator).toLowerCase();
|
|
if (provider === 'ollama') {
|
|
return (await listOllamaChatModelIds()).includes(model) ? model : null;
|
|
}
|
|
if (!server.vault?.get(provider)) return null;
|
|
|
|
const baseUrl = server.localConfig.litellmUrl.replace(/\/+$/, '');
|
|
if (!baseUrl) return null;
|
|
const response = await fetch(`${baseUrl}/models`, {
|
|
headers: { Authorization: `Bearer ${server.agentState.litellmApiKey}` },
|
|
});
|
|
if (!response.ok) return null;
|
|
const payload = await response.json() as { data?: Array<{ id?: string }> };
|
|
return payload.data?.some((entry) => entry.id === model) ? model : null;
|
|
}
|
|
|
|
export interface FleetSpawnInput {
|
|
task: string;
|
|
persona?: string;
|
|
model?: string;
|
|
parentWorkspaceId?: string;
|
|
goal?: string;
|
|
agentId?: string;
|
|
}
|
|
|
|
export interface FleetSpawnResult {
|
|
statusCode: number;
|
|
body: Record<string, unknown>;
|
|
}
|
|
|
|
type FleetResultRecorder = (input: {
|
|
run: CollaborationWorkerRun;
|
|
prompt: string;
|
|
result: AgentResponse;
|
|
workspaceMind: Parameters<FastifyInstance['agentState']['createSessionOrchestrator']>[0];
|
|
}) => Promise<CollaborationRunMemoryRefs>;
|
|
|
|
declare module 'fastify' {
|
|
interface FastifyInstance {
|
|
fleetResultRecorder?: FleetResultRecorder;
|
|
}
|
|
}
|
|
|
|
export function buildFleetAncestry(
|
|
workspaceName: string | undefined,
|
|
goal: string | undefined,
|
|
): GoalAncestry {
|
|
return {
|
|
...(workspaceName ? { project: workspaceName } : {}),
|
|
...(goal ? { goal } : {}),
|
|
};
|
|
}
|
|
|
|
/** Queue one genuinely isolated internal run and return immediately. */
|
|
export async function spawnIsolatedFleetRun(
|
|
server: FastifyInstance,
|
|
input: FleetSpawnInput,
|
|
): Promise<FleetSpawnResult> {
|
|
const task = input.task?.trim();
|
|
if (!task) return { statusCode: 400, body: { error: 'task is required' } };
|
|
const workspaceId = input.parentWorkspaceId
|
|
|| server.workspaceManager.getDefault()
|
|
|| server.workspaceManager.list()[0]?.id;
|
|
if (!workspaceId) return { statusCode: 404, body: { error: 'workspace_not_found' } };
|
|
const workspace = server.workspaceManager.get(workspaceId);
|
|
if (!workspace) return { statusCode: 404, body: { error: 'workspace_not_found', message: `Workspace ${workspaceId} does not exist` } };
|
|
|
|
let cwd: string;
|
|
try { cwd = resolveWorkspaceExecutionRoot(server.localConfig.dataDir, workspace); }
|
|
catch (err) {
|
|
return { statusCode: 409, body: { error: 'workspace_root_invalid', message: err instanceof Error ? err.message : String(err) } };
|
|
}
|
|
|
|
const activeRuns = server.agentRunRegistry.list({ source: 'fleet', limit: 1_000 })
|
|
.filter((run) => run.kind === 'worker' && ACTIVE.has(run.status)).length;
|
|
const maxRuns = server.sessionManager?.getMaxSessions?.() ?? 10;
|
|
const liveWorkspaceSessions = server.sessionManager?.size ?? 0;
|
|
if (activeRuns + liveWorkspaceSessions >= maxRuns) {
|
|
return { statusCode: 409, body: { error: 'fleet_capacity_reached', message: `Max concurrent sessions reached (${maxRuns})` } };
|
|
}
|
|
|
|
const sentinel = (model?: string | null) => !model || model.trim() === 'auto' || model.trim() === 'default';
|
|
const explicitModel = !sentinel(input.model) ? input.model!.trim() : undefined;
|
|
const workspaceModel = workspace.model;
|
|
const implicitWorkspaceModel = !sentinel(workspaceModel) ? workspaceModel : undefined;
|
|
const selectedModel = explicitModel
|
|
?? implicitWorkspaceModel
|
|
?? server.agentState.currentModel;
|
|
if (!selectedModel || sentinel(selectedModel)) {
|
|
return { statusCode: 503, body: { error: 'model_unavailable', message: 'No executable model is configured' } };
|
|
}
|
|
let model: string;
|
|
if (explicitModel) {
|
|
try {
|
|
const routable = await resolveExplicitFleetModel(server, explicitModel);
|
|
if (!routable) {
|
|
return {
|
|
statusCode: 409,
|
|
body: {
|
|
error: 'model_unavailable',
|
|
message: `Selected model "${explicitModel}" is not currently routable. Configure its provider or choose an available model.`,
|
|
},
|
|
};
|
|
}
|
|
model = routable;
|
|
} catch (err) {
|
|
return {
|
|
statusCode: 503,
|
|
body: {
|
|
error: 'model_validation_failed',
|
|
message: `Could not validate selected model "${explicitModel}": ${err instanceof Error ? err.message : String(err)}`,
|
|
},
|
|
};
|
|
}
|
|
} else {
|
|
try {
|
|
model = await resolveUsableModel(server, selectedModel);
|
|
} catch (err) {
|
|
const currentModel = server.agentState.currentModel?.trim();
|
|
const canRetryCurrentLocal = err instanceof OllamaModelNotLocalError
|
|
&& selectedModel === implicitWorkspaceModel
|
|
&& currentModel
|
|
&& currentModel !== selectedModel
|
|
&& isOfflineOllamaModelReference(currentModel);
|
|
if (!canRetryCurrentLocal) throw err;
|
|
model = await resolveUsableModel(server, currentModel);
|
|
}
|
|
}
|
|
const persona = input.persona ?? workspace.personaId ?? 'general-purpose';
|
|
const room = server.agentRunRegistry.createRoom({
|
|
workspaceIds: [workspaceId],
|
|
source: 'fleet',
|
|
executor: { kind: 'coordinator', agentId: input.agentId },
|
|
title: input.agentId ? `Agent ${input.agentId}` : 'Spawned agent',
|
|
task,
|
|
capabilities: { cancel: true },
|
|
});
|
|
const run = server.agentRunRegistry.createWorker({
|
|
parentRunId: room.id,
|
|
workspaceId,
|
|
source: 'fleet',
|
|
executor: { kind: 'waggle_agent', agentId: input.agentId, personaId: persona, model },
|
|
title: `${listPersonas().find((item) => item.id === persona)?.name ?? persona} ยท ${workspace.name}`,
|
|
task,
|
|
capabilities: { cancel: true },
|
|
});
|
|
const sessionId = `spawn-${run.id}`;
|
|
const assignmentId = publishFleetDance(server, run, 'request', 'task_delegation', {
|
|
task, phase: 'queued', model, persona,
|
|
})?.id;
|
|
void executeFleetRun(server, run, sessionId, cwd, model, persona, task, input.goal, assignmentId);
|
|
|
|
return {
|
|
statusCode: 202,
|
|
body: {
|
|
id: run.id,
|
|
runId: run.id,
|
|
roomId: room.id,
|
|
workspaceId,
|
|
sessionId,
|
|
status: run.status,
|
|
statusUrl: `/api/agent-runs/${run.id}`,
|
|
resumable: false,
|
|
task,
|
|
persona,
|
|
model,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function executeFleetRun(
|
|
server: FastifyInstance,
|
|
run: CollaborationWorkerRun,
|
|
sessionId: string,
|
|
cwd: string,
|
|
model: string,
|
|
personaId: string,
|
|
task: string,
|
|
goal: string | undefined,
|
|
assignmentId: string | undefined,
|
|
): Promise<void> {
|
|
const controller = new AbortController();
|
|
let settleExecution!: () => void;
|
|
const executionSettled = new Promise<void>((resolve) => { settleExecution = resolve; });
|
|
const unregister = server.agentRunRegistry.registerControls(run.id, {
|
|
cancel: async () => {
|
|
controller.abort();
|
|
await executionSettled;
|
|
},
|
|
});
|
|
let acquired = false;
|
|
let workspaceTurnScope: WorkspaceTurnScope | undefined;
|
|
let traceId: number | undefined;
|
|
let fleetSpendMeter: ModelSpendMeter | undefined;
|
|
try {
|
|
const mind = server.mindCache.acquire(run.workspaceId);
|
|
acquired = true;
|
|
const orchestrator = server.agentState.createSessionOrchestrator(mind);
|
|
const persona = listPersonas().find((item) => item.id === personaId) ?? null;
|
|
fleetSpendMeter = server.agentState.costTracker
|
|
? createModelSpendMeter(server.agentState.costTracker, (costUsd) => {
|
|
if (traceId === undefined) return;
|
|
server.traceStore?.recordCost(traceId, costUsd);
|
|
})
|
|
: undefined;
|
|
const underlyingRunner: AgentRunner = server.agentRunner ?? runAgentLoop;
|
|
const runner = fleetSpendMeter
|
|
? bindModelSpendBudget(
|
|
underlyingRunner,
|
|
fleetSpendMeter,
|
|
run.workspaceId,
|
|
listOllamaChatModelIds,
|
|
() => traceId,
|
|
)
|
|
: underlyingRunner;
|
|
let workerTools = server.agentState.buildToolsForSession(orchestrator, cwd, run.workspaceId);
|
|
if (persona) workerTools = applyPersonaToolFilter(workerTools, persona);
|
|
workerTools = filterAvailableTools(workerTools);
|
|
const workspaceTurnCoordinator = server.agentState.workspaceTurnCoordinator;
|
|
if (workspaceTurnCoordinator) {
|
|
workspaceTurnScope = workspaceTurnCoordinator.createScope(cwd, controller.signal);
|
|
workerTools = workspaceTurnScope.wrapTools(workerTools);
|
|
}
|
|
let tools = selectToolsForTurn(workerTools, {
|
|
message: task,
|
|
preferredToolNames: persona?.tools ?? [],
|
|
}).tools;
|
|
if (workspaceTurnScope) {
|
|
const workspaceAccess = workspaceTurnScope.classify(tools);
|
|
if (workspaceAccess !== 'none') await workspaceTurnScope.acquire(workspaceAccess);
|
|
const activeScope = workspaceTurnScope;
|
|
tools = server.agentState.bindWorkspaceCollaborationTools({
|
|
visibleTools: tools,
|
|
workerTools,
|
|
runLoop: runner,
|
|
signal: controller.signal,
|
|
runChildTransaction: (childTools, operation) => (
|
|
activeScope.runChildTransaction(childTools, operation)
|
|
),
|
|
defaultModel: model,
|
|
});
|
|
}
|
|
const taskShape = detectTaskShape(task);
|
|
const runBudget = selectAgentRunBudget({
|
|
taskShape: taskShape.type,
|
|
complexity: taskShape.complexity,
|
|
selectedToolNames: tools.map(tool => tool.name),
|
|
});
|
|
orchestrator.setGoalAncestry(buildFleetAncestry(server.workspaceManager.get(run.workspaceId)?.name, goal));
|
|
let systemPrompt: string;
|
|
if (isEnabled('PROMPT_ASSEMBLER')) {
|
|
const assembled = await orchestrator.buildAssembledPrompt(task, persona, { taskShape });
|
|
systemPrompt = assembled.system + (assembled.responseScaffold ? `\n\n## Response shape\n${assembled.responseScaffold}` : '');
|
|
} else {
|
|
systemPrompt = orchestrator.buildSystemPrompt();
|
|
if (persona?.systemPrompt) systemPrompt += `\n\n## Active persona\n${persona.systemPrompt}`;
|
|
}
|
|
|
|
persistMessage(server.localConfig.dataDir, run.workspaceId, sessionId, { role: 'user', content: task });
|
|
traceId = server.traceStore?.start({
|
|
sessionId,
|
|
personaId,
|
|
workspaceId: run.workspaceId,
|
|
model,
|
|
input: task,
|
|
tags: [`room:${run.roomId}`, `run:${run.id}`, ...(run.executor.agentId ? [`agent:${run.executor.agentId}`] : [])],
|
|
});
|
|
const traceRecorder = traceId !== undefined && server.traceStore ? new TraceRecorder(server.traceStore) : undefined;
|
|
server.agentRunRegistry.update(run.id, {
|
|
status: 'running',
|
|
result: { sessionId, ...(traceId !== undefined ? { traceId: String(traceId) } : {}) },
|
|
progress: { message: 'Agent started', phase: 'running' },
|
|
});
|
|
emitWaggleSignal({
|
|
type: 'agent:started', workspaceId: run.workspaceId, content: task.slice(0, 200),
|
|
metadata: { runId: run.id, roomId: run.roomId, sessionId, model, persona: personaId },
|
|
});
|
|
publishFleetDance(server, run, 'response', 'task_claim', { phase: 'running', task: 'claimed' }, assignmentId);
|
|
|
|
const result = await runner({
|
|
litellmUrl: server.localConfig.litellmUrl,
|
|
litellmApiKey: server.agentState.litellmApiKey,
|
|
model,
|
|
systemPrompt,
|
|
tools,
|
|
messages: [{ role: 'user', content: task }],
|
|
...runBudget,
|
|
signal: controller.signal,
|
|
...(traceRecorder && traceId !== undefined ? {
|
|
traceRecording: { recorder: traceRecorder, handle: { id: traceId, startedAt: Date.now() } },
|
|
} : {}),
|
|
onToolUse: (name, input) => {
|
|
const current = server.agentRunRegistry.get(run.id);
|
|
const toolsUsed = [...new Set([...(current?.metrics?.toolsUsed ?? []), name])];
|
|
server.agentRunRegistry.update(run.id, {
|
|
progress: { message: name, phase: 'tool' }, metrics: { toolsUsed },
|
|
});
|
|
emitWaggleSignal({
|
|
type: 'tool:called', workspaceId: run.workspaceId,
|
|
content: `${name}(${JSON.stringify(input).slice(0, 100)})`,
|
|
metadata: { runId: run.id, roomId: run.roomId, sessionId },
|
|
});
|
|
publishFleetDance(server, run, 'broadcast', 'discovery', { phase: 'tool', tool: name }, assignmentId);
|
|
},
|
|
});
|
|
persistMessage(server.localConfig.dataDir, run.workspaceId, sessionId, { role: 'assistant', content: result.content });
|
|
const memoryRefs = server.fleetResultRecorder
|
|
? await server.fleetResultRecorder({ run, prompt: task, result, workspaceMind: mind })
|
|
: await recordFleetResult(server, run, task, result, mind);
|
|
const totalTokens = result.usage.inputTokens + result.usage.outputTokens;
|
|
server.agentRunRegistry.update(run.id, {
|
|
status: controller.signal.aborted ? 'cancelling' : 'completed',
|
|
result: { summary: result.content, sessionId },
|
|
metrics: {
|
|
toolsUsed: result.toolsUsed,
|
|
inputTokens: result.usage.inputTokens,
|
|
outputTokens: result.usage.outputTokens,
|
|
},
|
|
memoryRefs,
|
|
progress: null,
|
|
});
|
|
server.agentRunRegistry.update(run.roomId, {
|
|
result: { summary: result.content, sessionId },
|
|
memoryRefs,
|
|
});
|
|
if (traceId !== undefined) {
|
|
server.traceStore?.finalize(traceId, {
|
|
outcome: controller.signal.aborted ? 'abandoned' : 'success',
|
|
output: result.content,
|
|
tokens: { input: result.usage.inputTokens, output: result.usage.outputTokens },
|
|
costUsd: fleetSpendMeter?.totalCostUsd(),
|
|
});
|
|
}
|
|
emitWaggleSignal({
|
|
type: controller.signal.aborted ? 'agent:cancelled' : 'agent:completed',
|
|
workspaceId: run.workspaceId,
|
|
content: controller.signal.aborted ? 'Cancelled' : `Completed ยท ${totalTokens.toLocaleString()} tokens`,
|
|
metadata: { runId: run.id, roomId: run.roomId, sessionId, toolsUsed: result.toolsUsed },
|
|
});
|
|
publishFleetDance(server, run, 'broadcast', 'routed_share', {
|
|
phase: controller.signal.aborted ? 'cancelled' : 'completed', result: result.content,
|
|
}, assignmentId);
|
|
} catch (err) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
const current = server.agentRunRegistry.get(run.id);
|
|
if (current && !['completed', 'failed', 'cancelled', 'interrupted'].includes(current.status)) {
|
|
server.agentRunRegistry.update(run.id, {
|
|
status: controller.signal.aborted ? 'cancelling' : 'failed',
|
|
result: { summary: message, error: message, sessionId },
|
|
progress: null,
|
|
});
|
|
}
|
|
try {
|
|
persistMessage(server.localConfig.dataDir, run.workspaceId, sessionId, {
|
|
role: 'assistant', content: controller.signal.aborted ? 'This run was cancelled.' : `I couldn't finish this run. ${message}`,
|
|
});
|
|
} catch { /* best effort */ }
|
|
if (traceId !== undefined) server.traceStore?.finalize(traceId, {
|
|
outcome: 'abandoned',
|
|
output: message,
|
|
costUsd: fleetSpendMeter?.totalCostUsd(),
|
|
});
|
|
emitWaggleSignal({
|
|
type: controller.signal.aborted ? 'agent:cancelled' : 'agent:error',
|
|
workspaceId: run.workspaceId, content: message.slice(0, 200),
|
|
metadata: { runId: run.id, roomId: run.roomId, sessionId },
|
|
});
|
|
publishFleetDance(server, run, 'broadcast', 'routed_share', {
|
|
phase: controller.signal.aborted ? 'cancelled' : 'failed', error: message,
|
|
}, assignmentId);
|
|
} finally {
|
|
try {
|
|
if (workspaceTurnScope) await workspaceTurnScope.release();
|
|
} finally {
|
|
try {
|
|
if (acquired) server.mindCache.release(run.workspaceId);
|
|
} finally {
|
|
settleExecution();
|
|
unregister();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
async function recordFleetResult(
|
|
server: FastifyInstance,
|
|
run: CollaborationWorkerRun,
|
|
prompt: string,
|
|
result: AgentResponse,
|
|
workspaceMind: Parameters<FastifyInstance['agentState']['createSessionOrchestrator']>[0],
|
|
): Promise<CollaborationRunMemoryRefs> {
|
|
const personalFrameIds: number[] = [];
|
|
const workspaceFrameIds: Record<string, number[]> = {};
|
|
const meta = JSON.stringify({ runId: run.id, roomId: run.roomId, workspaceId: run.workspaceId, source: 'fleet' });
|
|
try {
|
|
const personal = server.multiMind.personal;
|
|
new SessionStore(personal).ensure('agent-runs', 'agent-runs', 'Agent collaboration index');
|
|
const frames = new FrameStore(personal);
|
|
const frame = frames.createIFrame(
|
|
'agent-runs',
|
|
`[Agent run]\nRun: ${run.id}\nWorkspace: ${run.workspaceId}\nPersona: ${run.executor.personaId}\nSummary: ${result.content.slice(0, 1_000)}`,
|
|
'normal', 'agent_inferred',
|
|
);
|
|
frames.setMetadata(frame.id, meta);
|
|
personalFrameIds.push(frame.id);
|
|
} catch { /* workspace result remains authoritative */ }
|
|
try {
|
|
new SessionStore(workspaceMind).ensure('agent-runs', 'agent-runs', 'Agent collaboration results');
|
|
const frames = new FrameStore(workspaceMind);
|
|
const frame = frames.createIFrame(
|
|
'agent-runs',
|
|
`[Agent run result]\nRun: ${run.id}\nTask:\n${prompt}\n\nResult:\n${result.content.slice(0, 100_000)}`,
|
|
'normal', 'agent_inferred',
|
|
);
|
|
frames.setMetadata(frame.id, meta);
|
|
workspaceFrameIds[run.workspaceId] = [frame.id];
|
|
} catch { /* reflected in status below */ }
|
|
const personalOk = personalFrameIds.length > 0;
|
|
const workspaceOk = (workspaceFrameIds[run.workspaceId]?.length ?? 0) > 0;
|
|
return {
|
|
status: personalOk && workspaceOk ? 'complete' : (personalOk || workspaceOk ? 'partial' : 'failed'),
|
|
personalFrameIds,
|
|
workspaceFrameIds,
|
|
};
|
|
}
|
|
|
|
function publishFleetDance(
|
|
server: FastifyInstance,
|
|
run: CollaborationWorkerRun,
|
|
type: WaggleMessage['type'],
|
|
subtype: WaggleMessage['subtype'],
|
|
content: Record<string, unknown>,
|
|
referenceId?: string,
|
|
): WaggleMessage | undefined {
|
|
if (!server.signalBus) return undefined;
|
|
return server.signalBus.record({
|
|
id: randomUUID(),
|
|
teamId: `room::${run.roomId}`,
|
|
senderId: type === 'request' ? 'user' : `run::${run.id}`,
|
|
type,
|
|
subtype,
|
|
content: {
|
|
kind: 'waggle_agent_run', roomId: run.roomId, runId: run.id,
|
|
workspaceId: run.workspaceId, persona: run.executor.personaId, ...content,
|
|
},
|
|
referenceId: referenceId ?? null,
|
|
routing: null,
|
|
createdAt: new Date(),
|
|
});
|
|
}
|