Files
waggle-os/packages/agent/src/context-compressor.ts
Oleg Maslov b20b138fe4 moving
2026-09-02 10:14:22 +02:00

471 lines
17 KiB
TypeScript

/**
* ContextCompressor — 5-step pipeline for intelligent conversation compression.
*
* When a conversation exceeds a configurable fraction of the context window,
* this pipeline compresses it while preserving critical information:
*
* 1. Detect — estimate token count, check against threshold
* 2. Prune — replace old tool-result messages with "[Cleared]" (no LLM, free)
* 3. Protect — split into head (system + first N msgs), tail (recent work), middle
* 4. Summarize — call budget model on the middle using COMPACTION_PROMPT ($0 cost)
* 5. Inject — replace middle with summary, return compressed message array
*
* Iterative: when compressing again, the previous summary is fed to the summarizer
* so information accumulates rather than being lost.
*/
import { COMPACTION_PROMPT } from './behavioral-spec.js';
import { createCoreLogger, evaluateExternalMemoryIngress } from '@waggle/core';
const log = createCoreLogger('context-compressor');
// ── Types ────────────────────────────────────────────────────────────────
export interface CompressionConfig {
/** Total context window size in tokens (e.g. 128000 for Claude Sonnet) */
maxContextTokens: number;
/** Fraction of context window that triggers compression (default: 0.5) */
compressionThreshold: number;
/** Number of messages to protect at the start after system prompt (default: 3) */
protectedHeadMessages: number;
/** Approximate token budget to protect at the tail (default: 20000) */
protectedTailTokens: number;
/** Budget model identifier for the summarizer (e.g. "qwen/qwen3.6-plus:free") */
budgetModel: string;
/** LiteLLM proxy base URL */
litellmUrl: string;
/** LiteLLM API key */
litellmApiKey: string;
/** Custom fetch function (for testing/injection) */
fetch?: typeof globalThis.fetch;
}
export interface CompressionResult {
/** The (possibly compressed) messages to send to the agent loop */
messages: CompressibleMessage[];
/** Whether compression was actually performed */
compressed: boolean;
/** Estimated token count before compression */
originalTokens: number;
/** Estimated token count after compression */
compressedTokens: number;
/** Whether an LLM summary was generated this pass */
summaryGenerated: boolean;
/** The generated summary text (for iterative use on next compression) */
summary: string | null;
}
export interface CompressibleMessage {
role: string;
content: string;
}
function previousSummarySystemMessage(previousSummary: string): string {
return `You are summarizing a conversation that has been compressed before. Here is the previous summary:\n\n${previousSummary}\n\nNow incorporate the new messages below into an updated summary.`;
}
function safePreviousSummary(previousSummary?: string | null): string | null {
if (previousSummary === undefined || previousSummary === null) return null;
return evaluateExternalMemoryIngress({ content: previousSummarySystemMessage(previousSummary) }).action === 'allow'
? previousSummary
: null;
}
// ── Step 1: Token Estimation ─────────────────────────────────────────────
/**
* 9e: Model-aware token estimation.
*
* Chars-per-token ratios vary by content type and model family:
* - English prose: ~4.0 chars/token
* - Code: ~3.2 chars/token (shorter identifiers, symbols)
* - Non-English/mixed: ~2.5 chars/token (Unicode, CJK)
* - JSON/structured: ~3.5 chars/token
*
* This estimator detects content type and applies the appropriate ratio,
* improving accuracy from ~30-50% error down to ~10-15%.
*/
/** Detect the dominant content type of a string. */
function detectContentType(text: string): 'code' | 'json' | 'prose' | 'mixed' {
if (!text || text.length < 20) return 'prose';
// Sample first 2000 chars for detection
const sample = text.slice(0, 2000);
// JSON detection
const trimmed = sample.trimStart();
if (trimmed.startsWith('{') || trimmed.startsWith('[')) return 'json';
// Code detection: high density of code-specific characters
const codeChars = (sample.match(/[{}();=<>[\]|&!+\-*/\\]/g) || []).length;
const codeRatio = codeChars / sample.length;
if (codeRatio > 0.06) return 'code';
// Non-ASCII ratio for multilingual detection. The \x00-\x7F range boundary
// is intentional — we count every code point OUTSIDE the 7-bit ASCII block,
// so the control-char lower bound is the correct, deliberate range start.
// eslint-disable-next-line no-control-regex
const nonAscii = (sample.match(/[^\x00-\x7F]/g) || []).length;
if (nonAscii / sample.length > 0.15) return 'mixed';
return 'prose';
}
const CHARS_PER_TOKEN: Record<string, number> = {
prose: 4.0,
code: 3.2,
json: 3.5,
mixed: 2.5,
};
/**
* Estimate token count for a single string.
*/
export function estimateStringTokens(text: string): number {
if (!text) return 0;
const contentType = detectContentType(text);
const ratio = CHARS_PER_TOKEN[contentType];
return Math.ceil(text.length / ratio);
}
/**
* Estimate token count for a message array.
* Uses content-aware char/token ratios for better accuracy than the
* flat 4-chars heuristic.
*/
export function estimateTokens(messages: ReadonlyArray<CompressibleMessage>): number {
let tokens = 0;
for (const msg of messages) {
// Role overhead: ~4 tokens per message for role/formatting
tokens += 4;
tokens += estimateStringTokens(msg.content ?? '');
}
return tokens;
}
/**
* Check whether the conversation needs compression.
*/
export function needsCompression(
messages: ReadonlyArray<CompressibleMessage>,
config: Pick<CompressionConfig, 'maxContextTokens' | 'compressionThreshold'>,
): boolean {
const tokens = estimateTokens(messages);
return tokens > config.maxContextTokens * config.compressionThreshold;
}
// ── Step 2: Prune Tool Results ───────────────────────────────────────────
/**
* Replace old tool-result message content with a short placeholder.
* This is free (no LLM call) and removes the bulkiest content.
*
* Only prunes messages NOT in the protected tail region.
* Tool results in the tail are left intact since they're recent/relevant.
*/
export function pruneToolResults(
messages: ReadonlyArray<CompressibleMessage>,
protectedTailCount: number,
): CompressibleMessage[] {
const tailStart = Math.max(0, messages.length - protectedTailCount);
return messages.map((msg, i) => {
// Don't touch protected tail messages
if (i >= tailStart) return { ...msg };
// Prune tool-role messages (these are tool call results — often huge)
if (msg.role === 'tool') {
return { role: msg.role, content: '[Cleared: tool result]' };
}
// Prune assistant messages that contain large code blocks or tool output
if (msg.role === 'assistant' && msg.content && msg.content.length > 2000) {
// Check for tool-output patterns (JSON results, file contents, etc.)
const content = msg.content;
if (content.startsWith('{') || content.startsWith('[') || content.includes('```')) {
// Keep first 200 chars as context, clear the rest
const preview = content.slice(0, 200);
return { role: msg.role, content: `${preview}\n\n[Cleared: ${content.length} chars of detailed output]` };
}
}
return { ...msg };
});
}
// ── Step 3: Split Protected Regions ──────────────────────────────────────
export interface ProtectedRegions {
/** System prompt + first N user/assistant messages */
head: CompressibleMessage[];
/** Messages in the middle that can be summarized */
middle: CompressibleMessage[];
/** Recent messages (last ~protectedTailTokens worth) */
tail: CompressibleMessage[];
}
/**
* Split messages into head (protected), middle (compressible), tail (protected).
*
* Head: first message (system) + protectedHeadMessages additional messages.
* Tail: messages from the end that fit within protectedTailTokens.
* Middle: everything between head and tail.
*/
export function splitProtectedRegions(
messages: ReadonlyArray<CompressibleMessage>,
config: Pick<CompressionConfig, 'protectedHeadMessages' | 'protectedTailTokens'>,
): ProtectedRegions {
// Head: system prompt + first N messages
const headEnd = Math.min(1 + config.protectedHeadMessages, messages.length);
const head = messages.slice(0, headEnd);
// Tail: walk backwards from the end until we hit the token budget
let tailTokens = 0;
let tailStart = messages.length;
for (let i = messages.length - 1; i >= headEnd; i--) {
const msgTokens = estimateTokens([messages[i]]);
if (tailTokens + msgTokens > config.protectedTailTokens) break;
tailTokens += msgTokens;
tailStart = i;
}
const tail = messages.slice(tailStart);
const middle = messages.slice(headEnd, tailStart);
return { head, middle, tail };
}
// ── Step 4: Summarize Middle ─────────────────────────────────────────────
/**
* Call the budget model to summarize the compressible middle section.
* Uses COMPACTION_PROMPT from behavioral-spec.ts.
*
* If a previousSummary is provided, it's included so the model can build
* on accumulated context rather than losing older information.
*/
export async function summarizeMiddle(
middle: ReadonlyArray<CompressibleMessage>,
config: Pick<CompressionConfig, 'budgetModel' | 'litellmUrl' | 'litellmApiKey' | 'fetch'>,
previousSummary?: string | null,
): Promise<string> {
previousSummary = safePreviousSummary(previousSummary);
if (middle.length === 0) return previousSummary ?? '';
const fetchFn = config.fetch ?? globalThis.fetch;
// Build the summarization prompt
const summarizerMessages: Array<{ role: string; content: string }> = [];
// If we have a previous summary, include it as context
if (previousSummary) {
summarizerMessages.push({
role: 'system',
content: previousSummarySystemMessage(previousSummary),
});
}
// Add the middle messages as the conversation to summarize
for (const msg of middle) {
summarizerMessages.push({ role: msg.role === 'system' ? 'user' : msg.role, content: msg.content ?? '' });
}
// Add the compaction instruction as the final user message
summarizerMessages.push({ role: 'user', content: COMPACTION_PROMPT });
const body = {
model: config.budgetModel,
messages: summarizerMessages,
max_tokens: 2000,
temperature: 0.1,
};
let response: Response;
try {
response = await fetchFn(`${config.litellmUrl}/v1/chat/completions`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${config.litellmApiKey}`,
},
body: JSON.stringify(body),
signal: AbortSignal.timeout(60_000),
});
} catch (error) {
log.warn(`Summarizer request failed: ${error instanceof Error ? error.message : String(error)}`);
return buildFallbackSummary(middle, previousSummary);
}
if (!response.ok) {
try {
const errBody = await response.text();
log.warn(`Summarizer returned ${response.status}: ${errBody.slice(0, 200)}`);
} catch { /* ignore read errors */ }
return buildFallbackSummary(middle, previousSummary);
}
let result: {
choices?: Array<{
finish_reason?: string | null;
message?: {
content?: string | null;
tool_calls?: unknown[];
};
}>;
};
try {
const parsed = await response.json() as unknown;
if (typeof parsed !== 'object' || parsed === null) {
log.warn('Summarizer response was not a JSON object; using deterministic fallback');
return buildFallbackSummary(middle, previousSummary);
}
result = parsed as typeof result;
} catch (error) {
log.warn(`Summarizer response was not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
return buildFallbackSummary(middle, previousSummary);
}
const choice = result.choices?.[0];
if (choice?.finish_reason !== 'stop') {
log.warn(
`Summarizer returned an incomplete completion (finish_reason=${choice?.finish_reason ?? 'missing'}); using deterministic fallback`,
);
return buildFallbackSummary(middle, previousSummary);
}
if (choice.message?.tool_calls?.length) {
log.warn('Summarizer returned tool calls with finish_reason=stop; using deterministic fallback');
return buildFallbackSummary(middle, previousSummary);
}
const content = choice.message?.content;
if (typeof content !== 'string' || content.trim().length === 0) {
return buildFallbackSummary(middle, previousSummary);
}
return content;
}
/**
* Fallback summary when the LLM call fails.
* Extracts key signals from messages without any LLM.
*/
function buildFallbackSummary(
middle: ReadonlyArray<CompressibleMessage>,
previousSummary?: string | null,
): string {
const userMessages = middle.filter(m => m.role === 'user');
const firstLines = userMessages
.map(m => (m.content ?? '').split('\n')[0]?.trim())
.filter(line => line && line.length > 10 && line.length < 200)
.slice(0, 5);
const parts: string[] = [];
if (previousSummary) {
parts.push('## Previous Context\n' + previousSummary);
}
parts.push(`## Compressed Region (${middle.length} messages)`);
if (firstLines.length > 0) {
parts.push('Topics: ' + firstLines.join(' → '));
}
return parts.join('\n\n');
}
// ── Step 5: Compress Conversation (Orchestrator) ─────────────────────────
/**
* Run the full 5-step compression pipeline.
*
* @param messages Full conversation history
* @param config Compression configuration
* @param previousSummary Summary from a previous compression pass (for iterative use)
* @returns CompressionResult with the compressed messages and metadata
*/
export async function compressConversation(
messages: ReadonlyArray<CompressibleMessage>,
config: CompressionConfig,
previousSummary?: string | null,
): Promise<CompressionResult> {
const originalTokens = estimateTokens(messages);
const safeSummary = safePreviousSummary(previousSummary);
// Step 1: Detect — do we need compression?
if (!needsCompression(messages, config)) {
return {
messages: messages.map(m => ({ ...m })),
compressed: false,
originalTokens,
compressedTokens: originalTokens,
summaryGenerated: false,
summary: safeSummary,
};
}
// Step 2: Prune tool results in the non-tail region
// Estimate how many messages fit in the tail based on token budget
const avgTokensPerMsg = originalTokens / messages.length;
const estimatedTailCount = Math.max(5, Math.ceil(config.protectedTailTokens / avgTokensPerMsg));
const pruned = pruneToolResults(messages, estimatedTailCount);
// Step 3: Split into protected head, compressible middle, protected tail
const regions = splitProtectedRegions(pruned, config);
// If middle is empty or very small, no point summarizing
if (regions.middle.length <= 2) {
const result = [...regions.head, ...regions.middle, ...regions.tail];
return {
messages: result,
compressed: false,
originalTokens,
compressedTokens: estimateTokens(result),
summaryGenerated: false,
summary: safeSummary,
};
}
// Step 4: Summarize the middle
const summary = await summarizeMiddle(regions.middle, config, safeSummary);
// Step 5: Inject — replace middle with a single summary message
const summaryMessage: CompressibleMessage = {
role: 'system',
content: `[Conversation compressed — ${regions.middle.length} messages summarized]\n\n${summary}`,
};
if (evaluateExternalMemoryIngress({ content: summaryMessage.content }).action !== 'allow') {
const messagesWithoutSummary = [...regions.head, ...regions.tail];
return {
messages: messagesWithoutSummary,
compressed: true,
originalTokens,
compressedTokens: estimateTokens(messagesWithoutSummary),
summaryGenerated: false,
summary: null,
};
}
const compressed = [...regions.head, summaryMessage, ...regions.tail];
const compressedTokens = estimateTokens(compressed);
return {
messages: compressed,
compressed: true,
originalTokens,
compressedTokens,
summaryGenerated: true,
summary,
};
}
// ── Default Config Factory ───────────────────────────────────────────────
/** Sensible defaults for context compression */
export function createDefaultCompressionConfig(
overrides: Partial<CompressionConfig> & Pick<CompressionConfig, 'budgetModel' | 'litellmUrl' | 'litellmApiKey'>,
): CompressionConfig {
return {
maxContextTokens: 128000,
compressionThreshold: 0.5,
protectedHeadMessages: 3,
protectedTailTokens: 20000,
...overrides,
};
}