471 lines
17 KiB
TypeScript
471 lines
17 KiB
TypeScript
/**
|
|
* ContextCompressor — 5-step pipeline for intelligent conversation compression.
|
|
*
|
|
* When a conversation exceeds a configurable fraction of the context window,
|
|
* this pipeline compresses it while preserving critical information:
|
|
*
|
|
* 1. Detect — estimate token count, check against threshold
|
|
* 2. Prune — replace old tool-result messages with "[Cleared]" (no LLM, free)
|
|
* 3. Protect — split into head (system + first N msgs), tail (recent work), middle
|
|
* 4. Summarize — call budget model on the middle using COMPACTION_PROMPT ($0 cost)
|
|
* 5. Inject — replace middle with summary, return compressed message array
|
|
*
|
|
* Iterative: when compressing again, the previous summary is fed to the summarizer
|
|
* so information accumulates rather than being lost.
|
|
*/
|
|
|
|
import { COMPACTION_PROMPT } from './behavioral-spec.js';
|
|
import { createCoreLogger, evaluateExternalMemoryIngress } from '@waggle/core';
|
|
const log = createCoreLogger('context-compressor');
|
|
|
|
// ── Types ────────────────────────────────────────────────────────────────
|
|
|
|
export interface CompressionConfig {
|
|
/** Total context window size in tokens (e.g. 128000 for Claude Sonnet) */
|
|
maxContextTokens: number;
|
|
/** Fraction of context window that triggers compression (default: 0.5) */
|
|
compressionThreshold: number;
|
|
/** Number of messages to protect at the start after system prompt (default: 3) */
|
|
protectedHeadMessages: number;
|
|
/** Approximate token budget to protect at the tail (default: 20000) */
|
|
protectedTailTokens: number;
|
|
/** Budget model identifier for the summarizer (e.g. "qwen/qwen3.6-plus:free") */
|
|
budgetModel: string;
|
|
/** LiteLLM proxy base URL */
|
|
litellmUrl: string;
|
|
/** LiteLLM API key */
|
|
litellmApiKey: string;
|
|
/** Custom fetch function (for testing/injection) */
|
|
fetch?: typeof globalThis.fetch;
|
|
}
|
|
|
|
export interface CompressionResult {
|
|
/** The (possibly compressed) messages to send to the agent loop */
|
|
messages: CompressibleMessage[];
|
|
/** Whether compression was actually performed */
|
|
compressed: boolean;
|
|
/** Estimated token count before compression */
|
|
originalTokens: number;
|
|
/** Estimated token count after compression */
|
|
compressedTokens: number;
|
|
/** Whether an LLM summary was generated this pass */
|
|
summaryGenerated: boolean;
|
|
/** The generated summary text (for iterative use on next compression) */
|
|
summary: string | null;
|
|
}
|
|
|
|
export interface CompressibleMessage {
|
|
role: string;
|
|
content: string;
|
|
}
|
|
|
|
function previousSummarySystemMessage(previousSummary: string): string {
|
|
return `You are summarizing a conversation that has been compressed before. Here is the previous summary:\n\n${previousSummary}\n\nNow incorporate the new messages below into an updated summary.`;
|
|
}
|
|
|
|
function safePreviousSummary(previousSummary?: string | null): string | null {
|
|
if (previousSummary === undefined || previousSummary === null) return null;
|
|
return evaluateExternalMemoryIngress({ content: previousSummarySystemMessage(previousSummary) }).action === 'allow'
|
|
? previousSummary
|
|
: null;
|
|
}
|
|
|
|
// ── Step 1: Token Estimation ─────────────────────────────────────────────
|
|
|
|
/**
|
|
* 9e: Model-aware token estimation.
|
|
*
|
|
* Chars-per-token ratios vary by content type and model family:
|
|
* - English prose: ~4.0 chars/token
|
|
* - Code: ~3.2 chars/token (shorter identifiers, symbols)
|
|
* - Non-English/mixed: ~2.5 chars/token (Unicode, CJK)
|
|
* - JSON/structured: ~3.5 chars/token
|
|
*
|
|
* This estimator detects content type and applies the appropriate ratio,
|
|
* improving accuracy from ~30-50% error down to ~10-15%.
|
|
*/
|
|
|
|
/** Detect the dominant content type of a string. */
|
|
function detectContentType(text: string): 'code' | 'json' | 'prose' | 'mixed' {
|
|
if (!text || text.length < 20) return 'prose';
|
|
|
|
// Sample first 2000 chars for detection
|
|
const sample = text.slice(0, 2000);
|
|
|
|
// JSON detection
|
|
const trimmed = sample.trimStart();
|
|
if (trimmed.startsWith('{') || trimmed.startsWith('[')) return 'json';
|
|
|
|
// Code detection: high density of code-specific characters
|
|
const codeChars = (sample.match(/[{}();=<>[\]|&!+\-*/\\]/g) || []).length;
|
|
const codeRatio = codeChars / sample.length;
|
|
if (codeRatio > 0.06) return 'code';
|
|
|
|
// Non-ASCII ratio for multilingual detection. The \x00-\x7F range boundary
|
|
// is intentional — we count every code point OUTSIDE the 7-bit ASCII block,
|
|
// so the control-char lower bound is the correct, deliberate range start.
|
|
// eslint-disable-next-line no-control-regex
|
|
const nonAscii = (sample.match(/[^\x00-\x7F]/g) || []).length;
|
|
if (nonAscii / sample.length > 0.15) return 'mixed';
|
|
|
|
return 'prose';
|
|
}
|
|
|
|
const CHARS_PER_TOKEN: Record<string, number> = {
|
|
prose: 4.0,
|
|
code: 3.2,
|
|
json: 3.5,
|
|
mixed: 2.5,
|
|
};
|
|
|
|
/**
|
|
* Estimate token count for a single string.
|
|
*/
|
|
export function estimateStringTokens(text: string): number {
|
|
if (!text) return 0;
|
|
const contentType = detectContentType(text);
|
|
const ratio = CHARS_PER_TOKEN[contentType];
|
|
return Math.ceil(text.length / ratio);
|
|
}
|
|
|
|
/**
|
|
* Estimate token count for a message array.
|
|
* Uses content-aware char/token ratios for better accuracy than the
|
|
* flat 4-chars heuristic.
|
|
*/
|
|
export function estimateTokens(messages: ReadonlyArray<CompressibleMessage>): number {
|
|
let tokens = 0;
|
|
for (const msg of messages) {
|
|
// Role overhead: ~4 tokens per message for role/formatting
|
|
tokens += 4;
|
|
tokens += estimateStringTokens(msg.content ?? '');
|
|
}
|
|
return tokens;
|
|
}
|
|
|
|
/**
|
|
* Check whether the conversation needs compression.
|
|
*/
|
|
export function needsCompression(
|
|
messages: ReadonlyArray<CompressibleMessage>,
|
|
config: Pick<CompressionConfig, 'maxContextTokens' | 'compressionThreshold'>,
|
|
): boolean {
|
|
const tokens = estimateTokens(messages);
|
|
return tokens > config.maxContextTokens * config.compressionThreshold;
|
|
}
|
|
|
|
// ── Step 2: Prune Tool Results ───────────────────────────────────────────
|
|
|
|
/**
|
|
* Replace old tool-result message content with a short placeholder.
|
|
* This is free (no LLM call) and removes the bulkiest content.
|
|
*
|
|
* Only prunes messages NOT in the protected tail region.
|
|
* Tool results in the tail are left intact since they're recent/relevant.
|
|
*/
|
|
export function pruneToolResults(
|
|
messages: ReadonlyArray<CompressibleMessage>,
|
|
protectedTailCount: number,
|
|
): CompressibleMessage[] {
|
|
const tailStart = Math.max(0, messages.length - protectedTailCount);
|
|
|
|
return messages.map((msg, i) => {
|
|
// Don't touch protected tail messages
|
|
if (i >= tailStart) return { ...msg };
|
|
|
|
// Prune tool-role messages (these are tool call results — often huge)
|
|
if (msg.role === 'tool') {
|
|
return { role: msg.role, content: '[Cleared: tool result]' };
|
|
}
|
|
|
|
// Prune assistant messages that contain large code blocks or tool output
|
|
if (msg.role === 'assistant' && msg.content && msg.content.length > 2000) {
|
|
// Check for tool-output patterns (JSON results, file contents, etc.)
|
|
const content = msg.content;
|
|
if (content.startsWith('{') || content.startsWith('[') || content.includes('```')) {
|
|
// Keep first 200 chars as context, clear the rest
|
|
const preview = content.slice(0, 200);
|
|
return { role: msg.role, content: `${preview}\n\n[Cleared: ${content.length} chars of detailed output]` };
|
|
}
|
|
}
|
|
|
|
return { ...msg };
|
|
});
|
|
}
|
|
|
|
// ── Step 3: Split Protected Regions ──────────────────────────────────────
|
|
|
|
export interface ProtectedRegions {
|
|
/** System prompt + first N user/assistant messages */
|
|
head: CompressibleMessage[];
|
|
/** Messages in the middle that can be summarized */
|
|
middle: CompressibleMessage[];
|
|
/** Recent messages (last ~protectedTailTokens worth) */
|
|
tail: CompressibleMessage[];
|
|
}
|
|
|
|
/**
|
|
* Split messages into head (protected), middle (compressible), tail (protected).
|
|
*
|
|
* Head: first message (system) + protectedHeadMessages additional messages.
|
|
* Tail: messages from the end that fit within protectedTailTokens.
|
|
* Middle: everything between head and tail.
|
|
*/
|
|
export function splitProtectedRegions(
|
|
messages: ReadonlyArray<CompressibleMessage>,
|
|
config: Pick<CompressionConfig, 'protectedHeadMessages' | 'protectedTailTokens'>,
|
|
): ProtectedRegions {
|
|
// Head: system prompt + first N messages
|
|
const headEnd = Math.min(1 + config.protectedHeadMessages, messages.length);
|
|
const head = messages.slice(0, headEnd);
|
|
|
|
// Tail: walk backwards from the end until we hit the token budget
|
|
let tailTokens = 0;
|
|
let tailStart = messages.length;
|
|
for (let i = messages.length - 1; i >= headEnd; i--) {
|
|
const msgTokens = estimateTokens([messages[i]]);
|
|
if (tailTokens + msgTokens > config.protectedTailTokens) break;
|
|
tailTokens += msgTokens;
|
|
tailStart = i;
|
|
}
|
|
|
|
const tail = messages.slice(tailStart);
|
|
const middle = messages.slice(headEnd, tailStart);
|
|
|
|
return { head, middle, tail };
|
|
}
|
|
|
|
// ── Step 4: Summarize Middle ─────────────────────────────────────────────
|
|
|
|
/**
|
|
* Call the budget model to summarize the compressible middle section.
|
|
* Uses COMPACTION_PROMPT from behavioral-spec.ts.
|
|
*
|
|
* If a previousSummary is provided, it's included so the model can build
|
|
* on accumulated context rather than losing older information.
|
|
*/
|
|
export async function summarizeMiddle(
|
|
middle: ReadonlyArray<CompressibleMessage>,
|
|
config: Pick<CompressionConfig, 'budgetModel' | 'litellmUrl' | 'litellmApiKey' | 'fetch'>,
|
|
previousSummary?: string | null,
|
|
): Promise<string> {
|
|
previousSummary = safePreviousSummary(previousSummary);
|
|
if (middle.length === 0) return previousSummary ?? '';
|
|
|
|
const fetchFn = config.fetch ?? globalThis.fetch;
|
|
|
|
// Build the summarization prompt
|
|
const summarizerMessages: Array<{ role: string; content: string }> = [];
|
|
|
|
// If we have a previous summary, include it as context
|
|
if (previousSummary) {
|
|
summarizerMessages.push({
|
|
role: 'system',
|
|
content: previousSummarySystemMessage(previousSummary),
|
|
});
|
|
}
|
|
|
|
// Add the middle messages as the conversation to summarize
|
|
for (const msg of middle) {
|
|
summarizerMessages.push({ role: msg.role === 'system' ? 'user' : msg.role, content: msg.content ?? '' });
|
|
}
|
|
|
|
// Add the compaction instruction as the final user message
|
|
summarizerMessages.push({ role: 'user', content: COMPACTION_PROMPT });
|
|
|
|
const body = {
|
|
model: config.budgetModel,
|
|
messages: summarizerMessages,
|
|
max_tokens: 2000,
|
|
temperature: 0.1,
|
|
};
|
|
|
|
let response: Response;
|
|
try {
|
|
response = await fetchFn(`${config.litellmUrl}/v1/chat/completions`, {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
'Authorization': `Bearer ${config.litellmApiKey}`,
|
|
},
|
|
body: JSON.stringify(body),
|
|
signal: AbortSignal.timeout(60_000),
|
|
});
|
|
} catch (error) {
|
|
log.warn(`Summarizer request failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
return buildFallbackSummary(middle, previousSummary);
|
|
}
|
|
|
|
if (!response.ok) {
|
|
try {
|
|
const errBody = await response.text();
|
|
log.warn(`Summarizer returned ${response.status}: ${errBody.slice(0, 200)}`);
|
|
} catch { /* ignore read errors */ }
|
|
return buildFallbackSummary(middle, previousSummary);
|
|
}
|
|
|
|
let result: {
|
|
choices?: Array<{
|
|
finish_reason?: string | null;
|
|
message?: {
|
|
content?: string | null;
|
|
tool_calls?: unknown[];
|
|
};
|
|
}>;
|
|
};
|
|
try {
|
|
const parsed = await response.json() as unknown;
|
|
if (typeof parsed !== 'object' || parsed === null) {
|
|
log.warn('Summarizer response was not a JSON object; using deterministic fallback');
|
|
return buildFallbackSummary(middle, previousSummary);
|
|
}
|
|
result = parsed as typeof result;
|
|
} catch (error) {
|
|
log.warn(`Summarizer response was not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
|
|
return buildFallbackSummary(middle, previousSummary);
|
|
}
|
|
const choice = result.choices?.[0];
|
|
if (choice?.finish_reason !== 'stop') {
|
|
log.warn(
|
|
`Summarizer returned an incomplete completion (finish_reason=${choice?.finish_reason ?? 'missing'}); using deterministic fallback`,
|
|
);
|
|
return buildFallbackSummary(middle, previousSummary);
|
|
}
|
|
if (choice.message?.tool_calls?.length) {
|
|
log.warn('Summarizer returned tool calls with finish_reason=stop; using deterministic fallback');
|
|
return buildFallbackSummary(middle, previousSummary);
|
|
}
|
|
const content = choice.message?.content;
|
|
if (typeof content !== 'string' || content.trim().length === 0) {
|
|
return buildFallbackSummary(middle, previousSummary);
|
|
}
|
|
|
|
return content;
|
|
}
|
|
|
|
/**
|
|
* Fallback summary when the LLM call fails.
|
|
* Extracts key signals from messages without any LLM.
|
|
*/
|
|
function buildFallbackSummary(
|
|
middle: ReadonlyArray<CompressibleMessage>,
|
|
previousSummary?: string | null,
|
|
): string {
|
|
const userMessages = middle.filter(m => m.role === 'user');
|
|
const firstLines = userMessages
|
|
.map(m => (m.content ?? '').split('\n')[0]?.trim())
|
|
.filter(line => line && line.length > 10 && line.length < 200)
|
|
.slice(0, 5);
|
|
|
|
const parts: string[] = [];
|
|
if (previousSummary) {
|
|
parts.push('## Previous Context\n' + previousSummary);
|
|
}
|
|
parts.push(`## Compressed Region (${middle.length} messages)`);
|
|
if (firstLines.length > 0) {
|
|
parts.push('Topics: ' + firstLines.join(' → '));
|
|
}
|
|
return parts.join('\n\n');
|
|
}
|
|
|
|
// ── Step 5: Compress Conversation (Orchestrator) ─────────────────────────
|
|
|
|
/**
|
|
* Run the full 5-step compression pipeline.
|
|
*
|
|
* @param messages Full conversation history
|
|
* @param config Compression configuration
|
|
* @param previousSummary Summary from a previous compression pass (for iterative use)
|
|
* @returns CompressionResult with the compressed messages and metadata
|
|
*/
|
|
export async function compressConversation(
|
|
messages: ReadonlyArray<CompressibleMessage>,
|
|
config: CompressionConfig,
|
|
previousSummary?: string | null,
|
|
): Promise<CompressionResult> {
|
|
const originalTokens = estimateTokens(messages);
|
|
const safeSummary = safePreviousSummary(previousSummary);
|
|
|
|
// Step 1: Detect — do we need compression?
|
|
if (!needsCompression(messages, config)) {
|
|
return {
|
|
messages: messages.map(m => ({ ...m })),
|
|
compressed: false,
|
|
originalTokens,
|
|
compressedTokens: originalTokens,
|
|
summaryGenerated: false,
|
|
summary: safeSummary,
|
|
};
|
|
}
|
|
|
|
// Step 2: Prune tool results in the non-tail region
|
|
// Estimate how many messages fit in the tail based on token budget
|
|
const avgTokensPerMsg = originalTokens / messages.length;
|
|
const estimatedTailCount = Math.max(5, Math.ceil(config.protectedTailTokens / avgTokensPerMsg));
|
|
const pruned = pruneToolResults(messages, estimatedTailCount);
|
|
|
|
// Step 3: Split into protected head, compressible middle, protected tail
|
|
const regions = splitProtectedRegions(pruned, config);
|
|
|
|
// If middle is empty or very small, no point summarizing
|
|
if (regions.middle.length <= 2) {
|
|
const result = [...regions.head, ...regions.middle, ...regions.tail];
|
|
return {
|
|
messages: result,
|
|
compressed: false,
|
|
originalTokens,
|
|
compressedTokens: estimateTokens(result),
|
|
summaryGenerated: false,
|
|
summary: safeSummary,
|
|
};
|
|
}
|
|
|
|
// Step 4: Summarize the middle
|
|
const summary = await summarizeMiddle(regions.middle, config, safeSummary);
|
|
|
|
// Step 5: Inject — replace middle with a single summary message
|
|
const summaryMessage: CompressibleMessage = {
|
|
role: 'system',
|
|
content: `[Conversation compressed — ${regions.middle.length} messages summarized]\n\n${summary}`,
|
|
};
|
|
|
|
if (evaluateExternalMemoryIngress({ content: summaryMessage.content }).action !== 'allow') {
|
|
const messagesWithoutSummary = [...regions.head, ...regions.tail];
|
|
return {
|
|
messages: messagesWithoutSummary,
|
|
compressed: true,
|
|
originalTokens,
|
|
compressedTokens: estimateTokens(messagesWithoutSummary),
|
|
summaryGenerated: false,
|
|
summary: null,
|
|
};
|
|
}
|
|
|
|
const compressed = [...regions.head, summaryMessage, ...regions.tail];
|
|
const compressedTokens = estimateTokens(compressed);
|
|
|
|
return {
|
|
messages: compressed,
|
|
compressed: true,
|
|
originalTokens,
|
|
compressedTokens,
|
|
summaryGenerated: true,
|
|
summary,
|
|
};
|
|
}
|
|
|
|
// ── Default Config Factory ───────────────────────────────────────────────
|
|
|
|
/** Sensible defaults for context compression */
|
|
export function createDefaultCompressionConfig(
|
|
overrides: Partial<CompressionConfig> & Pick<CompressionConfig, 'budgetModel' | 'litellmUrl' | 'litellmApiKey'>,
|
|
): CompressionConfig {
|
|
return {
|
|
maxContextTokens: 128000,
|
|
compressionThreshold: 0.5,
|
|
protectedHeadMessages: 3,
|
|
protectedTailTokens: 20000,
|
|
...overrides,
|
|
};
|
|
}
|