Files
waggle-os/packages/agent/src/output-normalize.ts
Oleg Maslov 0c3e2ead3b
Some checks failed
Installer Smoke / installer-smoke (push) Has been cancelled
moving
2026-09-02 10:10:29 +02:00

280 lines
8.9 KiB
TypeScript

/**
* Output normalization layer — Phase 1.1 of agent-fix sprint
* (per decisions/2026-04-26-agent-fix-sprint-plan.md)
*
* Strips reasoning leakage, copied metadata, format scaffolding from
* model outputs. Produces a `(raw, normalized, actions)` triple so every
* transformation is auditable and the original signal is never lost.
*
* HARD CONSTRAINT (from work order):
* "unknown" → "" silently strips legitimate abstention signal — DO NOT.
* Map "unknown" / "Unknown" / "Unknown." / "N/A" → canonical "unknown"
* (preserved, never empty).
*
* Configurable per use case via PRESETS:
* - production: light touch — only obvious leakage (think tags),
* no aggressive label/fence stripping that could
* break user-facing markdown
* - benchmark-strict: full strip — for short-answer benchmarks where
* any extra text counts as wrong
* - benchmark-lenient: middle ground — strip leakage + metadata but
* preserve format
*/
export interface NormalizationAction {
rule: string;
before: string;
after: string;
}
export interface NormalizationResult {
raw: string;
normalized: string;
actions: NormalizationAction[];
}
export interface NormalizationConfig {
/** Strip <think>...</think> blocks (Qwen reasoning leakage). */
stripThinkTags: boolean;
/** Strip leading "Answer:" / "Final answer:" / "Final answer is:" labels. */
stripAnswerLabels: boolean;
/**
* Strip markdown fence markers when the WHOLE response is a single fenced
* block. Does not strip fences mid-response (those may be legitimate code).
*/
stripWholeResponseMarkdownFence: boolean;
/**
* Strip lines that are obviously copied retrieval / memory metadata, e.g.
* [memory:synth]
* # Recalled Memories
* [retrieved 1/8]
* <retrieval_context>...</retrieval_context>
*/
stripCopiedMetadata: boolean;
/**
* Aliases that map to the canonical "unknown" abstention. Comparison is
* case-insensitive and trailing-punctuation-tolerant. The canonical
* output is the literal string "unknown" (lowercase, no punctuation).
*
* IMPORTANT: this NEVER strips abstention to empty. If the input matches
* any alias, the output is exactly "unknown" — preserved as a signal.
*/
unknownAliases: readonly string[];
/**
* Collapse multiple blank lines to a single blank line. Off by default
* because some markdown structure (e.g. h2 + table) reads better with
* preserved blank-line spacing.
*/
collapseBlankLines: boolean;
/** Trim leading/trailing whitespace on the final normalized text. */
trimWhitespace: boolean;
}
const DEFAULT_UNKNOWN_ALIASES: readonly string[] = [
'unknown',
'unknown.',
'n/a',
'na',
'not available',
'not provided',
'i don\'t know',
"don't know",
'cannot determine',
'no answer',
];
export const PRESETS: Record<string, NormalizationConfig> = {
production: {
stripThinkTags: true,
stripAnswerLabels: false,
stripWholeResponseMarkdownFence: false,
stripCopiedMetadata: true,
unknownAliases: DEFAULT_UNKNOWN_ALIASES,
collapseBlankLines: false,
trimWhitespace: true,
},
'benchmark-strict': {
stripThinkTags: true,
stripAnswerLabels: true,
stripWholeResponseMarkdownFence: true,
stripCopiedMetadata: true,
unknownAliases: DEFAULT_UNKNOWN_ALIASES,
collapseBlankLines: true,
trimWhitespace: true,
},
'benchmark-lenient': {
stripThinkTags: true,
stripAnswerLabels: true,
stripWholeResponseMarkdownFence: false,
stripCopiedMetadata: true,
unknownAliases: DEFAULT_UNKNOWN_ALIASES,
collapseBlankLines: false,
trimWhitespace: true,
},
};
const THINK_TAG_REGEX = /<think>[\s\S]*?<\/think>/gi;
const ANSWER_LABEL_REGEX = /^\s*(?:final\s+answer\s+is\s*:|final\s+answer\s*:|answer\s*:)\s*/i;
const COPIED_METADATA_LINE_REGEX = /^\s*(?:\[(?:memory|retrieved|context|recall)[^\]]*\]|<\/?(?:retrieval|memory|context)_?\w*>|#+\s+recalled\s+memories?|#+\s+retrieved\s+context)\s*$/im;
const WHOLE_RESPONSE_FENCE_REGEX = /^\s*```(?:[a-z0-9_-]+)?\s*\n([\s\S]*?)\n\s*```\s*$/i;
function recordAction(
actions: NormalizationAction[],
rule: string,
before: string,
after: string,
): void {
if (before === after) return;
actions.push({ rule, before, after });
}
function applyStripThinkTags(text: string, actions: NormalizationAction[]): string {
if (!THINK_TAG_REGEX.test(text)) return text;
THINK_TAG_REGEX.lastIndex = 0;
const after = text.replace(THINK_TAG_REGEX, '');
recordAction(actions, 'strip-think-tags', text, after);
return after;
}
function applyStripAnswerLabels(text: string, actions: NormalizationAction[]): string {
if (!ANSWER_LABEL_REGEX.test(text)) return text;
const after = text.replace(ANSWER_LABEL_REGEX, '');
recordAction(actions, 'strip-answer-labels', text, after);
return after;
}
function applyStripWholeResponseMarkdownFence(
text: string,
actions: NormalizationAction[],
): string {
const m = text.match(WHOLE_RESPONSE_FENCE_REGEX);
if (!m) return text;
const after = m[1] ?? '';
recordAction(actions, 'strip-whole-response-markdown-fence', text, after);
return after;
}
function applyStripCopiedMetadata(text: string, actions: NormalizationAction[]): string {
const lines = text.split('\n');
const kept: string[] = [];
let stripped = 0;
for (const line of lines) {
if (COPIED_METADATA_LINE_REGEX.test(line)) {
stripped += 1;
continue;
}
kept.push(line);
}
if (stripped === 0) return text;
const after = kept.join('\n');
recordAction(actions, `strip-copied-metadata (${stripped} lines)`, text, after);
return after;
}
function applyCollapseBlankLines(text: string, actions: NormalizationAction[]): string {
const after = text.replace(/\n[ \t]*\n[ \t]*(\n[ \t]*)+/g, '\n\n');
if (after === text) return text;
recordAction(actions, 'collapse-blank-lines', text, after);
return after;
}
function applyTrimWhitespace(text: string, actions: NormalizationAction[]): string {
const after = text.trim();
if (after === text) return text;
recordAction(actions, 'trim-whitespace', text, after);
return after;
}
function applyUnknownAliases(
text: string,
unknownAliases: readonly string[],
actions: NormalizationAction[],
): string {
const candidate = text.trim().replace(/[.!?]+\s*$/, '').toLowerCase();
if (!candidate) return text;
for (const alias of unknownAliases) {
if (candidate === alias.toLowerCase()) {
const after = 'unknown';
if (text === after) return text;
recordAction(actions, `unknown-alias-canonicalize (matched "${alias}")`, text, after);
return after;
}
}
return text;
}
/**
* Normalize a model output per the supplied config. Returns the raw input,
* the normalized output, and an audit trail of every transformation applied
* (in execution order). Empty inputs pass through unchanged with no actions.
*
* Order of operations is fixed and deliberate:
* 1. strip <think> tags (largest substring removal first)
* 2. strip copied metadata (line-level removal, makes label match easier)
* 3. strip whole-response markdown fence (whole-content unwrap)
* 4. strip answer labels (prefix removal on remaining text)
* 5. collapse blank lines
* 6. trim whitespace
* 7. apply unknown aliases (LAST — preserves abstention signal as canonical)
*
* The unknown-alias step runs LAST so it sees the fully cleaned candidate
* and can collapse it to canonical "unknown" iff the entire content is an
* abstention. The hard constraint is that this NEVER produces empty output
* for a non-empty input that matched an alias.
*/
export function normalize(text: string, config: NormalizationConfig): NormalizationResult {
const raw = text;
const actions: NormalizationAction[] = [];
if (!text) {
return { raw, normalized: text, actions };
}
let cur = text;
if (config.stripThinkTags) {
cur = applyStripThinkTags(cur, actions);
}
if (config.stripCopiedMetadata) {
cur = applyStripCopiedMetadata(cur, actions);
}
if (config.stripWholeResponseMarkdownFence) {
cur = applyStripWholeResponseMarkdownFence(cur, actions);
}
if (config.stripAnswerLabels) {
cur = applyStripAnswerLabels(cur, actions);
}
if (config.collapseBlankLines) {
cur = applyCollapseBlankLines(cur, actions);
}
if (config.trimWhitespace) {
cur = applyTrimWhitespace(cur, actions);
}
if (config.unknownAliases.length > 0) {
cur = applyUnknownAliases(cur, config.unknownAliases, actions);
}
return { raw, normalized: cur, actions };
}
/**
* Convenience: normalize using one of the named PRESETS. Throws if the
* preset name is unknown.
*/
export function normalizeWithPreset(text: string, presetName: string): NormalizationResult {
const config = PRESETS[presetName];
if (!config) {
throw new Error(
`output-normalize: unknown preset "${presetName}" — ` +
`available: ${Object.keys(PRESETS).join(', ')}`,
);
}
return normalize(text, config);
}