Files
waggle-os/packages/agent/src/skill-hygiene.ts
Oleg Maslov 0c3e2ead3b
Some checks failed
Installer Smoke / installer-smoke (push) Has been cancelled
moving
2026-09-02 10:10:29 +02:00

176 lines
6.4 KiB
TypeScript

/**
* Skill hygiene judge (Odysseus §D1).
*
* A periodic, single-LLM-call-per-skill audit asking "is this skill still
* NECESSARY, REDUNDANT with a peer, or too GENERIC". Redundant/generic skills
* are DEMOTED to draft via the hygiene sidecar (never deleted/edited). Waggle
* dedups only at creation (skill-distillation.ts → search_skills); an auto-
* growing library bloats without this prune pass.
*
* Reuses judge.ts's LLM-call seam (JudgeLLMCall — the same Haiku-via-LiteLLM
* wiring as the evolution judge) and its balanced-brace JSON extractor. It does
* NOT reuse LLMJudge: that class emits a numeric correctness/procedure/
* conciseness score, structurally wrong for a 3-way categorical verdict.
*
* Fail-safe: any LLM error or unparseable response keeps the skill ACTIVE — a
* flaky judge must never silently disable a proven skill.
*/
import { type JudgeLLMCall, extractJsonCandidates } from './judge.js';
import { loadSkills, type LoadedSkill } from './prompt-loader.js';
import {
demoteSkillToDraft,
isSkillDraft,
type SkillHygieneVerdict,
} from './skill-hygiene-store.js';
export interface SkillForHygiene {
name: string;
content: string;
}
const PEER_PREVIEW_CHARS = 240;
const TARGET_CONTENT_CHARS = 2000;
const HYGIENE_VERDICTS: readonly SkillHygieneVerdict[] = ['necessary', 'redundant', 'generic'];
export const SKILL_HYGIENE_RUBRIC = `You are auditing ONE skill in an AI agent's skill library for hygiene.
A "skill" is a reusable instruction document the agent loads into its system prompt.
Decide whether the TARGET skill should stay active or be demoted to draft.
Classify the TARGET as EXACTLY one of:
- "necessary": specific and useful, and NOT already covered by a peer skill below. KEEP.
- "redundant": substantially overlaps one of the PEER skills; the peer already covers it. DEMOTE.
- "generic": so vague or general it adds nothing the base agent cannot already do. DEMOTE.
Be conservative: only choose "redundant" or "generic" when you are confident. When unsure, choose "necessary".
Return ONLY a JSON object on a single line, no markdown:
{"verdict":"necessary","reason":"<one short sentence>"}`;
function oneLine(s: string): string {
return s.replace(/\s+/g, ' ').trim();
}
export function buildHygienePrompt(target: SkillForHygiene, peers: SkillForHygiene[]): string {
const peerList = peers.length
? peers.map(p => `- ${p.name}: ${oneLine(p.content).slice(0, PEER_PREVIEW_CHARS)}`).join('\n')
: '(none)';
return `${SKILL_HYGIENE_RUBRIC}
PEER SKILLS (name: preview):
${peerList}
TARGET SKILL: ${target.name}
TARGET CONTENT:
${target.content.slice(0, TARGET_CONTENT_CHARS)}
Return the JSON now.`;
}
export interface SkillHygieneJudgement {
name: string;
verdict: SkillHygieneVerdict;
reason: string;
parsed: boolean;
}
export function parseHygieneVerdict(
raw: string,
): { verdict: SkillHygieneVerdict; reason: string } | null {
if (!raw) return null;
const cleaned = raw.replace(/```json\s*/gi, '').replace(/```\s*/g, '').trim();
for (const candidate of extractJsonCandidates(cleaned)) {
try {
const obj = JSON.parse(candidate) as Record<string, unknown>;
const verdict = typeof obj.verdict === 'string' ? obj.verdict.toLowerCase().trim() : '';
if ((HYGIENE_VERDICTS as readonly string[]).includes(verdict)) {
const reason = typeof obj.reason === 'string' ? obj.reason : '';
return { verdict: verdict as SkillHygieneVerdict, reason };
}
} catch {
// try next candidate
}
}
return null;
}
/** One LLM call. Fail-safe: errors / unparseable ⇒ 'necessary' (keep active). */
export async function judgeSkillHygiene(
llmCall: JudgeLLMCall,
target: SkillForHygiene,
peers: SkillForHygiene[],
): Promise<SkillHygieneJudgement> {
let raw = '';
try {
raw = await llmCall(buildHygienePrompt(target, peers));
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
return { name: target.name, verdict: 'necessary', reason: `judge call failed — kept active (${msg})`, parsed: false };
}
const parsed = parseHygieneVerdict(raw);
if (!parsed) {
return { name: target.name, verdict: 'necessary', reason: 'unparseable judge response — kept active', parsed: false };
}
return { name: target.name, verdict: parsed.verdict, reason: parsed.reason || 'No reason given.', parsed: true };
}
export interface SkillHygieneScanOptions {
dryRun?: boolean;
now?: () => Date;
}
export interface SkillHygieneScanReport {
scanned: number;
judged: SkillHygieneJudgement[];
demoted: string[];
kept: string[];
errors: Array<{ name: string; error: string }>;
dryRun: boolean;
}
/**
* Run the hygiene judge over a set of skills. One LLM call per skill (no agent
* re-run). Already-draft skills are skipped (don't re-judge what's demoted) and
* excluded from peer context. Demotes any clearly-parsed redundant/generic
* verdict to draft via the sidecar; NEVER deletes/edits a SKILL.md.
*/
export async function runSkillHygieneScan(
waggleHome: string,
skills: SkillForHygiene[],
llmCall: JudgeLLMCall,
opts: SkillHygieneScanOptions = {},
): Promise<SkillHygieneScanReport> {
const dryRun = opts.dryRun ?? false;
const now = opts.now ?? (() => new Date());
const report: SkillHygieneScanReport = {
scanned: 0, judged: [], demoted: [], kept: [], errors: [], dryRun,
};
const active = skills.filter(s => !isSkillDraft(waggleHome, s.name));
for (const target of active) {
report.scanned++;
const peers = active.filter(s => s.name !== target.name);
try {
const judgement = await judgeSkillHygiene(llmCall, target, peers);
report.judged.push(judgement);
if (judgement.parsed && judgement.verdict !== 'necessary') {
if (!dryRun) demoteSkillToDraft(waggleHome, target.name, judgement.verdict, judgement.reason, now);
report.demoted.push(target.name);
} else {
report.kept.push(target.name);
}
} catch (err) {
report.errors.push({ name: target.name, error: err instanceof Error ? err.message : String(err) });
}
}
return report;
}
/**
* loadSkills minus any skill demoted to draft. The runtime injects these into
* the active system prompt instead of loadSkills so demotion takes effect.
* loadSkills stays the full listing for the Skills Hub card.
*/
export function loadActiveSkills(waggleHome: string): LoadedSkill[] {
return loadSkills(waggleHome).filter(s => !isSkillDraft(waggleHome, s.name));
}