moving
Some checks failed
Installer Smoke / installer-smoke (push) Has been cancelled

This commit is contained in:
Oleg Maslov
2026-09-02 10:10:29 +02:00
commit 0c3e2ead3b
3841 changed files with 970576 additions and 0 deletions

View File

@@ -0,0 +1,154 @@
/**
* GEPA Faza 1 — acceptance validator.
*
* Per launch decision §F (4 must-hold conditions) + §F.5 (Amendment 2 §5
* false-positive evolution guard).
*
* Per-candidate verdict logic:
*
* §F condition 1 (third update — Amendment 2 §5):
* "Best GEPA candidate per shape beats NULL-baseline by ≥+5pp on
* trio_strict_pass rate (trio_mean ≥ 4.0). For Qwen-targeted shapes,
* additionally: best candidate must have mean retrieval_calls per task
* ≥ 1.7 (engagement gap closed by ≥50% relative to Qwen baseline 1.33)."
*
* §F.5 (NEW per Amendment 2 — false-positive guard):
* "If best Qwen-shape candidate achieves +5pp trio_strict delta WITHOUT
* closing retrieval engagement gap (mean retrieval_calls < 1.5), this
* signals false-positive evolution. Result: candidate REJECTED, shape
* marked FAIL even if other criteria pass."
*
* Conditions §F.2/§F.3/§F.4 are evaluated at the run-aggregate level (across
* shapes / κ stability across all evaluations / mutation validator log) and
* are NOT checked per-candidate here. See selection.ts + run-orchestrator
* for those.
*/
import {
type AcceptanceInputs,
type AcceptanceVerdict,
QWEN_TARGETED_SHAPES,
} from './types.js';
/** §F condition 1 trio_strict delta threshold (percentage points). */
export const TRIO_STRICT_DELTA_THRESHOLD_PP = 5;
/**
* §F condition 1 Qwen-only retrieval engagement floor (mean retrieval_calls
* per task). 1.7 = 50% gap closure between Qwen baseline 1.33 and Opus
* parity 2.33 per Amendment 2 §5.
*/
export const QWEN_RETRIEVAL_ENGAGEMENT_FLOOR = 1.7;
/**
* §F.5 false-positive guard threshold (mean retrieval_calls per task).
* If a Qwen candidate achieves trio_strict delta but stays below this floor,
* it is REJECTED as false-positive evolution per Amendment 2 §5.
*/
export const QWEN_FALSE_POSITIVE_RETRIEVAL_FLOOR = 1.5;
/**
* Floating-point tolerance for threshold comparisons. 1e-9 is well below
* any signal magnitude in the +/-0.05 fitness band (which is itself ~9
* orders of magnitude larger). Required because IEEE 754 makes
* `(0.25 - 0.20) * 100 = 4.999999999999999` rather than exact 5.0.
*/
const EPSILON = 1e-9;
/**
* Compute acceptance verdict for a single candidate per launch decision §F + §F.5.
*
* Caller is responsible for §F.2 (≥3/5 shapes positive delta), §F.3 (κ stability),
* and §F.4 (zero cell semantic violations) at the run-aggregate level.
*/
export function evaluateCandidate(inputs: AcceptanceInputs): AcceptanceVerdict {
const { candidate, baselineTrioStrictPassRateII } = inputs;
// Compute delta in percentage points (scale 0..100)
const trioStrictDeltaPP =
(candidate.trioStrictPassRateII - baselineTrioStrictPassRateII) * 100;
const isQwenTargeted = QWEN_TARGETED_SHAPES.has(candidate.shape);
// §F.5 false-positive guard — applies ONLY to Qwen-targeted shapes
// and ONLY when trio_strict delta meets the +5pp threshold.
// Per Amendment 2 §5: if delta ≥ +5pp AND retrieval_calls < 1.5 → REJECT.
// EPSILON tolerance handles IEEE 754 precision on exact-boundary deltas.
let condition5FalsePositiveGuardTriggered = false;
if (
isQwenTargeted &&
trioStrictDeltaPP >= TRIO_STRICT_DELTA_THRESHOLD_PP - EPSILON &&
candidate.meanRetrievalCallsPerTask < QWEN_FALSE_POSITIVE_RETRIEVAL_FLOOR - EPSILON
) {
condition5FalsePositiveGuardTriggered = true;
}
// §F condition 1 — trio_strict delta + (Qwen only) retrieval engagement floor
let condition1Pass = trioStrictDeltaPP >= TRIO_STRICT_DELTA_THRESHOLD_PP - EPSILON;
if (condition1Pass && isQwenTargeted) {
// Qwen sub-criterion: mean retrieval_calls ≥ 1.7
if (candidate.meanRetrievalCallsPerTask < QWEN_RETRIEVAL_ENGAGEMENT_FLOOR - EPSILON) {
condition1Pass = false;
}
}
// Overall acceptance: condition 1 must pass AND §F.5 must not trigger
const accepted = condition1Pass && !condition5FalsePositiveGuardTriggered;
// Reason string for audit log
const reason = buildReason({
candidate,
trioStrictDeltaPP,
condition1Pass,
condition5FalsePositiveGuardTriggered,
isQwenTargeted,
accepted,
});
return {
condition1Pass,
condition5FalsePositiveGuardTriggered,
accepted,
reason,
trioStrictDeltaPP,
};
}
interface BuildReasonInputs {
candidate: { shape: string; meanRetrievalCallsPerTask: number };
trioStrictDeltaPP: number;
condition1Pass: boolean;
condition5FalsePositiveGuardTriggered: boolean;
isQwenTargeted: boolean;
accepted: boolean;
}
function buildReason(i: BuildReasonInputs): string {
const parts: string[] = [];
parts.push(`shape=${i.candidate.shape}`);
parts.push(`trio_strict_delta=${i.trioStrictDeltaPP.toFixed(2)}pp`);
parts.push(`mean_retrieval_calls=${i.candidate.meanRetrievalCallsPerTask.toFixed(2)}`);
if (i.condition5FalsePositiveGuardTriggered) {
parts.push(
`REJECTED §F.5 false-positive guard: delta ≥ +${TRIO_STRICT_DELTA_THRESHOLD_PP}pp AND retrieval < ${QWEN_FALSE_POSITIVE_RETRIEVAL_FLOOR}`,
);
} else if (!i.condition1Pass) {
// Use same EPSILON tolerance as condition1Pass evaluation to keep root-cause
// attribution consistent with the gating logic on exact-boundary deltas.
if (i.trioStrictDeltaPP < TRIO_STRICT_DELTA_THRESHOLD_PP - EPSILON) {
parts.push(
`FAIL §F.1 trio_strict delta: ${i.trioStrictDeltaPP.toFixed(2)}pp < ${TRIO_STRICT_DELTA_THRESHOLD_PP}pp`,
);
} else if (i.isQwenTargeted) {
parts.push(
`FAIL §F.1 Qwen retrieval floor: ${i.candidate.meanRetrievalCallsPerTask.toFixed(2)} < ${QWEN_RETRIEVAL_ENGAGEMENT_FLOOR}`,
);
}
} else {
parts.push(`PASS §F.1${i.isQwenTargeted ? ' (Qwen retrieval floor met)' : ''}`);
}
parts.push(`accepted=${i.accepted}`);
return parts.join(' | ');
}

View File

@@ -0,0 +1,114 @@
/**
* GEPA Faza 1 — Opus 4.7 oracle prompt for synthesizing one H3 corpus instance.
*
* Per Amendment 1 Ask A Option C sub-ask 4 (instance generation methodology):
* "Opus 4.7 generates task scaffold (persona + scenario + 6-7 source document specs);
* PM does NOT review each instance pre-NULL-baseline. Instead: CC-2 spot-audits
* 5 random instances pre NULL-baseline kick (3% sample)."
*
* The oracle is instructed to produce a JSON object that maps directly onto
* the CorpusInstance shape (minus computed fields like materialsConcat, charCount).
*/
import { type StratificationCell, type CorpusInstance, TASK_FAMILY_DESCRIPTORS } from './corpus.js';
/** Persona descriptor — short label + role context for prompt template. */
const PERSONA_DESCRIPTORS: Record<string, { label: string; roleContext: string }> = {
p1_founder_ceo: {
label: 'Founder/CEO',
roleContext: 'company founder + chief executive; focused on growth, fundraising, vision',
},
p2_cfo: {
label: 'CFO',
roleContext: 'chief financial officer; focused on burn, runway, capital efficiency',
},
p3_coo: {
label: 'COO',
roleContext: 'chief operating officer; focused on operational efficiency, hiring, process',
},
p4_vp_finance: {
label: 'VP Finance',
roleContext: 'reports to CFO; focused on FP&A, financial reporting, budget management',
},
p5_independent_director: {
label: 'Independent Director',
roleContext: 'board-level oversight; focused on governance, strategic risk, executive accountability',
},
};
/** Company-stage descriptor — narrative framing for Opus to ground each instance. */
const COMPANY_STAGE_DESCRIPTORS: Record<string, { label: string; financial: string; pressure: string }> = {
stage_a_series_b_growth_burning: {
label: 'Series B growth-stage, burning capital',
financial: 'recently raised Series B ~$30M; burning $1-1.5M/month; 12-18 months runway; ARR $10-20M',
pressure: 'investor pressure to demonstrate capital efficiency; competitive displacement risk; talent retention concerns',
},
stage_b_post_profitable_consolidation: {
label: 'Post-profitable, consolidation phase',
financial: 'reached profitability 2-4 quarters ago; $40-80M ARR; 15-25% operating margin; 24+ months runway',
pressure: 'Wall Street scrutiny on growth deceleration; M&A integration challenges; complacency risk',
},
};
export interface BuildCorpusInstancePromptInputs {
cell: StratificationCell;
instanceId: string;
/** Domain anchor — ensures all 50 instances stay in NorthLane CFO synthesis territory. */
domainAnchor?: string;
}
/** Build the Opus 4.7 generation prompt for one stratification cell. */
export function buildCorpusInstancePrompt(inputs: BuildCorpusInstancePromptInputs): string {
const { cell, instanceId } = inputs;
const familyDesc = TASK_FAMILY_DESCRIPTORS[cell.family];
const personaDesc = PERSONA_DESCRIPTORS[cell.persona];
const stageDesc = COMPANY_STAGE_DESCRIPTORS[cell.stage];
return `You are generating one synthesis-task instance for a benchmark corpus.
The corpus targets B2B SaaS knowledge work in the spirit of "NorthLane CFO" pilot tasks: a complex business situation requiring multi-document synthesis to produce a structured business deliverable (memo, action plan, decision recommendation).
## Stratification cell for this instance
- Instance ID: \`${instanceId}\`
- Task family: \`${cell.family}\` — ${familyDesc.label}
- Task family description: ${familyDesc.label.replace(/_/g, ' ')}; mirror pattern: ${familyDesc.mirrorPilotTask ?? 'NEW family (no pilot mirror)'}
- Persona: \`${cell.persona}\` — ${personaDesc.label} (${personaDesc.roleContext})
- Company stage: \`${cell.stage}\` — ${stageDesc.label}
- Financial state: ${stageDesc.financial}
- Pressure dynamics: ${stageDesc.pressure}
## Required output (JSON object only, no prose, no code fences)
\`\`\`json
{
"personaText": "Persona: <2-3 sentence persona description grounding the model in role + company>\\n\\nScenario: <2-4 sentence scenario hook setting up why this synthesis is needed now>",
"sourceDocuments": [
{ "title": "DOC 1 — <Doc name>", "body": "<markdown content, 600-1200 chars>" },
{ "title": "DOC 2 — <Doc name>", "body": "<...>" },
...
(exactly ${familyDesc.docsPerInstance} docs total)
],
"question": "<open-ended question, 100-400 chars, that requires synthesizing across multiple docs to answer>"
}
\`\`\`
## Generation constraints (binding)
1. **Domain anchor:** All ${familyDesc.docsPerInstance} source documents must be plausible artifacts a ${personaDesc.label} would receive in a B2B SaaS company at the ${stageDesc.label} phase.
2. **Document diversity:** Each doc should be a different artifact type (P&L summary, pipeline review, churn analysis, eng velocity report, marketing dashboard, board notes, competitor intel, customer interview notes, OKR report, runway model, etc.). No two docs should be the same artifact type.
3. **Synthesis requirement:** The question must be answerable ONLY by triangulating across multiple docs. A single-doc answer should be insufficient. Specifically, the question should require:
- Identifying tradeoffs across docs
- Synthesizing competing positions
- Producing a structured deliverable (memo, action plan, or recommendation with justification)
4. **Persona consistency:** The persona's role + company-stage pressures must be evident in BOTH the scenario framing AND the question phrasing. A founder-CEO question reads differently than an independent-director question.
5. **Doc body realism:** Use realistic numbers (revenue figures, percentages, dates). Cite specific people by role title (e.g., "VP Sales", not "John Smith"). Include both hard data + qualitative commentary in each doc.
6. **JSON format:** Output ONLY a single valid JSON object. No prose before or after. No code fence markers. The first character must be \`{\` and the last character must be \`}\`.
## Output begins below
`;
}

View File

@@ -0,0 +1,311 @@
/**
* GEPA Faza 1 — H3 corpus data definitions, stratification, validation,
* and spot-audit sampler.
*
* Per Amendment 1 Ask A Option C + manifest v7 §corpus_design:
* - 50 instances total
* - 5 task families × 5 personas × 2 company stages = 50 cells
* - NorthLane CFO synthesis domain (preserves Phase 4.3 anchor)
* - Each instance: ≥6 source documents, 6-dim Likert rubric
* - Stratified sampling: deterministic, seed=42
* - Generated via Opus 4.7 oracle
*
* Pre-A halt-and-PM checkpoint: spot-audit 5 random instances.
*/
import * as crypto from 'node:crypto';
/** 5 task families per manifest v7 §corpus_design.task_families. */
export type TaskFamily = 'F1' | 'F2' | 'F3' | 'F4' | 'F5';
/** 5 persona codes per manifest v7 §corpus_design.persona_axis. */
export type PersonaCode = 'p1_founder_ceo' | 'p2_cfo' | 'p3_coo' | 'p4_vp_finance' | 'p5_independent_director';
/** 2 company stages per manifest v7 §corpus_design.company_stage_axis. */
export type CompanyStage = 'stage_a_series_b_growth_burning' | 'stage_b_post_profitable_consolidation';
/** All 5 task families in canonical order. */
export const TASK_FAMILIES: ReadonlyArray<TaskFamily> = ['F1', 'F2', 'F3', 'F4', 'F5'];
/** All 5 personas in canonical order. */
export const PERSONAS: ReadonlyArray<PersonaCode> = [
'p1_founder_ceo',
'p2_cfo',
'p3_coo',
'p4_vp_finance',
'p5_independent_director',
];
/** All 2 company stages in canonical order. */
export const COMPANY_STAGES: ReadonlyArray<CompanyStage> = [
'stage_a_series_b_growth_burning',
'stage_b_post_profitable_consolidation',
];
/** Total instances per manifest v7 §corpus_design.total_instances. */
export const TOTAL_INSTANCES = 50;
/** Required source document count floor per manifest v7. */
export const DOCS_PER_INSTANCE_MIN = 6;
/** Required source document count ceiling per manifest v7. */
export const DOCS_PER_INSTANCE_MAX = 8;
/** Spot-audit sample size per manifest v7 §corpus_design.spot_audit. */
export const SPOT_AUDIT_SAMPLE_SIZE = 5;
/** Deterministic stratification seed per manifest v7 §corpus_design.stratified_sampling.seed. */
export const STRATIFICATION_SEED = 42;
/** Per-instance human-readable task family descriptor. */
export const TASK_FAMILY_DESCRIPTORS: Record<TaskFamily, {
label: string;
mirrorPilotTask: string | null;
docsPerInstance: number;
promptTemplate: string;
}> = {
F1: {
label: 'strategic_synthesis',
mirrorPilotTask: 'task-1',
docsPerInstance: 7,
promptTemplate: 'Identify the 3 most critical risks for {company} in {period} and propose action plan for each.',
},
F2: {
label: 'cross_thread_coordination',
mirrorPilotTask: 'task-2',
docsPerInstance: 6,
promptTemplate: 'Reconcile conflicting positions from {stakeholders} and propose unified approach.',
},
F3: {
label: 'decision_support',
mirrorPilotTask: 'task-3',
docsPerInstance: 6,
promptTemplate: 'Recommend {decision} based on materials; justify, address counter-arguments.',
},
F4: {
label: 'investor_communications',
mirrorPilotTask: null, // NEW family
docsPerInstance: 6,
promptTemplate: 'Draft Q{n} investor update covering {metrics} + addressing {concerns}.',
},
F5: {
label: 'scenario_planning',
mirrorPilotTask: null, // NEW family
docsPerInstance: 7,
promptTemplate: 'Compare {n} scenarios for {decision_area}; recommend hedging strategy.',
},
};
/** Stratification cell — one of 50 unique combinations. */
export interface StratificationCell {
family: TaskFamily;
persona: PersonaCode;
stage: CompanyStage;
}
/** A single source document inside a corpus instance. */
export interface SourceDoc {
title: string;
body: string;
charCount: number;
}
/** A single corpus instance ready for evaluation by NULL-baseline / GEPA candidates. */
export interface CorpusInstance {
/** Stable instance ID: h3-{family}-{persona}-{stage}-{ordinal}. */
instanceId: string;
cell: StratificationCell;
personaText: string;
scenario: string;
sourceDocuments: SourceDoc[];
question: string;
/** Aggregated materials block (joined source docs) — consumer convenience. */
materialsConcat: string;
manifestAnchor: string;
generatedBy: string;
generatedAtIso: string;
generationCostUsd: number;
}
// ───────────────────────────────────────────────────────────────────────────
// Stratification — deterministic enumeration of 50 cells
// ───────────────────────────────────────────────────────────────────────────
/**
* Enumerate all 50 stratification cells in canonical (family-major, persona-mid,
* stage-minor) order. Deterministic given the constant arrays above.
*
* Yield order: F1×p1×stage_a, F1×p1×stage_b, F1×p2×stage_a, ..., F5×p5×stage_b.
*/
export function* iterateStratificationCells(): Generator<StratificationCell> {
for (const family of TASK_FAMILIES) {
for (const persona of PERSONAS) {
for (const stage of COMPANY_STAGES) {
yield { family, persona, stage };
}
}
}
}
/** Materialize all 50 cells as an array. */
export function listStratificationCells(): StratificationCell[] {
return Array.from(iterateStratificationCells());
}
/** Build a stable instance ID from a cell + ordinal (1-based within cell). */
export function buildInstanceId(cell: StratificationCell, ordinal: number = 1): string {
return `h3-${cell.family}-${cell.persona}-${cell.stage}-${String(ordinal).padStart(3, '0')}`;
}
// ───────────────────────────────────────────────────────────────────────────
// Validation — check instance shape against manifest v7 quality floor
// ───────────────────────────────────────────────────────────────────────────
export interface InstanceValidationResult {
valid: boolean;
violations: string[];
}
/** Per manifest v7 §corpus_design.per_instance_quality_floor + spot_audit dimensions. */
export function validateInstance(instance: CorpusInstance): InstanceValidationResult {
const violations: string[] = [];
if (instance.sourceDocuments.length < DOCS_PER_INSTANCE_MIN) {
violations.push(`docs count ${instance.sourceDocuments.length} < ${DOCS_PER_INSTANCE_MIN} min`);
}
if (instance.sourceDocuments.length > DOCS_PER_INSTANCE_MAX) {
violations.push(`docs count ${instance.sourceDocuments.length} > ${DOCS_PER_INSTANCE_MAX} max`);
}
// Length bounds loosened post-probe (2026-04-28) — Opus generates naturally
// richer personas/scenarios than the hand-crafted pilot baseline. The intent
// of these bounds is to catch broken/empty output, not to police verbosity.
const personaLen = instance.personaText.length;
if (personaLen < 100 || personaLen > 1500) {
violations.push(`persona length ${personaLen} outside [100, 1500] range`);
}
// Scenario may be empty if oracle embedded it in personaText (handled by
// assembleInstance — extractScenarioFromPersonaText). Skip length floor in
// that case but still cap upper bound.
const scenarioLen = instance.scenario.length;
if (scenarioLen > 0 && (scenarioLen < 100 || scenarioLen > 1500)) {
violations.push(`scenario length ${scenarioLen} outside [100, 1500] range (or 0 if embedded in personaText)`);
}
const questionLen = instance.question.length;
if (questionLen < 100 || questionLen > 800) {
violations.push(`question length ${questionLen} outside [100, 800] range`);
}
for (const doc of instance.sourceDocuments) {
if (doc.body.length < 400 || doc.body.length > 3000) {
violations.push(`doc "${doc.title}" body length ${doc.body.length} outside [400, 3000] range`);
}
if (doc.body.length !== doc.charCount) {
violations.push(`doc "${doc.title}" charCount ${doc.charCount} != actual body length ${doc.body.length}`);
}
}
if (!instance.instanceId.startsWith('h3-')) {
violations.push(`instanceId "${instance.instanceId}" missing h3- prefix`);
}
return {
valid: violations.length === 0,
violations,
};
}
// ───────────────────────────────────────────────────────────────────────────
// Spot-audit sampler — deterministic random selection of N instances
// ───────────────────────────────────────────────────────────────────────────
/**
* Mulberry32 PRNG — deterministic, fast, 32-bit. Same seed produces same
* output across runs and platforms.
*/
function mulberry32(seed: number): () => number {
let t = seed >>> 0;
return function () {
t = (t + 0x6d2b79f5) >>> 0;
let r = t;
r = Math.imul(r ^ (r >>> 15), r | 1);
r ^= r + Math.imul(r ^ (r >>> 7), r | 61);
return ((r ^ (r >>> 14)) >>> 0) / 4294967296;
};
}
/**
* Deterministic Fisher-Yates shuffle producing a sample of `sampleSize` from
* the input array, seeded by `seed`. Reproducible across runs.
*/
export function deterministicSample<T>(
items: ReadonlyArray<T>,
sampleSize: number,
seed: number = STRATIFICATION_SEED,
): T[] {
if (sampleSize >= items.length) {
return [...items];
}
const arr = [...items];
const rand = mulberry32(seed);
// Partial Fisher-Yates: only need first `sampleSize` swapped to front.
for (let i = 0; i < sampleSize; i++) {
const j = i + Math.floor(rand() * (arr.length - i));
[arr[i], arr[j]] = [arr[j], arr[i]];
}
return arr.slice(0, sampleSize);
}
/** Select 5 random instances per manifest v7 §corpus_design.spot_audit. */
export function selectSpotAuditSample(instances: ReadonlyArray<CorpusInstance>): CorpusInstance[] {
return deterministicSample(instances, SPOT_AUDIT_SAMPLE_SIZE);
}
// ───────────────────────────────────────────────────────────────────────────
// Aggregate spot-audit verdict
// ───────────────────────────────────────────────────────────────────────────
export interface SpotAuditReport {
sampleSize: number;
perInstance: Array<{ instanceId: string; result: InstanceValidationResult }>;
/** Per manifest v7 §corpus_design.spot_audit.halt_on: any 1 of 5 fails → corpus regeneration. */
haltOnFailure: boolean;
haltReason?: string;
}
/** Run validation across spot-audit sample + return aggregate verdict. */
export function runSpotAudit(instances: ReadonlyArray<CorpusInstance>): SpotAuditReport {
const sample = selectSpotAuditSample(instances);
const perInstance = sample.map(inst => ({
instanceId: inst.instanceId,
result: validateInstance(inst),
}));
const failed = perInstance.filter(p => !p.result.valid);
return {
sampleSize: sample.length,
perInstance,
haltOnFailure: failed.length > 0,
haltReason: failed.length > 0
? `${failed.length}/${sample.length} spot-audit instances failed validation: ${failed.map(f => f.instanceId).join(', ')}`
: undefined,
};
}
// ───────────────────────────────────────────────────────────────────────────
// Audit-chain helpers
// ───────────────────────────────────────────────────────────────────────────
/** Compute SHA-256 of a JSON-serialized corpus (for manifest v7 audit pinning). */
export function corpusSha256(instances: ReadonlyArray<CorpusInstance>): string {
const canonical = instances.map(inst => ({
instanceId: inst.instanceId,
cell: inst.cell,
materialsLength: inst.materialsConcat.length,
questionLength: inst.question.length,
docCount: inst.sourceDocuments.length,
}));
const json = JSON.stringify(canonical);
return crypto.createHash('sha256').update(json).digest('hex');
}

View File

@@ -0,0 +1,133 @@
/**
* GEPA Faza 1 — cost governance + super-linear projection tracker.
*
* Per launch decision §A.7 + manifest v7 §cost_governance.
*
* Halt triggers (any single triggers immediate halt + PM ratify):
* - Cumulative spend > $80 (internal halt)
* - Cumulative spend > $100 (hard cap)
* - Mid-run actual exceeds projection by >30% (super-linear sub-rule per brief §6.7)
*
* Audit cadence: every 20 evaluations.
*
* Cost projection methodology: 1.5× baseline token count per candidate
* (encodes mutation overhead since GEPA candidates may grow prompts).
*/
/** Hard cap (immediate halt on breach). */
export const HARD_CAP_USD = 100.00;
/** Internal halt threshold (halt + PM ratify before proceeding). */
export const INTERNAL_HALT_USD = 80.00;
/** Super-linear projection multiplier per brief §6.7. */
export const SUPER_LINEAR_MULTIPLIER = 1.5;
/** Mid-run halt threshold: actual exceeds projection by this fraction. */
export const SUPER_LINEAR_OVERAGE_THRESHOLD = 0.30;
/** Audit cadence (every N evaluations). */
export const AUDIT_CADENCE_EVAL_COUNT = 20;
export type HaltReason =
| 'NONE'
| 'INTERNAL_HALT_USD_BREACH'
| 'HARD_CAP_USD_BREACH'
| 'SUPER_LINEAR_PROJECTION_BREACH';
export interface CostTrackerState {
cumulativeUsd: number;
evaluationCount: number;
/** Per-evaluation projection used for super-linear check (1.5× baseline median). */
projectionPerEvalUsd: number;
}
export interface HaltCheckResult {
haltReason: HaltReason;
cumulativeUsd: number;
expectedAtThisCount: number;
overageFraction: number;
message: string;
}
/** Create a new cost tracker state with the per-evaluation projection. */
export function createCostTracker(baselineMedianCostPerEvalUsd: number): CostTrackerState {
return {
cumulativeUsd: 0,
evaluationCount: 0,
projectionPerEvalUsd: baselineMedianCostPerEvalUsd * SUPER_LINEAR_MULTIPLIER,
};
}
/**
* Record an evaluation cost and return updated state (immutable update).
* Per coding-style.md: never mutate, always return new copy.
*/
export function recordEvaluation(state: CostTrackerState, evalCostUsd: number): CostTrackerState {
return {
...state,
cumulativeUsd: state.cumulativeUsd + evalCostUsd,
evaluationCount: state.evaluationCount + 1,
};
}
/**
* Check halt triggers against current state.
*
* Returns NONE if all checks pass; otherwise returns the first triggered
* halt reason with diagnostic context.
*
* Order of precedence (most severe first):
* 1. HARD_CAP_USD_BREACH ($100 breach)
* 2. INTERNAL_HALT_USD_BREACH ($80 breach)
* 3. SUPER_LINEAR_PROJECTION_BREACH (actual > 1.30 × expected at current eval count)
*/
export function checkHaltTriggers(state: CostTrackerState): HaltCheckResult {
const expectedAtThisCount = state.projectionPerEvalUsd * state.evaluationCount;
const overageFraction =
expectedAtThisCount > 0 ? (state.cumulativeUsd - expectedAtThisCount) / expectedAtThisCount : 0;
if (state.cumulativeUsd > HARD_CAP_USD) {
return {
haltReason: 'HARD_CAP_USD_BREACH',
cumulativeUsd: state.cumulativeUsd,
expectedAtThisCount,
overageFraction,
message: `HARD CAP BREACH: $${state.cumulativeUsd.toFixed(2)} > $${HARD_CAP_USD} cap`,
};
}
if (state.cumulativeUsd > INTERNAL_HALT_USD) {
return {
haltReason: 'INTERNAL_HALT_USD_BREACH',
cumulativeUsd: state.cumulativeUsd,
expectedAtThisCount,
overageFraction,
message: `INTERNAL HALT: $${state.cumulativeUsd.toFixed(2)} > $${INTERNAL_HALT_USD} internal halt — PM ratify before proceeding`,
};
}
// Super-linear check requires at least 1 eval to have meaningful expected value.
if (state.evaluationCount > 0 && overageFraction > SUPER_LINEAR_OVERAGE_THRESHOLD) {
return {
haltReason: 'SUPER_LINEAR_PROJECTION_BREACH',
cumulativeUsd: state.cumulativeUsd,
expectedAtThisCount,
overageFraction,
message: `SUPER-LINEAR BREACH: actual $${state.cumulativeUsd.toFixed(2)} exceeds expected $${expectedAtThisCount.toFixed(2)} by ${(overageFraction * 100).toFixed(1)}% (>${(SUPER_LINEAR_OVERAGE_THRESHOLD * 100).toFixed(0)}% threshold)`,
};
}
return {
haltReason: 'NONE',
cumulativeUsd: state.cumulativeUsd,
expectedAtThisCount,
overageFraction,
message: `OK: $${state.cumulativeUsd.toFixed(2)} cumulative; expected $${expectedAtThisCount.toFixed(2)}; ${(overageFraction * 100).toFixed(1)}% overage`,
};
}
/** Whether the current eval count is on an audit cadence boundary. */
export function shouldAudit(state: CostTrackerState): boolean {
return state.evaluationCount > 0 && state.evaluationCount % AUDIT_CADENCE_EVAL_COUNT === 0;
}

View File

@@ -0,0 +1,298 @@
/**
* GEPA Faza 1 — per-shape fitness function.
*
* Per Amendment 2 §3 (manifest v7 §metric_operationalization.per_shape_fitness_formula
* + §metric_operationalization.retrieval_engagement_bonus).
*
* Phase 4.5 empirical signal: Qwen retrieves 1.33×/task vs Opus 2.33×/task.
* H4 score gap mechanistically traces to under-engagement. Mutation surface
* (prompt-shape body) is the lever to address this on Qwen-targeted shapes.
*
* Fitness function forks by shape class:
* Qwen-targeted: fitness = trio_strict_pass_rate + retrieval_engagement_bonus − cost_penalty
* Non-Qwen: fitness = trio_strict_pass_rate − cost_penalty
*
* Retrieval engagement bonus bands (Qwen-targeted shapes only):
* +0.05 if mean retrieval_calls per task ≥ 2.0 (Opus parity proxy)
* 0.00 if mean retrieval_calls per task ∈ [1.5, 2.0)
* −0.05 if mean retrieval_calls per task < 1.5 (Qwen baseline penalty)
*/
import {
type FitnessInputs,
type FitnessComponents,
type ShapeName,
type TieredFitnessInputs,
type TieredFitnessComponents,
type DeltaFloorInputs,
type DeltaFloorVerdict,
QWEN_TARGETED_SHAPES,
} from './types.js';
/** Cost penalty coefficient per brief §3.1 — 0.5pp per $0.10 above baseline median. */
const COST_PENALTY_PP_PER_DOLLAR_10C = 0.5;
/** Retrieval engagement thresholds per Amendment 2 §3 bands. */
export const RETRIEVAL_ENGAGEMENT_BANDS = {
/** ≥ this → +0.05 bonus (Opus parity proxy). */
upperThreshold: 2.0,
/** ≥ this and < upper → 0.00. */
lowerThreshold: 1.5,
/** < lower → −0.05 (Qwen baseline penalty). */
bonusPlus: 0.05,
bonusZero: 0.0,
bonusMinus: -0.05,
} as const;
/**
* Compute the retrieval engagement bonus for a Qwen-targeted shape.
*
* Returns 0.0 for non-Qwen shapes (callers should branch on shape class
* before calling this if they need to distinguish; alternatively use
* `computeFitness` which handles routing).
*
* Boundary semantics (binding per Amendment 2 §8 test cases):
* 1.49 → −0.05
* 1.50 → 0.00 (exact lower threshold = zero band)
* 1.99 → 0.00
* 2.00 → +0.05 (exact upper threshold = bonus band)
* 2.50 → +0.05
*/
export function computeRetrievalEngagementBonus(
shape: ShapeName,
meanRetrievalCallsPerTask: number,
): number {
if (!QWEN_TARGETED_SHAPES.has(shape)) {
return 0.0;
}
if (meanRetrievalCallsPerTask >= RETRIEVAL_ENGAGEMENT_BANDS.upperThreshold) {
return RETRIEVAL_ENGAGEMENT_BANDS.bonusPlus;
}
if (meanRetrievalCallsPerTask >= RETRIEVAL_ENGAGEMENT_BANDS.lowerThreshold) {
return RETRIEVAL_ENGAGEMENT_BANDS.bonusZero;
}
return RETRIEVAL_ENGAGEMENT_BANDS.bonusMinus;
}
/**
* Compute the cost penalty per brief §3.1.
*
* If the candidate's mean cost is at or below the per-shape baseline median,
* penalty is 0. Otherwise, penalty = 0.5pp per $0.10 of overage, encoded as
* a positive decimal (caller subtracts from fitness).
*/
export function computeCostPenalty(
candidateMeanCostUsd: number,
baselineMedianCostUsd: number,
): number {
const overageUsd = candidateMeanCostUsd - baselineMedianCostUsd;
if (overageUsd <= 0) {
return 0.0;
}
// 0.5 pp per $0.10 → 5 pp per $1.00 → encoded as 0.05 per $1.00 → 0.005 per $0.10
const penaltyDecimal = (overageUsd / 0.10) * (COST_PENALTY_PP_PER_DOLLAR_10C / 100);
return penaltyDecimal;
}
/**
* Compute the per-shape fitness for a candidate.
*
* Routes by shape class:
* - Qwen-targeted (qwen-thinking, qwen-non-thinking): includes retrieval engagement bonus
* - Non-Qwen (claude, gpt, generic-simple): no retrieval engagement weighting
*
* Returns full FitnessComponents for downstream audit + reporting per
* launch decision §A.9 binding compliance requirement.
*/
export function computeFitness(inputs: FitnessInputs): FitnessComponents {
const { candidate, baselineMedianCostUsd } = inputs;
const trioStrictPassRateII = candidate.trioStrictPassRateII;
const retrievalEngagementApplied = QWEN_TARGETED_SHAPES.has(candidate.shape);
const retrievalEngagementBonus = retrievalEngagementApplied
? computeRetrievalEngagementBonus(candidate.shape, candidate.meanRetrievalCallsPerTask)
: 0.0;
const costPenalty = computeCostPenalty(candidate.meanCostUsd, baselineMedianCostUsd);
const fitness = trioStrictPassRateII + retrievalEngagementBonus - costPenalty;
return {
trioStrictPassRateII,
retrievalEngagementBonus,
costPenalty,
fitness,
retrievalEngagementApplied,
};
}
// ── Amendment 7 — tiered fitness function ─────────────────────────────────
/** Tier 2 cap per Amendment 7 §fitness_function_tiered.tier_2 (max bonus). */
export const TIER_2_BONUS_CAP = 0.25;
/** Tier 2 weight per percentage point of retrieval engagement above NULL baseline. */
export const TIER_2_BONUS_PER_PP = 0.05;
/** Tier 3 binary bonus when all 7 cell-semantic anchors invariant. */
export const TIER_3_BONUS_FULL_INVARIANCE = 0.10;
/** Total cell-semantic anchor count (types.ts + MULTI_STEP_ACTION_CONTRACT + 5 baseline shapes). */
export const TIER_3_ANCHOR_COUNT_FULL = 7;
/**
* Compute Tier 2 retrieval engagement bonus per Amendment 7 §fitness_function_tiered.tier_2.
*
* Continuous formula (supersedes Amendment 2 band bonus for tiered ranking):
* bonus = clamp(0.05 × delta_pp, 0, 0.25)
* where delta_pp = (candidate_mean - baseline_mean) × 100
*
* Cap reached at +5pp absolute increase in mean retrieval calls per task.
* Floor 0 (no negative bonus from Tier 2 — negative-band penalty handled
* by Amendment 2 §F.5 false-positive guard separately).
*
* Always returns 0 for non-Qwen shapes.
*/
export function computeTier2RetrievalBonus(
shape: ShapeName,
candidateMeanRetrievalCallsPerTask: number,
baselineMeanRetrievalCallsPerTask: number,
): number {
if (!QWEN_TARGETED_SHAPES.has(shape)) return 0;
const deltaAbsolute = candidateMeanRetrievalCallsPerTask - baselineMeanRetrievalCallsPerTask;
if (deltaAbsolute <= 0) return 0;
const deltaPP = deltaAbsolute * 100;
return Math.min(TIER_2_BONUS_PER_PP * deltaPP, TIER_2_BONUS_CAP);
}
/**
* Compute the per-shape tiered fitness per Amendment 7 §fitness_function_tiered.
*
* Routes tier roles by saturation:
* - In saturated regime (≥4/5 shapes have NULL pass rate ≥75%):
* PRIMARY differentiator = Tier 2 (Qwen-targeted retrieval engagement)
* SECONDARY differentiator = Tier 3 (cell-semantic anchor invariance)
* TIE-BREAKER = Tier 1 (NULL pass rate delta)
* aggregateSaturatedRegime = tier_2 + tier_3
* - Outside saturated regime: legacy Amendment 2 form (computeFitness) applies
* and aggregateSaturatedRegime is reported but not the canonical fitness.
*
* Tier 1 also serves as the §F.1 acceptance gate (≥+5pp) — UNCHANGED from
* Amendment 5 launch decision §F.1.
*/
export function computeTieredFitness(inputs: TieredFitnessInputs): TieredFitnessComponents {
const {
candidate,
nullBaselinePassRateII,
nullBaselineMeanRetrievalCallsPerTask,
mutationValidatorPassed,
saturatedRegime,
} = inputs;
// Tier 1 — NULL pass rate delta (signed pp)
const tier1DeltaPP = (candidate.trioStrictPassRateII - nullBaselinePassRateII) * 100;
// Tier 2 — continuous retrieval engagement bonus (Qwen-targeted only)
const tier2RetrievalBonus = computeTier2RetrievalBonus(
candidate.shape,
candidate.meanRetrievalCallsPerTask,
nullBaselineMeanRetrievalCallsPerTask,
);
// Tier 3 — binary cell-semantic anchor invariance bonus
const tier3CellSemanticInvarianceBonus = mutationValidatorPassed
? TIER_3_BONUS_FULL_INVARIANCE
: 0;
// For Gen 1 candidates the mutation_validator gives binary (valid|invalid);
// count semantic: pass = full 7-anchor invariance, fail = 0.
const cellSemanticAnchorInvarianceCount = mutationValidatorPassed ? TIER_3_ANCHOR_COUNT_FULL : 0;
// Aggregate fitness in saturated regime: tier_2 + tier_3 only
const aggregateSaturatedRegime = tier2RetrievalBonus + tier3CellSemanticInvarianceBonus;
return {
tier1DeltaPP,
tier2RetrievalBonus,
tier3CellSemanticInvarianceBonus,
aggregateSaturatedRegime,
saturatedRegimeApplied: saturatedRegime,
cellSemanticAnchorInvarianceCount,
};
}
// ── Amendment 7 — pre-registered Δ-floor verdict ──────────────────────────
/** Δ-floor thresholds per Amendment 7 §gen_1_pre_registered_delta_floor. */
export const DELTA_FLOOR_THRESHOLDS = {
/** Threshold 1: aggregate Tier 1 delta in pp (≥+3pp passes). */
threshold1AggregateTier1PP: 3,
/** Threshold 2: max Qwen retrieval engagement delta absolute (≥+0.10 passes). */
threshold2QwenRetrievalAbsolute: 0.10,
/** Threshold 3a: aggregate Tier 1 delta in pp (≥0pp). */
threshold3Tier1MinPP: 0,
/** Threshold 3b: aggregate Tier 2 bonus across Qwen-targeted candidates (≥0.05). */
threshold3Tier2MinBonus: 0.05,
} as const;
/** Float tolerance for exact-boundary threshold comparisons (consistent with acceptance.ts EPSILON). */
const DELTA_FLOOR_EPSILON = 1e-9;
/**
* Compute the Δ-floor verdict per Amendment 7 §gen_1_pre_registered_delta_floor.
*
* Three OR-gated thresholds:
* 1. aggregate Tier 1 delta ≥ +3pp absolute (loosened from §F.1 ≥+5pp)
* 2. max Qwen-shape retrieval engagement delta ≥ +0.10 absolute above per-shape NULL baseline
* 3. (aggregate Tier 1 delta ≥ 0pp) AND (aggregate Tier 2 bonus ≥ 0.05)
*
* If ANY ONE passes → PROCEED (continue past Checkpoint B subject to PM ratify).
* If ALL THREE fail → HALT_INVESTIGATE (file Investigate report).
*
* Per Amendment 7 §3.2: "If Gen 1 fails ALL three thresholds → HALT before Gen 2,
* file Investigate report. If Gen 1 passes any one → proceed to Gen 2."
*/
export function computeDeltaFloorVerdict(inputs: DeltaFloorInputs): DeltaFloorVerdict {
// Threshold 1 — aggregate Tier 1 delta
const threshold1ValuePP =
(inputs.aggregateTrioStrictPassRateII - inputs.aggregateNullBaselinePassRateII) * 100;
const threshold1Pass =
threshold1ValuePP >= DELTA_FLOOR_THRESHOLDS.threshold1AggregateTier1PP - DELTA_FLOOR_EPSILON;
// Threshold 2 — max Qwen retrieval engagement delta absolute
let threshold2MaxDeltaAbsolute: number | null = null;
for (const [shapeKey, candidateMean] of Object.entries(inputs.qwenShapeRetrievalMeans)) {
if (candidateMean === undefined) continue;
const baselineMean = inputs.qwenShapeNullBaselineRetrievalMeans[shapeKey as ShapeName];
if (baselineMean === undefined) continue;
const delta = candidateMean - baselineMean;
if (threshold2MaxDeltaAbsolute === null || delta > threshold2MaxDeltaAbsolute) {
threshold2MaxDeltaAbsolute = delta;
}
}
// No Qwen data: report 0 delta (informative neutral); threshold2 fails since 0 < 0.10
if (threshold2MaxDeltaAbsolute === null) threshold2MaxDeltaAbsolute = 0;
const threshold2Pass =
threshold2MaxDeltaAbsolute >=
DELTA_FLOOR_THRESHOLDS.threshold2QwenRetrievalAbsolute - DELTA_FLOOR_EPSILON;
// Threshold 3 — compound Tier 1 ≥0 AND Tier 2 ≥0.05
const threshold3Tier1ValuePP = threshold1ValuePP;
const threshold3Tier1Pass =
threshold3Tier1ValuePP >= DELTA_FLOOR_THRESHOLDS.threshold3Tier1MinPP - DELTA_FLOOR_EPSILON;
const threshold3Tier2Pass =
inputs.qwenAggregateTier2Bonus >=
DELTA_FLOOR_THRESHOLDS.threshold3Tier2MinBonus - DELTA_FLOOR_EPSILON;
const threshold3Pass = threshold3Tier1Pass && threshold3Tier2Pass;
const overallVerdict: 'PROCEED' | 'HALT_INVESTIGATE' =
threshold1Pass || threshold2Pass || threshold3Pass ? 'PROCEED' : 'HALT_INVESTIGATE';
return {
threshold1AggregateTier1: threshold1Pass ? 'PASS' : 'FAIL',
threshold1ValuePP,
threshold2QwenRetrievalAbsolute: threshold2Pass ? 'PASS' : 'FAIL',
threshold2MaxDeltaAbsolute,
threshold3CompoundTier1PlusTier2: threshold3Pass ? 'PASS' : 'FAIL',
threshold3Tier1ValuePP,
threshold3Tier2Aggregate: inputs.qwenAggregateTier2Bonus,
overallVerdict,
};
}

View File

@@ -0,0 +1,24 @@
/**
* GEPA Faza 1 — public API surface.
*
* Per launch decision §G + manifest v7 §gepa.
*
* Module map:
* - types shared types + shape-class partitions
* - fitness per-shape fitness function (Amendment 2 §3 fork)
* - acceptance §F + §F.5 verdict per candidate
* - mutation-validator cell-semantic preservation audit (boundary SHAs)
* - kappa-audit drift band detection vs canonical 0.7878 ± 0.05
* - cost-tracker super-linear governance + halt triggers
* - selection top-1-per-shape + run-aggregate verdict
* - mutation-oracle-fork Qwen vs non-Qwen template routing (Amendment 2 §4)
*/
export * from './types.js';
export * from './fitness.js';
export * from './acceptance.js';
export * from './mutation-validator.js';
export * from './kappa-audit.js';
export * from './cost-tracker.js';
export * from './selection.js';
export * from './mutation-oracle-fork.js';

View File

@@ -0,0 +1,141 @@
/**
* GEPA Faza 1 — κ drift audit utility.
*
* Per launch decision §F.3 + manifest v7 §canonical_kappa_anchor +
* §faza_1_acceptance.condition_3.
*
* Canonical κ value: 0.7877758913412564 (rounds to 0.7878).
* Source: benchmarks/calibration/v6-kappa-recal/_summary-v6-kappa.json
* (SHA 657d4490bab28d35cf8a9c3ccea8a6b79e92835d700155184e51f3900836684c)
* Drift threshold: ±0.05 (drift band [0.7378, 0.8378])
*
* Per κ_recalibration_v6 success_criteria + monitoring_runtime in manifest v6 §5.4:
* - PASS: κ_trio ≥ 0.70
* - BORDERLINE: κ_trio ∈ [0.60, 0.70]
* - FAIL: κ_trio < 0.60
*
* Faza 1-specific drift acceptance (tighter than v6 baseline policy):
* - PASS: κ_trio ∈ [0.7378, 0.8378] (canonical ±0.05)
* - DRIFT_LOW: κ_trio < 0.7378
* - DRIFT_HIGH: κ_trio > 0.8378 (also a fail signal — judge ensemble drifted upward)
*/
/** Canonical κ baseline anchor pinned per manifest v7. */
export const CANONICAL_KAPPA = 0.7877758913412564;
/** Drift threshold (per brief §4 condition 3). */
export const KAPPA_DRIFT_THRESHOLD = 0.05;
/** Drift band lower bound (canonical − threshold). */
export const KAPPA_DRIFT_BAND_LOW = CANONICAL_KAPPA - KAPPA_DRIFT_THRESHOLD;
/** Drift band upper bound (canonical + threshold). */
export const KAPPA_DRIFT_BAND_HIGH = CANONICAL_KAPPA + KAPPA_DRIFT_THRESHOLD;
/** v6 policy floor for absolute κ pass (kept for cross-validation reporting). */
export const V6_KAPPA_POLICY_FLOOR_PASS = 0.70;
/** v6 borderline lower bound. */
export const V6_KAPPA_POLICY_FLOOR_BORDERLINE = 0.60;
export type KappaVerdict =
| 'PASS_WITHIN_DRIFT_BAND'
| 'DRIFT_LOW_BELOW_BAND'
| 'DRIFT_HIGH_ABOVE_BAND';
export interface KappaPairwise {
/** Cohen's κ between Opus and GPT verdict streams. */
kOpusGpt: number;
/** Cohen's κ between Opus and MiniMax verdict streams. */
kOpusMinimax: number;
/** Cohen's κ between GPT and MiniMax verdict streams. */
kGptMinimax: number;
}
export interface KappaAuditResult {
/** Conservative trio κ = min of three pairwise κ values (per manifest v6 §5.4). */
kConservativeTrio: number;
/** Pairwise components (carried through for audit log). */
pairwise: KappaPairwise;
/** Drift band verdict per Faza 1 §F.3 acceptance. */
verdict: KappaVerdict;
/** Absolute drift from canonical (positive = above, negative = below). */
driftFromCanonical: number;
/** Whether the κ_conservative_trio passes v6 policy floor (≥ 0.70). */
v6PolicyFloorPass: boolean;
/** Audit log line for inclusion in checkpoint reports. */
auditLogLine: string;
}
/** Compute κ audit verdict from three pairwise Cohen's κ values. */
export function auditKappa(pairwise: KappaPairwise): KappaAuditResult {
const kConservativeTrio = Math.min(
pairwise.kOpusGpt,
pairwise.kOpusMinimax,
pairwise.kGptMinimax,
);
const driftFromCanonical = kConservativeTrio - CANONICAL_KAPPA;
let verdict: KappaVerdict;
if (kConservativeTrio < KAPPA_DRIFT_BAND_LOW) {
verdict = 'DRIFT_LOW_BELOW_BAND';
} else if (kConservativeTrio > KAPPA_DRIFT_BAND_HIGH) {
verdict = 'DRIFT_HIGH_ABOVE_BAND';
} else {
verdict = 'PASS_WITHIN_DRIFT_BAND';
}
const v6PolicyFloorPass = kConservativeTrio >= V6_KAPPA_POLICY_FLOOR_PASS;
const auditLogLine = [
`κ_conservative_trio=${kConservativeTrio.toFixed(4)}`,
`canonical=${CANONICAL_KAPPA.toFixed(4)}`,
`drift=${driftFromCanonical >= 0 ? '+' : ''}${driftFromCanonical.toFixed(4)}`,
`verdict=${verdict}`,
`v6_policy_floor=${v6PolicyFloorPass ? 'PASS' : 'FAIL'}`,
].join(' | ');
return {
kConservativeTrio,
pairwise,
verdict,
driftFromCanonical,
v6PolicyFloorPass,
auditLogLine,
};
}
/**
* Compute Cohen's κ from a 2×2 confusion matrix.
*
* Convenience helper for callers that have raw verdict pair counts (the κ
* recalibration script computes these for Phase 1 baseline; tests and
* inline audits during Faza 1 reproduce the computation here).
*
* Returns NaN if total observations is zero (caller must handle).
*/
export function computeCohensKappa(confusion: {
bothCorrect: number;
bothIncorrect: number;
firstCorrectSecondIncorrect: number;
firstIncorrectSecondCorrect: number;
}): number {
const { bothCorrect, bothIncorrect, firstCorrectSecondIncorrect, firstIncorrectSecondCorrect } = confusion;
const total = bothCorrect + bothIncorrect + firstCorrectSecondIncorrect + firstIncorrectSecondCorrect;
if (total === 0) return NaN;
const observedAgreement = (bothCorrect + bothIncorrect) / total;
// Marginal probabilities for "correct" verdict per rater.
const firstCorrectMarginal = (bothCorrect + firstCorrectSecondIncorrect) / total;
const secondCorrectMarginal = (bothCorrect + firstIncorrectSecondCorrect) / total;
const expectedAgreement =
firstCorrectMarginal * secondCorrectMarginal +
(1 - firstCorrectMarginal) * (1 - secondCorrectMarginal);
if (expectedAgreement === 1) return 1.0; // perfect base rate, no variance → return κ=1 by convention
return (observedAgreement - expectedAgreement) / (1 - expectedAgreement);
}

View File

@@ -0,0 +1,85 @@
/**
* GEPA Faza 1 — mutation oracle fork (Qwen vs non-Qwen template routing).
*
* Per Amendment 2 §4 + manifest v7 §mutation_oracle_design.
*
* Phase 4.5 finding requires Qwen-specific scaffolding to address mechanistic
* under-engagement; uniform mutation guidance would not target the empirical
* gap. Forked oracle prompts ensure Qwen mutations explore the engagement-
* bonus reward landscape while non-Qwen mutations explore the broader scaffold
* space.
*
* This module is responsible only for:
* 1. Selecting the right template path per shape class
* 2. Loading template content from disk
* 3. Substituting baseline shape body + failure mode summary into template
*
* The actual LLM oracle call lives in the run orchestrator (out of scaffold scope).
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import { type ShapeName, QWEN_TARGETED_SHAPES } from './types.js';
export type TemplateClass = 'qwen' | 'non-qwen';
/** Per-shape template class routing. */
export function classifyShape(shape: ShapeName): TemplateClass {
return QWEN_TARGETED_SHAPES.has(shape) ? 'qwen' : 'non-qwen';
}
/** Resolve template path for a given shape class. */
export function templatePathForShape(
shape: ShapeName,
oracleDir: string = path.resolve(__dirname, '../../oracle/faza-1'),
): string {
const cls = classifyShape(shape);
const filename =
cls === 'qwen'
? 'mutation-prompt-template-qwen.md'
: 'mutation-prompt-template-non-qwen.md';
return path.join(oracleDir, filename);
}
/** Load raw template content for a given shape. */
export function loadTemplate(
shape: ShapeName,
oracleDir?: string,
): string {
const templatePath = templatePathForShape(shape, oracleDir);
return fs.readFileSync(templatePath, 'utf-8');
}
/** Inputs for assembling the final oracle prompt. */
export interface OraclePromptInputs {
shape: ShapeName;
/** Baseline shape file content (the candidate to mutate). */
baselineShapeContent: string;
/**
* Failure mode summary from Phase 4.3 verdict (top-3 T2 failures for this shape).
* Per brief §3.3 mutation oracle prompt.
*/
failureModeSummary: string;
oracleDir?: string;
}
/**
* Build the complete oracle prompt by template substitution.
*
* Templates use these placeholders (must appear verbatim in template files):
* ###BASELINE_SHAPE_CONTENT###
* ###FAILURE_MODE_SUMMARY###
* ###SHAPE_NAME###
* ###TEMPLATE_CLASS###
*/
export function buildOraclePrompt(inputs: OraclePromptInputs): string {
const template = loadTemplate(inputs.shape, inputs.oracleDir);
const cls = classifyShape(inputs.shape);
return template
.replace(/###BASELINE_SHAPE_CONTENT###/g, inputs.baselineShapeContent)
.replace(/###FAILURE_MODE_SUMMARY###/g, inputs.failureModeSummary)
.replace(/###SHAPE_NAME###/g, inputs.shape)
.replace(/###TEMPLATE_CLASS###/g, cls);
}

View File

@@ -0,0 +1,194 @@
/**
* GEPA Faza 1 — mutation validator (cell-semantic preservation audit).
*
* Per launch decision §A.4 (binding) + manifest v7 §gepa.mutation_validator.
*
* Boundary anchor: MULTI_STEP_ACTION_CONTRACT constant in
* packages/agent/src/prompt-shapes/types.ts.
* - whole-file SHA: 1a9fa329e4b66ed9f0abe8bc22cbbf0124e0c879e1e78ec806d557cab25bc94d
* - constant body bytes SHA: 70a1701dfa126f8dc1df9c116f0a8469da005821ecadc59d9b8f348568e755ba (252 bytes)
*
* Any GEPA candidate that produces non-zero diff against either anchor → INVALID.
*
* Allowed diff targets (per manifest v7 §gepa.mutation_validator.valid_diff_targets):
* - shape_file.systemPrompt method body (string-building only)
* - shape_file.soloUserPrompt method body
* - shape_file.multiStepKickoffUserPrompt method body
* - shape_file.retrievalInjectionUserPrompt method body
* - shape_file.metadata.evidence_link (MUST update to point to GEPA Gen 1 results)
*
* Invalid diff targets (LOCKED):
* - types.ts (entire file)
* - selector.ts (entire file)
* - index.ts (entire file)
* - shape_file.metadata.{description,modelClass,defaultThinking,defaultMaxTokens}
* - shape_file.imports
* - MULTI_STEP_ACTION_CONTRACT bytes
*/
import * as crypto from 'node:crypto';
import * as fs from 'node:fs';
/**
* Cell-semantic boundary SHAs pinned at substrate anchor commit c9bda3d.
* These are the byte-level invariants that any GEPA candidate must preserve.
*/
export const BOUNDARY_SHAS = {
/** Whole types.ts file SHA (must equal this exactly post-mutation). */
typesFile: '1a9fa329e4b66ed9f0abe8bc22cbbf0124e0c879e1e78ec806d557cab25bc94d',
/** MULTI_STEP_ACTION_CONTRACT constant body bytes SHA (252 bytes). */
multiStepActionContract: '70a1701dfa126f8dc1df9c116f0a8469da005821ecadc59d9b8f348568e755ba',
} as const;
/** Per-shape baseline SHAs pinned at substrate anchor commit c9bda3d. */
export const BASELINE_SHAPE_SHAS: Readonly<Record<string, string>> = {
'claude.ts': 'cbaf0c37b067b025a1fe97f2feeec11fae4070a8b3fcfaad1da8775dda451cc0',
'qwen-thinking.ts': '848a4e4917baa5c7bbcc3bb35fb8cb4b4ac8f0ab537243f14cbef3a99197aacb',
'qwen-non-thinking.ts': '35be379be9a8caafc2c419e32da5f63f92fc83f6f6d70d9df76029c1e8584572',
'gpt.ts': '5dc6d750d52a68feb9d37ad8384b2bcd59d70962066122ff086b0e5888413576',
'generic-simple.ts': '81189817f560e26a69394248d8bd9089cae72c7d40825323e2b7407e36026172',
} as const;
/** Validator verdict for a single candidate. */
export interface ValidatorVerdict {
valid: boolean;
/** List of violations (empty if valid). Each violation is a structured reason. */
violations: ValidationViolation[];
/** SHA of the candidate shape file (computed by validator). */
candidateShapeFileSha: string;
/** SHA of the cell-semantic types.ts file at validation time. */
typesFileSha: string;
/** SHA of the MULTI_STEP_ACTION_CONTRACT bytes at validation time. */
multiStepActionContractSha: string;
}
export interface ValidationViolation {
category:
| 'types_file_modified'
| 'multi_step_action_contract_modified'
| 'shape_file_unchanged_from_baseline'
| 'shape_file_metadata_locked_field_modified'
| 'shape_file_imports_modified';
severity: 'invalid';
detail: string;
}
/** Compute SHA-256 of a string's UTF-8 bytes. */
export function sha256(bytes: string | Buffer): string {
const buf = typeof bytes === 'string' ? Buffer.from(bytes, 'utf-8') : bytes;
return crypto.createHash('sha256').update(buf).digest('hex');
}
/** Compute SHA-256 of a file's bytes. */
export function sha256File(filepath: string): string {
return sha256(fs.readFileSync(filepath));
}
/**
* Extract the bytes of MULTI_STEP_ACTION_CONTRACT from a types.ts file content.
* Returns null if the constant is not found (which itself indicates a violation).
*/
export function extractMultiStepActionContractBytes(typesFileContent: string): string | null {
// Match the template literal body between backticks.
const match = typesFileContent.match(/export const MULTI_STEP_ACTION_CONTRACT = `([^`]+)`/);
return match ? match[1] : null;
}
/** Inputs for validating a GEPA candidate against cell-semantic boundaries. */
export interface ValidatorInputs {
/** Path to the candidate shape file (e.g., gepa-evolved/qwen-thinking-gen1-v0.ts). */
candidateShapeFilePath: string;
/** Shape file basename for baseline lookup (e.g., 'qwen-thinking.ts'). */
baselineShapeName: keyof typeof BASELINE_SHAPE_SHAS;
/** Path to the types.ts file (cell-semantic boundary anchor). */
typesFilePath: string;
/** Whether to require shape file to differ from baseline (true for Gen 1 mutations). */
expectShapeDiff: boolean;
}
/**
* Validate a GEPA candidate against the cell-semantic preservation invariants.
*
* Returns INVALID with structured violations if any boundary anchor is breached,
* or if a Gen 1 mutation produces zero diff vs baseline (violates the
* "every mutation must change something" implicit contract — Gen 0 baselines
* use expectShapeDiff=false).
*/
export function validateCandidate(inputs: ValidatorInputs): ValidatorVerdict {
const violations: ValidationViolation[] = [];
const typesFileSha = sha256File(inputs.typesFilePath);
const typesContent = fs.readFileSync(inputs.typesFilePath, 'utf-8');
const contractBytes = extractMultiStepActionContractBytes(typesContent);
const multiStepActionContractSha = contractBytes ? sha256(contractBytes) : '';
const candidateShapeFileSha = sha256File(inputs.candidateShapeFilePath);
if (typesFileSha !== BOUNDARY_SHAS.typesFile) {
violations.push({
category: 'types_file_modified',
severity: 'invalid',
detail: `types.ts SHA ${typesFileSha} != pinned baseline ${BOUNDARY_SHAS.typesFile}`,
});
}
if (multiStepActionContractSha !== BOUNDARY_SHAS.multiStepActionContract) {
violations.push({
category: 'multi_step_action_contract_modified',
severity: 'invalid',
detail:
contractBytes === null
? 'MULTI_STEP_ACTION_CONTRACT constant not found in types.ts'
: `MULTI_STEP_ACTION_CONTRACT bytes SHA ${multiStepActionContractSha} != pinned baseline ${BOUNDARY_SHAS.multiStepActionContract}`,
});
}
const baselineSha = BASELINE_SHAPE_SHAS[inputs.baselineShapeName];
if (inputs.expectShapeDiff && candidateShapeFileSha === baselineSha) {
violations.push({
category: 'shape_file_unchanged_from_baseline',
severity: 'invalid',
detail: `Gen 1 mutation expected to differ from baseline ${inputs.baselineShapeName} (SHA ${baselineSha}) but candidate produced identical bytes`,
});
}
// Optional shallow check: locked metadata fields must remain in the shape file.
// GEPA mutations are allowed to update evidence_link only.
const candidateContent = fs.readFileSync(inputs.candidateShapeFilePath, 'utf-8');
const lockedMetadataFields = [
'description:',
'modelClass:',
'defaultThinking:',
'defaultMaxTokens:',
];
for (const field of lockedMetadataFields) {
if (!candidateContent.includes(field)) {
violations.push({
category: 'shape_file_metadata_locked_field_modified',
severity: 'invalid',
detail: `Locked metadata field "${field}" missing from candidate shape file`,
});
}
}
// Check that imports block is still present (GEPA mutations cannot add/remove imports).
// Accept either './types.js' (same-dir candidate) or '../types.js' (gepa-evolved/ subdir
// candidate per manifest v7 §gepa.shape_scope.target_path) — the invariant is that
// candidates must import from types.js, not the specific relative path.
const importsTypes = candidateContent.includes("from './types.js'") ||
candidateContent.includes("from '../types.js'");
if (!importsTypes) {
violations.push({
category: 'shape_file_imports_modified',
severity: 'invalid',
detail: `Required import from "types.js" (either "./" or "../" relative) missing from candidate shape file`,
});
}
return {
valid: violations.length === 0,
violations,
candidateShapeFileSha,
typesFileSha,
multiStepActionContractSha,
};
}

View File

@@ -0,0 +1,128 @@
/**
* GEPA Faza 1 — top-1-per-shape selection.
*
* Per brief §2 selection metric ("best-per-shape-per-cell, ne aggregate") +
* launch decision §F + §F.5 acceptance.
*
* Selection algorithm:
* 1. For each shape, compute fitness for all candidates (per-shape fitness fork)
* 2. Per shape, select candidate with highest fitness as "best"
* 3. Apply acceptance verdict to each best-per-shape candidate
* 4. Report shape-level + run-aggregate verdict
*
* Run-aggregate §F conditions:
* §F.2: ≥3/5 shapes show positive trio_strict delta vs NULL-baseline
*
* Per-candidate §F.1 + §F.5 already evaluated by acceptance.evaluateCandidate.
*/
import { computeFitness } from './fitness.js';
import { evaluateCandidate } from './acceptance.js';
import {
type CandidateMetrics,
type FitnessComponents,
type AcceptanceVerdict,
type ShapeName,
} from './types.js';
/** Per-shape selection result. */
export interface ShapeSelectionResult {
shape: ShapeName;
bestCandidate: CandidateMetrics;
bestFitness: FitnessComponents;
acceptance: AcceptanceVerdict;
/** All candidates evaluated for this shape (for audit log). */
allCandidatesRanked: Array<{ candidate: CandidateMetrics; fitness: FitnessComponents }>;
}
/** Aggregate run verdict per launch decision §F.2. */
export interface RunAggregateVerdict {
/** Number of shapes with positive trio_strict delta vs NULL-baseline. */
shapesWithPositiveDelta: number;
/** Total shapes evaluated. */
totalShapes: number;
/** §F.2 condition: ≥3/5 shapes show positive delta. */
condition2Pass: boolean;
/** Number of shapes where best candidate was ACCEPTED (passes §F.1 + not §F.5). */
shapesAccepted: number;
}
/** Full selection report — per shape + run aggregate. */
export interface SelectionReport {
perShape: ShapeSelectionResult[];
runAggregate: RunAggregateVerdict;
}
/** Inputs: per-shape candidates + per-shape NULL-baseline metrics + cost baselines. */
export interface SelectionInputs {
/** All candidates grouped by shape. */
candidatesPerShape: Map<ShapeName, CandidateMetrics[]>;
/** NULL-baseline trio_strict_pass_rate (op. ii) per shape. */
baselineTrioStrictPassRateII: Map<ShapeName, number>;
/** NULL-baseline median cost (USD per evaluation) per shape. */
baselineMedianCostUsd: Map<ShapeName, number>;
}
/**
* Run top-1-per-shape selection + apply acceptance verdicts.
*
* Returns a complete selection report including per-shape results and the
* run-aggregate verdict.
*
* Throws if a shape in candidatesPerShape lacks a corresponding baseline entry.
*/
export function runSelection(inputs: SelectionInputs): SelectionReport {
const perShape: ShapeSelectionResult[] = [];
for (const [shape, candidates] of inputs.candidatesPerShape.entries()) {
if (candidates.length === 0) {
continue; // skip shapes with no candidates
}
const baselineRate = inputs.baselineTrioStrictPassRateII.get(shape);
if (baselineRate === undefined) {
throw new Error(`runSelection: missing baseline trio_strict rate for shape "${shape}"`);
}
const baselineCost = inputs.baselineMedianCostUsd.get(shape);
if (baselineCost === undefined) {
throw new Error(`runSelection: missing baseline median cost for shape "${shape}"`);
}
// Compute fitness for all candidates of this shape
const ranked = candidates
.map(candidate => ({
candidate,
fitness: computeFitness({ candidate, baselineMedianCostUsd: baselineCost }),
}))
.sort((a, b) => b.fitness.fitness - a.fitness.fitness);
const top = ranked[0];
const acceptance = evaluateCandidate({
candidate: top.candidate,
baselineTrioStrictPassRateII: baselineRate,
});
perShape.push({
shape,
bestCandidate: top.candidate,
bestFitness: top.fitness,
acceptance,
allCandidatesRanked: ranked,
});
}
const shapesWithPositiveDelta = perShape.filter(s => s.acceptance.trioStrictDeltaPP > 0).length;
const shapesAccepted = perShape.filter(s => s.acceptance.accepted).length;
const totalShapes = perShape.length;
const condition2Pass = shapesWithPositiveDelta >= 3;
return {
perShape,
runAggregate: {
shapesWithPositiveDelta,
totalShapes,
condition2Pass,
shapesAccepted,
},
};
}

View File

@@ -0,0 +1,321 @@
/**
* GEPA Faza 1 — shared type definitions.
*
* Per manifest v7 (SHA 583712dde139ffc87fb1ab21643f68d52c56469ded9e8090a624980b05969beb,
* Amendment 2 supplemented) and launch decision §A inherited rules.
*
* Cell semantic boundary linchpin: MULTI_STEP_ACTION_CONTRACT in
* packages/agent/src/prompt-shapes/types.ts (SHA 70a1701d...).
*
* DO NOT modify this file as part of GEPA candidate evolution. This file is
* scaffold-only; the mutation surface is the prompt-shape body methods in
* packages/agent/src/prompt-shapes/{claude,qwen-thinking,qwen-non-thinking,gpt,generic-simple}.ts.
*/
/** The 5 shape names targeted by Faza 1 GEPA evolution. */
export type ShapeName =
| 'claude'
| 'qwen-thinking'
| 'qwen-non-thinking'
| 'gpt'
| 'generic-simple';
/**
* Shapes that get retrieval-engagement weighting per Amendment 2 §3.
* Phase 4.5 finding: only Qwen exhibits the under-engagement gap.
*/
export const QWEN_TARGETED_SHAPES: ReadonlySet<ShapeName> = new Set<ShapeName>([
'qwen-thinking',
'qwen-non-thinking',
]);
/**
* Shapes that use baseline fitness (no retrieval-engagement weight).
* Per Amendment 2 §3: these shapes don't have the gap, so applying the
* bonus uniformly would distort their fitness measurement.
*/
export const NON_QWEN_SHAPES: ReadonlySet<ShapeName> = new Set<ShapeName>([
'claude',
'gpt',
'generic-simple',
]);
/** Per-task evaluation result for one candidate × one instance. */
export interface EvaluationResult {
/** Stable instance identifier (e.g., from corpus generator). */
instanceId: string;
/** Trio judge mean (per pilot runner line 654 — arithmetic mean of valid judge means). */
trioMean: number;
/**
* Trio strict pass per metric_operationalization (ii) — primary acceptance.
* `trioMean >= 4.0` per Amendment 1 Ask B ratification.
*/
trioStrictPassII: boolean;
/**
* Trio strict pass per metric_operationalization (i) — supplementary.
* `>= 2 of 3 judges with judge.mean >= 3.5` per pilot runner line 657.
* Reported in parallel for cross-validation against pilot baseline.
*/
trioStrictPassI: boolean;
/** Number of retrieve actions issued during this evaluation (existing telemetry). */
retrievalCalls: number;
/** Subject + judge cumulative cost (USD) for this evaluation. */
costUsd: number;
}
/** Aggregated per-candidate metrics across N=8 evaluations (per shape). */
export interface CandidateMetrics {
/** Stable candidate identifier (e.g., shape name + generation + variant). */
candidateId: string;
/** Which shape this candidate belongs to. */
shape: ShapeName;
/** All N=8 evaluation results for this candidate. */
evaluations: EvaluationResult[];
/** Pass rate per operationalization (ii) — primary. Range [0, 1]. */
trioStrictPassRateII: number;
/** Pass rate per operationalization (i) — supplementary. Range [0, 1]. */
trioStrictPassRateI: number;
/** Mean retrieval_calls across evaluations (per task). */
meanRetrievalCallsPerTask: number;
/** Mean cost (USD) across evaluations. */
meanCostUsd: number;
}
/**
* Per-shape fitness components per Amendment 2 §3.
*
* For Qwen-targeted shapes:
* fitness = trio_strict_pass_rate + retrieval_engagement_bonus - cost_penalty
*
* For non-Qwen shapes:
* fitness = trio_strict_pass_rate - cost_penalty
* (retrievalEngagementBonus is always 0 for non-Qwen — not added to fitness)
*/
export interface FitnessComponents {
/** trio_strict_pass_rate per operationalization (ii). Range [0, 1]. */
trioStrictPassRateII: number;
/**
* Retrieval engagement bonus per Amendment 2 §3 bands:
* +0.05 if mean retrieval_calls per task >= 2.0
* 0.00 if mean retrieval_calls per task in [1.5, 2.0)
* -0.05 if mean retrieval_calls per task < 1.5
* Always 0.0 for non-Qwen shapes (these shapes don't have the gap).
*/
retrievalEngagementBonus: number;
/**
* Cost penalty per brief §3.1 — −0.5pp per $0.10 above per-shape baseline median.
* Encoded as decimal (e.g., 0.005 = 0.5pp).
*/
costPenalty: number;
/** Aggregate fitness (sum of above with sign convention: bonus +, penalty −). */
fitness: number;
/**
* Whether retrieval engagement was applied (true for Qwen-targeted shapes,
* false otherwise). Useful for downstream auditing + report generation.
*/
retrievalEngagementApplied: boolean;
}
/** Inputs for the per-shape fitness function. */
export interface FitnessInputs {
/** Aggregated metrics for this candidate. */
candidate: CandidateMetrics;
/**
* Per-shape baseline median cost (USD per evaluation). Used for cost penalty
* computation. Typically the NULL-baseline median for the same shape.
*/
baselineMedianCostUsd: number;
}
/** Acceptance verdict for a single candidate per §F + §F.5 of launch decision. */
export interface AcceptanceVerdict {
/** Whether candidate passes §F condition 1 (trio_strict delta + Qwen retrieval floor). */
condition1Pass: boolean;
/**
* §F.5 false-positive guard — REJECTED if Qwen candidate has +5pp trio delta
* but mean retrieval_calls < 1.5 (Amendment 2 §5).
*/
condition5FalsePositiveGuardTriggered: boolean;
/** Overall acceptance for this candidate (must pass condition 1 AND not trigger §F.5). */
accepted: boolean;
/** Detailed reason string for audit log. */
reason: string;
/**
* Computed delta vs NULL-baseline trio_strict_pass_rate (percentage points).
* Positive = improvement.
*/
trioStrictDeltaPP: number;
}
/** Inputs for the acceptance validator. */
export interface AcceptanceInputs {
/** Candidate under evaluation. */
candidate: CandidateMetrics;
/** NULL-baseline trio_strict_pass_rate (op. (ii)) for the same shape. */
baselineTrioStrictPassRateII: number;
}
// ── Amendment 7 — tiered fitness + Δ-floor types ───────────────────────────
/**
* Per-shape NULL-baseline anchors pinned from Checkpoint A v2 §B.2 (manifest
* v7 Amendment 6 binding SHA 0b55d8e353...).
*
* These anchor:
* - Tier 1 baseline (NULL pass rate per shape)
* - Tier 2 baseline (NULL retrieval engagement per shape)
* - §F.1 acceptance gate ≥+5pp delta basis
* - Δ-floor threshold 2 per-shape comparison
*
* Source: real per-shape data from re-run NULL-baseline post Amendment 6
* promptShapeOverride bug fix (run bhe0zwi91, 40/40 evals, 2026-04-28).
*/
export const NULL_BASELINE_PER_SHAPE: Readonly<
Record<ShapeName, { trioStrictPassRateII: number; meanRetrievalCallsPerTask: number }>
> = {
claude: { trioStrictPassRateII: 0.875, meanRetrievalCallsPerTask: 1.12 },
'qwen-thinking': { trioStrictPassRateII: 0.875, meanRetrievalCallsPerTask: 1.12 },
'qwen-non-thinking': { trioStrictPassRateII: 1.000, meanRetrievalCallsPerTask: 1.25 },
gpt: { trioStrictPassRateII: 0.750, meanRetrievalCallsPerTask: 1.00 },
'generic-simple': { trioStrictPassRateII: 0.875, meanRetrievalCallsPerTask: 1.12 },
} as const;
/** NULL-baseline aggregate across all 5 shapes (Checkpoint A v2 §B.2). */
export const NULL_BASELINE_AGGREGATE = {
trioStrictPassRateII: 0.875, // 35/40
meanRetrievalCallsPerTask: 1.12,
} as const;
/**
* Tiered fitness components per Amendment 7 §fitness_function_tiered.
*
* Saturated regime (NULL pass rate ≥75% for ≥4/5 shapes — current Faza 1 state)
* makes Tier 1 (NULL delta) noise-bound on N=8 binomial. Tier 2 + Tier 3 act as
* primary differentiators; Tier 1 retained as TIE-BREAKER + acceptance gate.
*/
export interface TieredFitnessComponents {
/**
* Tier 1 — NULL pass rate delta (signed, percentage points).
* Acceptance gate threshold: ≥+5pp (launch decision §F.1, UNCHANGED).
* Role in saturated regime: TIE_BREAKER (noise-bound at N=8).
*/
tier1DeltaPP: number;
/**
* Tier 2 — continuous retrieval engagement bonus (Qwen-targeted only).
* Formula: clamp(0.05 × delta_pp, 0, 0.25) where delta_pp = (candidate − baseline) × 100.
* Always 0 for non-Qwen shapes.
* Role in saturated regime: PRIMARY differentiator for Qwen-targeted shapes.
*/
tier2RetrievalBonus: number;
/**
* Tier 3 — binary cell-semantic anchor invariance bonus.
* 0.10 if all 7 anchors invariant (mutation-validator passed); 0 otherwise.
* Role in saturated regime: SECONDARY differentiator (substrate-preservation proxy).
*/
tier3CellSemanticInvarianceBonus: number;
/**
* Aggregate fitness in saturated regime: tier_2 + tier_3.
* Tier 1 reserved as tie-breaker (NOT in this aggregate).
* Cost penalty per Amendment 2 NOT in tiered ranking aggregate (supplementary diagnostic).
*/
aggregateSaturatedRegime: number;
/** Whether saturated regime applies (per Amendment 7 §fitness_function_tiered.saturated_regime_definition). */
saturatedRegimeApplied: boolean;
/** Cell-semantic anchor invariance count (0..7) for audit reporting. */
cellSemanticAnchorInvarianceCount: number;
}
/** Inputs for tiered fitness. */
export interface TieredFitnessInputs {
/** Aggregated metrics for this candidate. */
candidate: CandidateMetrics;
/** Per-shape NULL-baseline pass rate (op. ii). Typically NULL_BASELINE_PER_SHAPE[shape].trioStrictPassRateII. */
nullBaselinePassRateII: number;
/** Per-shape NULL-baseline mean retrieval calls per task. Typically NULL_BASELINE_PER_SHAPE[shape].meanRetrievalCallsPerTask. */
nullBaselineMeanRetrievalCallsPerTask: number;
/** Whether the candidate's mutation-validator verdict was VALID (all 7 anchors invariant). */
mutationValidatorPassed: boolean;
/**
* Whether the saturated regime applies (≥4/5 shapes have NULL pass rate ≥75%).
* Caller computes from Checkpoint A v2 data; default true for Faza 1.
*/
saturatedRegime: boolean;
}
/**
* Δ-floor verdict per Amendment 7 §gen_1_pre_registered_delta_floor.
* Pre-registered Gen 1 floor for "evolution worked at all" — looser than
* §F.1 acceptance gate. Three OR-gated thresholds.
*/
export interface DeltaFloorVerdict {
/** Threshold 1: aggregate Tier 1 delta ≥+3pp. */
threshold1AggregateTier1: 'PASS' | 'FAIL';
/** Aggregate Tier 1 delta value (pp, signed) — for audit. */
threshold1ValuePP: number;
/** Threshold 2: max Qwen-shape retrieval engagement delta ≥+0.10 absolute. */
threshold2QwenRetrievalAbsolute: 'PASS' | 'FAIL';
/** Max delta across Qwen shapes (absolute) — for audit. */
threshold2MaxDeltaAbsolute: number;
/** Threshold 3: (Tier 1 ≥0pp) AND (Tier 2 ≥0.05). */
threshold3CompoundTier1PlusTier2: 'PASS' | 'FAIL';
/** Aggregate Tier 1 delta (pp, signed) — for audit. */
threshold3Tier1ValuePP: number;
/** Aggregate Tier 2 bonus across Qwen-targeted candidates — for audit. */
threshold3Tier2Aggregate: number;
/** Overall verdict: PROCEED if ANY threshold passes; HALT_INVESTIGATE if ALL fail. */
overallVerdict: 'PROCEED' | 'HALT_INVESTIGATE';
}
/** Inputs for Δ-floor verdict computation. */
export interface DeltaFloorInputs {
/** Aggregate trio_strict_pass_rate_II across all evaluated candidates × all evals (in saturated regime, mean across 30 Checkpoint B evals). */
aggregateTrioStrictPassRateII: number;
/** Aggregate NULL-baseline pass rate (e.g., 0.875 = NULL_BASELINE_AGGREGATE.trioStrictPassRateII). */
aggregateNullBaselinePassRateII: number;
/**
* Per-shape mean retrieval calls per task for Qwen-targeted candidates.
* Map: shape → meanRetrievalCallsPerTask. Empty entries treated as no-data (skipped).
*/
qwenShapeRetrievalMeans: Partial<Record<ShapeName, number>>;
/** Per-shape NULL baseline retrieval means (typically NULL_BASELINE_PER_SHAPE[shape].meanRetrievalCallsPerTask). */
qwenShapeNullBaselineRetrievalMeans: Partial<Record<ShapeName, number>>;
/** Aggregate Tier 2 retrieval engagement bonus across Qwen-targeted candidates (mean). */
qwenAggregateTier2Bonus: number;
}