This commit is contained in:
154
benchmarks/gepa/src/faza-1/acceptance.ts
Normal file
154
benchmarks/gepa/src/faza-1/acceptance.ts
Normal file
@@ -0,0 +1,154 @@
|
||||
/**
|
||||
* GEPA Faza 1 — acceptance validator.
|
||||
*
|
||||
* Per launch decision §F (4 must-hold conditions) + §F.5 (Amendment 2 §5
|
||||
* false-positive evolution guard).
|
||||
*
|
||||
* Per-candidate verdict logic:
|
||||
*
|
||||
* §F condition 1 (third update — Amendment 2 §5):
|
||||
* "Best GEPA candidate per shape beats NULL-baseline by ≥+5pp on
|
||||
* trio_strict_pass rate (trio_mean ≥ 4.0). For Qwen-targeted shapes,
|
||||
* additionally: best candidate must have mean retrieval_calls per task
|
||||
* ≥ 1.7 (engagement gap closed by ≥50% relative to Qwen baseline 1.33)."
|
||||
*
|
||||
* §F.5 (NEW per Amendment 2 — false-positive guard):
|
||||
* "If best Qwen-shape candidate achieves +5pp trio_strict delta WITHOUT
|
||||
* closing retrieval engagement gap (mean retrieval_calls < 1.5), this
|
||||
* signals false-positive evolution. Result: candidate REJECTED, shape
|
||||
* marked FAIL even if other criteria pass."
|
||||
*
|
||||
* Conditions §F.2/§F.3/§F.4 are evaluated at the run-aggregate level (across
|
||||
* shapes / κ stability across all evaluations / mutation validator log) and
|
||||
* are NOT checked per-candidate here. See selection.ts + run-orchestrator
|
||||
* for those.
|
||||
*/
|
||||
|
||||
import {
|
||||
type AcceptanceInputs,
|
||||
type AcceptanceVerdict,
|
||||
QWEN_TARGETED_SHAPES,
|
||||
} from './types.js';
|
||||
|
||||
/** §F condition 1 trio_strict delta threshold (percentage points). */
|
||||
export const TRIO_STRICT_DELTA_THRESHOLD_PP = 5;
|
||||
|
||||
/**
|
||||
* §F condition 1 Qwen-only retrieval engagement floor (mean retrieval_calls
|
||||
* per task). 1.7 = 50% gap closure between Qwen baseline 1.33 and Opus
|
||||
* parity 2.33 per Amendment 2 §5.
|
||||
*/
|
||||
export const QWEN_RETRIEVAL_ENGAGEMENT_FLOOR = 1.7;
|
||||
|
||||
/**
|
||||
* §F.5 false-positive guard threshold (mean retrieval_calls per task).
|
||||
* If a Qwen candidate achieves trio_strict delta but stays below this floor,
|
||||
* it is REJECTED as false-positive evolution per Amendment 2 §5.
|
||||
*/
|
||||
export const QWEN_FALSE_POSITIVE_RETRIEVAL_FLOOR = 1.5;
|
||||
|
||||
/**
|
||||
* Floating-point tolerance for threshold comparisons. 1e-9 is well below
|
||||
* any signal magnitude in the +/-0.05 fitness band (which is itself ~9
|
||||
* orders of magnitude larger). Required because IEEE 754 makes
|
||||
* `(0.25 - 0.20) * 100 = 4.999999999999999` rather than exact 5.0.
|
||||
*/
|
||||
const EPSILON = 1e-9;
|
||||
|
||||
/**
|
||||
* Compute acceptance verdict for a single candidate per launch decision §F + §F.5.
|
||||
*
|
||||
* Caller is responsible for §F.2 (≥3/5 shapes positive delta), §F.3 (κ stability),
|
||||
* and §F.4 (zero cell semantic violations) at the run-aggregate level.
|
||||
*/
|
||||
export function evaluateCandidate(inputs: AcceptanceInputs): AcceptanceVerdict {
|
||||
const { candidate, baselineTrioStrictPassRateII } = inputs;
|
||||
|
||||
// Compute delta in percentage points (scale 0..100)
|
||||
const trioStrictDeltaPP =
|
||||
(candidate.trioStrictPassRateII - baselineTrioStrictPassRateII) * 100;
|
||||
|
||||
const isQwenTargeted = QWEN_TARGETED_SHAPES.has(candidate.shape);
|
||||
|
||||
// §F.5 false-positive guard — applies ONLY to Qwen-targeted shapes
|
||||
// and ONLY when trio_strict delta meets the +5pp threshold.
|
||||
// Per Amendment 2 §5: if delta ≥ +5pp AND retrieval_calls < 1.5 → REJECT.
|
||||
// EPSILON tolerance handles IEEE 754 precision on exact-boundary deltas.
|
||||
let condition5FalsePositiveGuardTriggered = false;
|
||||
if (
|
||||
isQwenTargeted &&
|
||||
trioStrictDeltaPP >= TRIO_STRICT_DELTA_THRESHOLD_PP - EPSILON &&
|
||||
candidate.meanRetrievalCallsPerTask < QWEN_FALSE_POSITIVE_RETRIEVAL_FLOOR - EPSILON
|
||||
) {
|
||||
condition5FalsePositiveGuardTriggered = true;
|
||||
}
|
||||
|
||||
// §F condition 1 — trio_strict delta + (Qwen only) retrieval engagement floor
|
||||
let condition1Pass = trioStrictDeltaPP >= TRIO_STRICT_DELTA_THRESHOLD_PP - EPSILON;
|
||||
if (condition1Pass && isQwenTargeted) {
|
||||
// Qwen sub-criterion: mean retrieval_calls ≥ 1.7
|
||||
if (candidate.meanRetrievalCallsPerTask < QWEN_RETRIEVAL_ENGAGEMENT_FLOOR - EPSILON) {
|
||||
condition1Pass = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Overall acceptance: condition 1 must pass AND §F.5 must not trigger
|
||||
const accepted = condition1Pass && !condition5FalsePositiveGuardTriggered;
|
||||
|
||||
// Reason string for audit log
|
||||
const reason = buildReason({
|
||||
candidate,
|
||||
trioStrictDeltaPP,
|
||||
condition1Pass,
|
||||
condition5FalsePositiveGuardTriggered,
|
||||
isQwenTargeted,
|
||||
accepted,
|
||||
});
|
||||
|
||||
return {
|
||||
condition1Pass,
|
||||
condition5FalsePositiveGuardTriggered,
|
||||
accepted,
|
||||
reason,
|
||||
trioStrictDeltaPP,
|
||||
};
|
||||
}
|
||||
|
||||
interface BuildReasonInputs {
|
||||
candidate: { shape: string; meanRetrievalCallsPerTask: number };
|
||||
trioStrictDeltaPP: number;
|
||||
condition1Pass: boolean;
|
||||
condition5FalsePositiveGuardTriggered: boolean;
|
||||
isQwenTargeted: boolean;
|
||||
accepted: boolean;
|
||||
}
|
||||
|
||||
function buildReason(i: BuildReasonInputs): string {
|
||||
const parts: string[] = [];
|
||||
parts.push(`shape=${i.candidate.shape}`);
|
||||
parts.push(`trio_strict_delta=${i.trioStrictDeltaPP.toFixed(2)}pp`);
|
||||
parts.push(`mean_retrieval_calls=${i.candidate.meanRetrievalCallsPerTask.toFixed(2)}`);
|
||||
|
||||
if (i.condition5FalsePositiveGuardTriggered) {
|
||||
parts.push(
|
||||
`REJECTED §F.5 false-positive guard: delta ≥ +${TRIO_STRICT_DELTA_THRESHOLD_PP}pp AND retrieval < ${QWEN_FALSE_POSITIVE_RETRIEVAL_FLOOR}`,
|
||||
);
|
||||
} else if (!i.condition1Pass) {
|
||||
// Use same EPSILON tolerance as condition1Pass evaluation to keep root-cause
|
||||
// attribution consistent with the gating logic on exact-boundary deltas.
|
||||
if (i.trioStrictDeltaPP < TRIO_STRICT_DELTA_THRESHOLD_PP - EPSILON) {
|
||||
parts.push(
|
||||
`FAIL §F.1 trio_strict delta: ${i.trioStrictDeltaPP.toFixed(2)}pp < ${TRIO_STRICT_DELTA_THRESHOLD_PP}pp`,
|
||||
);
|
||||
} else if (i.isQwenTargeted) {
|
||||
parts.push(
|
||||
`FAIL §F.1 Qwen retrieval floor: ${i.candidate.meanRetrievalCallsPerTask.toFixed(2)} < ${QWEN_RETRIEVAL_ENGAGEMENT_FLOOR}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
parts.push(`PASS §F.1${i.isQwenTargeted ? ' (Qwen retrieval floor met)' : ''}`);
|
||||
}
|
||||
|
||||
parts.push(`accepted=${i.accepted}`);
|
||||
return parts.join(' | ');
|
||||
}
|
||||
114
benchmarks/gepa/src/faza-1/corpus-prompt.ts
Normal file
114
benchmarks/gepa/src/faza-1/corpus-prompt.ts
Normal file
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* GEPA Faza 1 — Opus 4.7 oracle prompt for synthesizing one H3 corpus instance.
|
||||
*
|
||||
* Per Amendment 1 Ask A Option C sub-ask 4 (instance generation methodology):
|
||||
* "Opus 4.7 generates task scaffold (persona + scenario + 6-7 source document specs);
|
||||
* PM does NOT review each instance pre-NULL-baseline. Instead: CC-2 spot-audits
|
||||
* 5 random instances pre NULL-baseline kick (3% sample)."
|
||||
*
|
||||
* The oracle is instructed to produce a JSON object that maps directly onto
|
||||
* the CorpusInstance shape (minus computed fields like materialsConcat, charCount).
|
||||
*/
|
||||
|
||||
import { type StratificationCell, type CorpusInstance, TASK_FAMILY_DESCRIPTORS } from './corpus.js';
|
||||
|
||||
/** Persona descriptor — short label + role context for prompt template. */
|
||||
const PERSONA_DESCRIPTORS: Record<string, { label: string; roleContext: string }> = {
|
||||
p1_founder_ceo: {
|
||||
label: 'Founder/CEO',
|
||||
roleContext: 'company founder + chief executive; focused on growth, fundraising, vision',
|
||||
},
|
||||
p2_cfo: {
|
||||
label: 'CFO',
|
||||
roleContext: 'chief financial officer; focused on burn, runway, capital efficiency',
|
||||
},
|
||||
p3_coo: {
|
||||
label: 'COO',
|
||||
roleContext: 'chief operating officer; focused on operational efficiency, hiring, process',
|
||||
},
|
||||
p4_vp_finance: {
|
||||
label: 'VP Finance',
|
||||
roleContext: 'reports to CFO; focused on FP&A, financial reporting, budget management',
|
||||
},
|
||||
p5_independent_director: {
|
||||
label: 'Independent Director',
|
||||
roleContext: 'board-level oversight; focused on governance, strategic risk, executive accountability',
|
||||
},
|
||||
};
|
||||
|
||||
/** Company-stage descriptor — narrative framing for Opus to ground each instance. */
|
||||
const COMPANY_STAGE_DESCRIPTORS: Record<string, { label: string; financial: string; pressure: string }> = {
|
||||
stage_a_series_b_growth_burning: {
|
||||
label: 'Series B growth-stage, burning capital',
|
||||
financial: 'recently raised Series B ~$30M; burning $1-1.5M/month; 12-18 months runway; ARR $10-20M',
|
||||
pressure: 'investor pressure to demonstrate capital efficiency; competitive displacement risk; talent retention concerns',
|
||||
},
|
||||
stage_b_post_profitable_consolidation: {
|
||||
label: 'Post-profitable, consolidation phase',
|
||||
financial: 'reached profitability 2-4 quarters ago; $40-80M ARR; 15-25% operating margin; 24+ months runway',
|
||||
pressure: 'Wall Street scrutiny on growth deceleration; M&A integration challenges; complacency risk',
|
||||
},
|
||||
};
|
||||
|
||||
export interface BuildCorpusInstancePromptInputs {
|
||||
cell: StratificationCell;
|
||||
instanceId: string;
|
||||
/** Domain anchor — ensures all 50 instances stay in NorthLane CFO synthesis territory. */
|
||||
domainAnchor?: string;
|
||||
}
|
||||
|
||||
/** Build the Opus 4.7 generation prompt for one stratification cell. */
|
||||
export function buildCorpusInstancePrompt(inputs: BuildCorpusInstancePromptInputs): string {
|
||||
const { cell, instanceId } = inputs;
|
||||
const familyDesc = TASK_FAMILY_DESCRIPTORS[cell.family];
|
||||
const personaDesc = PERSONA_DESCRIPTORS[cell.persona];
|
||||
const stageDesc = COMPANY_STAGE_DESCRIPTORS[cell.stage];
|
||||
|
||||
return `You are generating one synthesis-task instance for a benchmark corpus.
|
||||
|
||||
The corpus targets B2B SaaS knowledge work in the spirit of "NorthLane CFO" pilot tasks: a complex business situation requiring multi-document synthesis to produce a structured business deliverable (memo, action plan, decision recommendation).
|
||||
|
||||
## Stratification cell for this instance
|
||||
- Instance ID: \`${instanceId}\`
|
||||
- Task family: \`${cell.family}\` — ${familyDesc.label}
|
||||
- Task family description: ${familyDesc.label.replace(/_/g, ' ')}; mirror pattern: ${familyDesc.mirrorPilotTask ?? 'NEW family (no pilot mirror)'}
|
||||
- Persona: \`${cell.persona}\` — ${personaDesc.label} (${personaDesc.roleContext})
|
||||
- Company stage: \`${cell.stage}\` — ${stageDesc.label}
|
||||
- Financial state: ${stageDesc.financial}
|
||||
- Pressure dynamics: ${stageDesc.pressure}
|
||||
|
||||
## Required output (JSON object only, no prose, no code fences)
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"personaText": "Persona: <2-3 sentence persona description grounding the model in role + company>\\n\\nScenario: <2-4 sentence scenario hook setting up why this synthesis is needed now>",
|
||||
"sourceDocuments": [
|
||||
{ "title": "DOC 1 — <Doc name>", "body": "<markdown content, 600-1200 chars>" },
|
||||
{ "title": "DOC 2 — <Doc name>", "body": "<...>" },
|
||||
...
|
||||
(exactly ${familyDesc.docsPerInstance} docs total)
|
||||
],
|
||||
"question": "<open-ended question, 100-400 chars, that requires synthesizing across multiple docs to answer>"
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
## Generation constraints (binding)
|
||||
|
||||
1. **Domain anchor:** All ${familyDesc.docsPerInstance} source documents must be plausible artifacts a ${personaDesc.label} would receive in a B2B SaaS company at the ${stageDesc.label} phase.
|
||||
|
||||
2. **Document diversity:** Each doc should be a different artifact type (P&L summary, pipeline review, churn analysis, eng velocity report, marketing dashboard, board notes, competitor intel, customer interview notes, OKR report, runway model, etc.). No two docs should be the same artifact type.
|
||||
|
||||
3. **Synthesis requirement:** The question must be answerable ONLY by triangulating across multiple docs. A single-doc answer should be insufficient. Specifically, the question should require:
|
||||
- Identifying tradeoffs across docs
|
||||
- Synthesizing competing positions
|
||||
- Producing a structured deliverable (memo, action plan, or recommendation with justification)
|
||||
|
||||
4. **Persona consistency:** The persona's role + company-stage pressures must be evident in BOTH the scenario framing AND the question phrasing. A founder-CEO question reads differently than an independent-director question.
|
||||
|
||||
5. **Doc body realism:** Use realistic numbers (revenue figures, percentages, dates). Cite specific people by role title (e.g., "VP Sales", not "John Smith"). Include both hard data + qualitative commentary in each doc.
|
||||
|
||||
6. **JSON format:** Output ONLY a single valid JSON object. No prose before or after. No code fence markers. The first character must be \`{\` and the last character must be \`}\`.
|
||||
|
||||
## Output begins below
|
||||
`;
|
||||
}
|
||||
311
benchmarks/gepa/src/faza-1/corpus.ts
Normal file
311
benchmarks/gepa/src/faza-1/corpus.ts
Normal file
@@ -0,0 +1,311 @@
|
||||
/**
|
||||
* GEPA Faza 1 — H3 corpus data definitions, stratification, validation,
|
||||
* and spot-audit sampler.
|
||||
*
|
||||
* Per Amendment 1 Ask A Option C + manifest v7 §corpus_design:
|
||||
* - 50 instances total
|
||||
* - 5 task families × 5 personas × 2 company stages = 50 cells
|
||||
* - NorthLane CFO synthesis domain (preserves Phase 4.3 anchor)
|
||||
* - Each instance: ≥6 source documents, 6-dim Likert rubric
|
||||
* - Stratified sampling: deterministic, seed=42
|
||||
* - Generated via Opus 4.7 oracle
|
||||
*
|
||||
* Pre-A halt-and-PM checkpoint: spot-audit 5 random instances.
|
||||
*/
|
||||
|
||||
import * as crypto from 'node:crypto';
|
||||
|
||||
/** 5 task families per manifest v7 §corpus_design.task_families. */
|
||||
export type TaskFamily = 'F1' | 'F2' | 'F3' | 'F4' | 'F5';
|
||||
|
||||
/** 5 persona codes per manifest v7 §corpus_design.persona_axis. */
|
||||
export type PersonaCode = 'p1_founder_ceo' | 'p2_cfo' | 'p3_coo' | 'p4_vp_finance' | 'p5_independent_director';
|
||||
|
||||
/** 2 company stages per manifest v7 §corpus_design.company_stage_axis. */
|
||||
export type CompanyStage = 'stage_a_series_b_growth_burning' | 'stage_b_post_profitable_consolidation';
|
||||
|
||||
/** All 5 task families in canonical order. */
|
||||
export const TASK_FAMILIES: ReadonlyArray<TaskFamily> = ['F1', 'F2', 'F3', 'F4', 'F5'];
|
||||
|
||||
/** All 5 personas in canonical order. */
|
||||
export const PERSONAS: ReadonlyArray<PersonaCode> = [
|
||||
'p1_founder_ceo',
|
||||
'p2_cfo',
|
||||
'p3_coo',
|
||||
'p4_vp_finance',
|
||||
'p5_independent_director',
|
||||
];
|
||||
|
||||
/** All 2 company stages in canonical order. */
|
||||
export const COMPANY_STAGES: ReadonlyArray<CompanyStage> = [
|
||||
'stage_a_series_b_growth_burning',
|
||||
'stage_b_post_profitable_consolidation',
|
||||
];
|
||||
|
||||
/** Total instances per manifest v7 §corpus_design.total_instances. */
|
||||
export const TOTAL_INSTANCES = 50;
|
||||
|
||||
/** Required source document count floor per manifest v7. */
|
||||
export const DOCS_PER_INSTANCE_MIN = 6;
|
||||
|
||||
/** Required source document count ceiling per manifest v7. */
|
||||
export const DOCS_PER_INSTANCE_MAX = 8;
|
||||
|
||||
/** Spot-audit sample size per manifest v7 §corpus_design.spot_audit. */
|
||||
export const SPOT_AUDIT_SAMPLE_SIZE = 5;
|
||||
|
||||
/** Deterministic stratification seed per manifest v7 §corpus_design.stratified_sampling.seed. */
|
||||
export const STRATIFICATION_SEED = 42;
|
||||
|
||||
/** Per-instance human-readable task family descriptor. */
|
||||
export const TASK_FAMILY_DESCRIPTORS: Record<TaskFamily, {
|
||||
label: string;
|
||||
mirrorPilotTask: string | null;
|
||||
docsPerInstance: number;
|
||||
promptTemplate: string;
|
||||
}> = {
|
||||
F1: {
|
||||
label: 'strategic_synthesis',
|
||||
mirrorPilotTask: 'task-1',
|
||||
docsPerInstance: 7,
|
||||
promptTemplate: 'Identify the 3 most critical risks for {company} in {period} and propose action plan for each.',
|
||||
},
|
||||
F2: {
|
||||
label: 'cross_thread_coordination',
|
||||
mirrorPilotTask: 'task-2',
|
||||
docsPerInstance: 6,
|
||||
promptTemplate: 'Reconcile conflicting positions from {stakeholders} and propose unified approach.',
|
||||
},
|
||||
F3: {
|
||||
label: 'decision_support',
|
||||
mirrorPilotTask: 'task-3',
|
||||
docsPerInstance: 6,
|
||||
promptTemplate: 'Recommend {decision} based on materials; justify, address counter-arguments.',
|
||||
},
|
||||
F4: {
|
||||
label: 'investor_communications',
|
||||
mirrorPilotTask: null, // NEW family
|
||||
docsPerInstance: 6,
|
||||
promptTemplate: 'Draft Q{n} investor update covering {metrics} + addressing {concerns}.',
|
||||
},
|
||||
F5: {
|
||||
label: 'scenario_planning',
|
||||
mirrorPilotTask: null, // NEW family
|
||||
docsPerInstance: 7,
|
||||
promptTemplate: 'Compare {n} scenarios for {decision_area}; recommend hedging strategy.',
|
||||
},
|
||||
};
|
||||
|
||||
/** Stratification cell — one of 50 unique combinations. */
|
||||
export interface StratificationCell {
|
||||
family: TaskFamily;
|
||||
persona: PersonaCode;
|
||||
stage: CompanyStage;
|
||||
}
|
||||
|
||||
/** A single source document inside a corpus instance. */
|
||||
export interface SourceDoc {
|
||||
title: string;
|
||||
body: string;
|
||||
charCount: number;
|
||||
}
|
||||
|
||||
/** A single corpus instance ready for evaluation by NULL-baseline / GEPA candidates. */
|
||||
export interface CorpusInstance {
|
||||
/** Stable instance ID: h3-{family}-{persona}-{stage}-{ordinal}. */
|
||||
instanceId: string;
|
||||
cell: StratificationCell;
|
||||
personaText: string;
|
||||
scenario: string;
|
||||
sourceDocuments: SourceDoc[];
|
||||
question: string;
|
||||
/** Aggregated materials block (joined source docs) — consumer convenience. */
|
||||
materialsConcat: string;
|
||||
manifestAnchor: string;
|
||||
generatedBy: string;
|
||||
generatedAtIso: string;
|
||||
generationCostUsd: number;
|
||||
}
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// Stratification — deterministic enumeration of 50 cells
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Enumerate all 50 stratification cells in canonical (family-major, persona-mid,
|
||||
* stage-minor) order. Deterministic given the constant arrays above.
|
||||
*
|
||||
* Yield order: F1×p1×stage_a, F1×p1×stage_b, F1×p2×stage_a, ..., F5×p5×stage_b.
|
||||
*/
|
||||
export function* iterateStratificationCells(): Generator<StratificationCell> {
|
||||
for (const family of TASK_FAMILIES) {
|
||||
for (const persona of PERSONAS) {
|
||||
for (const stage of COMPANY_STAGES) {
|
||||
yield { family, persona, stage };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Materialize all 50 cells as an array. */
|
||||
export function listStratificationCells(): StratificationCell[] {
|
||||
return Array.from(iterateStratificationCells());
|
||||
}
|
||||
|
||||
/** Build a stable instance ID from a cell + ordinal (1-based within cell). */
|
||||
export function buildInstanceId(cell: StratificationCell, ordinal: number = 1): string {
|
||||
return `h3-${cell.family}-${cell.persona}-${cell.stage}-${String(ordinal).padStart(3, '0')}`;
|
||||
}
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// Validation — check instance shape against manifest v7 quality floor
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface InstanceValidationResult {
|
||||
valid: boolean;
|
||||
violations: string[];
|
||||
}
|
||||
|
||||
/** Per manifest v7 §corpus_design.per_instance_quality_floor + spot_audit dimensions. */
|
||||
export function validateInstance(instance: CorpusInstance): InstanceValidationResult {
|
||||
const violations: string[] = [];
|
||||
|
||||
if (instance.sourceDocuments.length < DOCS_PER_INSTANCE_MIN) {
|
||||
violations.push(`docs count ${instance.sourceDocuments.length} < ${DOCS_PER_INSTANCE_MIN} min`);
|
||||
}
|
||||
if (instance.sourceDocuments.length > DOCS_PER_INSTANCE_MAX) {
|
||||
violations.push(`docs count ${instance.sourceDocuments.length} > ${DOCS_PER_INSTANCE_MAX} max`);
|
||||
}
|
||||
|
||||
// Length bounds loosened post-probe (2026-04-28) — Opus generates naturally
|
||||
// richer personas/scenarios than the hand-crafted pilot baseline. The intent
|
||||
// of these bounds is to catch broken/empty output, not to police verbosity.
|
||||
const personaLen = instance.personaText.length;
|
||||
if (personaLen < 100 || personaLen > 1500) {
|
||||
violations.push(`persona length ${personaLen} outside [100, 1500] range`);
|
||||
}
|
||||
|
||||
// Scenario may be empty if oracle embedded it in personaText (handled by
|
||||
// assembleInstance — extractScenarioFromPersonaText). Skip length floor in
|
||||
// that case but still cap upper bound.
|
||||
const scenarioLen = instance.scenario.length;
|
||||
if (scenarioLen > 0 && (scenarioLen < 100 || scenarioLen > 1500)) {
|
||||
violations.push(`scenario length ${scenarioLen} outside [100, 1500] range (or 0 if embedded in personaText)`);
|
||||
}
|
||||
|
||||
const questionLen = instance.question.length;
|
||||
if (questionLen < 100 || questionLen > 800) {
|
||||
violations.push(`question length ${questionLen} outside [100, 800] range`);
|
||||
}
|
||||
|
||||
for (const doc of instance.sourceDocuments) {
|
||||
if (doc.body.length < 400 || doc.body.length > 3000) {
|
||||
violations.push(`doc "${doc.title}" body length ${doc.body.length} outside [400, 3000] range`);
|
||||
}
|
||||
if (doc.body.length !== doc.charCount) {
|
||||
violations.push(`doc "${doc.title}" charCount ${doc.charCount} != actual body length ${doc.body.length}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!instance.instanceId.startsWith('h3-')) {
|
||||
violations.push(`instanceId "${instance.instanceId}" missing h3- prefix`);
|
||||
}
|
||||
|
||||
return {
|
||||
valid: violations.length === 0,
|
||||
violations,
|
||||
};
|
||||
}
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// Spot-audit sampler — deterministic random selection of N instances
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Mulberry32 PRNG — deterministic, fast, 32-bit. Same seed produces same
|
||||
* output across runs and platforms.
|
||||
*/
|
||||
function mulberry32(seed: number): () => number {
|
||||
let t = seed >>> 0;
|
||||
return function () {
|
||||
t = (t + 0x6d2b79f5) >>> 0;
|
||||
let r = t;
|
||||
r = Math.imul(r ^ (r >>> 15), r | 1);
|
||||
r ^= r + Math.imul(r ^ (r >>> 7), r | 61);
|
||||
return ((r ^ (r >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Deterministic Fisher-Yates shuffle producing a sample of `sampleSize` from
|
||||
* the input array, seeded by `seed`. Reproducible across runs.
|
||||
*/
|
||||
export function deterministicSample<T>(
|
||||
items: ReadonlyArray<T>,
|
||||
sampleSize: number,
|
||||
seed: number = STRATIFICATION_SEED,
|
||||
): T[] {
|
||||
if (sampleSize >= items.length) {
|
||||
return [...items];
|
||||
}
|
||||
const arr = [...items];
|
||||
const rand = mulberry32(seed);
|
||||
// Partial Fisher-Yates: only need first `sampleSize` swapped to front.
|
||||
for (let i = 0; i < sampleSize; i++) {
|
||||
const j = i + Math.floor(rand() * (arr.length - i));
|
||||
[arr[i], arr[j]] = [arr[j], arr[i]];
|
||||
}
|
||||
return arr.slice(0, sampleSize);
|
||||
}
|
||||
|
||||
/** Select 5 random instances per manifest v7 §corpus_design.spot_audit. */
|
||||
export function selectSpotAuditSample(instances: ReadonlyArray<CorpusInstance>): CorpusInstance[] {
|
||||
return deterministicSample(instances, SPOT_AUDIT_SAMPLE_SIZE);
|
||||
}
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// Aggregate spot-audit verdict
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface SpotAuditReport {
|
||||
sampleSize: number;
|
||||
perInstance: Array<{ instanceId: string; result: InstanceValidationResult }>;
|
||||
/** Per manifest v7 §corpus_design.spot_audit.halt_on: any 1 of 5 fails → corpus regeneration. */
|
||||
haltOnFailure: boolean;
|
||||
haltReason?: string;
|
||||
}
|
||||
|
||||
/** Run validation across spot-audit sample + return aggregate verdict. */
|
||||
export function runSpotAudit(instances: ReadonlyArray<CorpusInstance>): SpotAuditReport {
|
||||
const sample = selectSpotAuditSample(instances);
|
||||
const perInstance = sample.map(inst => ({
|
||||
instanceId: inst.instanceId,
|
||||
result: validateInstance(inst),
|
||||
}));
|
||||
const failed = perInstance.filter(p => !p.result.valid);
|
||||
return {
|
||||
sampleSize: sample.length,
|
||||
perInstance,
|
||||
haltOnFailure: failed.length > 0,
|
||||
haltReason: failed.length > 0
|
||||
? `${failed.length}/${sample.length} spot-audit instances failed validation: ${failed.map(f => f.instanceId).join(', ')}`
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// Audit-chain helpers
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Compute SHA-256 of a JSON-serialized corpus (for manifest v7 audit pinning). */
|
||||
export function corpusSha256(instances: ReadonlyArray<CorpusInstance>): string {
|
||||
const canonical = instances.map(inst => ({
|
||||
instanceId: inst.instanceId,
|
||||
cell: inst.cell,
|
||||
materialsLength: inst.materialsConcat.length,
|
||||
questionLength: inst.question.length,
|
||||
docCount: inst.sourceDocuments.length,
|
||||
}));
|
||||
const json = JSON.stringify(canonical);
|
||||
return crypto.createHash('sha256').update(json).digest('hex');
|
||||
}
|
||||
133
benchmarks/gepa/src/faza-1/cost-tracker.ts
Normal file
133
benchmarks/gepa/src/faza-1/cost-tracker.ts
Normal file
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* GEPA Faza 1 — cost governance + super-linear projection tracker.
|
||||
*
|
||||
* Per launch decision §A.7 + manifest v7 §cost_governance.
|
||||
*
|
||||
* Halt triggers (any single triggers immediate halt + PM ratify):
|
||||
* - Cumulative spend > $80 (internal halt)
|
||||
* - Cumulative spend > $100 (hard cap)
|
||||
* - Mid-run actual exceeds projection by >30% (super-linear sub-rule per brief §6.7)
|
||||
*
|
||||
* Audit cadence: every 20 evaluations.
|
||||
*
|
||||
* Cost projection methodology: 1.5× baseline token count per candidate
|
||||
* (encodes mutation overhead since GEPA candidates may grow prompts).
|
||||
*/
|
||||
|
||||
/** Hard cap (immediate halt on breach). */
|
||||
export const HARD_CAP_USD = 100.00;
|
||||
|
||||
/** Internal halt threshold (halt + PM ratify before proceeding). */
|
||||
export const INTERNAL_HALT_USD = 80.00;
|
||||
|
||||
/** Super-linear projection multiplier per brief §6.7. */
|
||||
export const SUPER_LINEAR_MULTIPLIER = 1.5;
|
||||
|
||||
/** Mid-run halt threshold: actual exceeds projection by this fraction. */
|
||||
export const SUPER_LINEAR_OVERAGE_THRESHOLD = 0.30;
|
||||
|
||||
/** Audit cadence (every N evaluations). */
|
||||
export const AUDIT_CADENCE_EVAL_COUNT = 20;
|
||||
|
||||
export type HaltReason =
|
||||
| 'NONE'
|
||||
| 'INTERNAL_HALT_USD_BREACH'
|
||||
| 'HARD_CAP_USD_BREACH'
|
||||
| 'SUPER_LINEAR_PROJECTION_BREACH';
|
||||
|
||||
export interface CostTrackerState {
|
||||
cumulativeUsd: number;
|
||||
evaluationCount: number;
|
||||
/** Per-evaluation projection used for super-linear check (1.5× baseline median). */
|
||||
projectionPerEvalUsd: number;
|
||||
}
|
||||
|
||||
export interface HaltCheckResult {
|
||||
haltReason: HaltReason;
|
||||
cumulativeUsd: number;
|
||||
expectedAtThisCount: number;
|
||||
overageFraction: number;
|
||||
message: string;
|
||||
}
|
||||
|
||||
/** Create a new cost tracker state with the per-evaluation projection. */
|
||||
export function createCostTracker(baselineMedianCostPerEvalUsd: number): CostTrackerState {
|
||||
return {
|
||||
cumulativeUsd: 0,
|
||||
evaluationCount: 0,
|
||||
projectionPerEvalUsd: baselineMedianCostPerEvalUsd * SUPER_LINEAR_MULTIPLIER,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Record an evaluation cost and return updated state (immutable update).
|
||||
* Per coding-style.md: never mutate, always return new copy.
|
||||
*/
|
||||
export function recordEvaluation(state: CostTrackerState, evalCostUsd: number): CostTrackerState {
|
||||
return {
|
||||
...state,
|
||||
cumulativeUsd: state.cumulativeUsd + evalCostUsd,
|
||||
evaluationCount: state.evaluationCount + 1,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check halt triggers against current state.
|
||||
*
|
||||
* Returns NONE if all checks pass; otherwise returns the first triggered
|
||||
* halt reason with diagnostic context.
|
||||
*
|
||||
* Order of precedence (most severe first):
|
||||
* 1. HARD_CAP_USD_BREACH ($100 breach)
|
||||
* 2. INTERNAL_HALT_USD_BREACH ($80 breach)
|
||||
* 3. SUPER_LINEAR_PROJECTION_BREACH (actual > 1.30 × expected at current eval count)
|
||||
*/
|
||||
export function checkHaltTriggers(state: CostTrackerState): HaltCheckResult {
|
||||
const expectedAtThisCount = state.projectionPerEvalUsd * state.evaluationCount;
|
||||
const overageFraction =
|
||||
expectedAtThisCount > 0 ? (state.cumulativeUsd - expectedAtThisCount) / expectedAtThisCount : 0;
|
||||
|
||||
if (state.cumulativeUsd > HARD_CAP_USD) {
|
||||
return {
|
||||
haltReason: 'HARD_CAP_USD_BREACH',
|
||||
cumulativeUsd: state.cumulativeUsd,
|
||||
expectedAtThisCount,
|
||||
overageFraction,
|
||||
message: `HARD CAP BREACH: $${state.cumulativeUsd.toFixed(2)} > $${HARD_CAP_USD} cap`,
|
||||
};
|
||||
}
|
||||
|
||||
if (state.cumulativeUsd > INTERNAL_HALT_USD) {
|
||||
return {
|
||||
haltReason: 'INTERNAL_HALT_USD_BREACH',
|
||||
cumulativeUsd: state.cumulativeUsd,
|
||||
expectedAtThisCount,
|
||||
overageFraction,
|
||||
message: `INTERNAL HALT: $${state.cumulativeUsd.toFixed(2)} > $${INTERNAL_HALT_USD} internal halt — PM ratify before proceeding`,
|
||||
};
|
||||
}
|
||||
|
||||
// Super-linear check requires at least 1 eval to have meaningful expected value.
|
||||
if (state.evaluationCount > 0 && overageFraction > SUPER_LINEAR_OVERAGE_THRESHOLD) {
|
||||
return {
|
||||
haltReason: 'SUPER_LINEAR_PROJECTION_BREACH',
|
||||
cumulativeUsd: state.cumulativeUsd,
|
||||
expectedAtThisCount,
|
||||
overageFraction,
|
||||
message: `SUPER-LINEAR BREACH: actual $${state.cumulativeUsd.toFixed(2)} exceeds expected $${expectedAtThisCount.toFixed(2)} by ${(overageFraction * 100).toFixed(1)}% (>${(SUPER_LINEAR_OVERAGE_THRESHOLD * 100).toFixed(0)}% threshold)`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
haltReason: 'NONE',
|
||||
cumulativeUsd: state.cumulativeUsd,
|
||||
expectedAtThisCount,
|
||||
overageFraction,
|
||||
message: `OK: $${state.cumulativeUsd.toFixed(2)} cumulative; expected $${expectedAtThisCount.toFixed(2)}; ${(overageFraction * 100).toFixed(1)}% overage`,
|
||||
};
|
||||
}
|
||||
|
||||
/** Whether the current eval count is on an audit cadence boundary. */
|
||||
export function shouldAudit(state: CostTrackerState): boolean {
|
||||
return state.evaluationCount > 0 && state.evaluationCount % AUDIT_CADENCE_EVAL_COUNT === 0;
|
||||
}
|
||||
298
benchmarks/gepa/src/faza-1/fitness.ts
Normal file
298
benchmarks/gepa/src/faza-1/fitness.ts
Normal file
@@ -0,0 +1,298 @@
|
||||
/**
|
||||
* GEPA Faza 1 — per-shape fitness function.
|
||||
*
|
||||
* Per Amendment 2 §3 (manifest v7 §metric_operationalization.per_shape_fitness_formula
|
||||
* + §metric_operationalization.retrieval_engagement_bonus).
|
||||
*
|
||||
* Phase 4.5 empirical signal: Qwen retrieves 1.33×/task vs Opus 2.33×/task.
|
||||
* H4 score gap mechanistically traces to under-engagement. Mutation surface
|
||||
* (prompt-shape body) is the lever to address this on Qwen-targeted shapes.
|
||||
*
|
||||
* Fitness function forks by shape class:
|
||||
* Qwen-targeted: fitness = trio_strict_pass_rate + retrieval_engagement_bonus − cost_penalty
|
||||
* Non-Qwen: fitness = trio_strict_pass_rate − cost_penalty
|
||||
*
|
||||
* Retrieval engagement bonus bands (Qwen-targeted shapes only):
|
||||
* +0.05 if mean retrieval_calls per task ≥ 2.0 (Opus parity proxy)
|
||||
* 0.00 if mean retrieval_calls per task ∈ [1.5, 2.0)
|
||||
* −0.05 if mean retrieval_calls per task < 1.5 (Qwen baseline penalty)
|
||||
*/
|
||||
|
||||
import {
|
||||
type FitnessInputs,
|
||||
type FitnessComponents,
|
||||
type ShapeName,
|
||||
type TieredFitnessInputs,
|
||||
type TieredFitnessComponents,
|
||||
type DeltaFloorInputs,
|
||||
type DeltaFloorVerdict,
|
||||
QWEN_TARGETED_SHAPES,
|
||||
} from './types.js';
|
||||
|
||||
/** Cost penalty coefficient per brief §3.1 — 0.5pp per $0.10 above baseline median. */
|
||||
const COST_PENALTY_PP_PER_DOLLAR_10C = 0.5;
|
||||
|
||||
/** Retrieval engagement thresholds per Amendment 2 §3 bands. */
|
||||
export const RETRIEVAL_ENGAGEMENT_BANDS = {
|
||||
/** ≥ this → +0.05 bonus (Opus parity proxy). */
|
||||
upperThreshold: 2.0,
|
||||
/** ≥ this and < upper → 0.00. */
|
||||
lowerThreshold: 1.5,
|
||||
/** < lower → −0.05 (Qwen baseline penalty). */
|
||||
bonusPlus: 0.05,
|
||||
bonusZero: 0.0,
|
||||
bonusMinus: -0.05,
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Compute the retrieval engagement bonus for a Qwen-targeted shape.
|
||||
*
|
||||
* Returns 0.0 for non-Qwen shapes (callers should branch on shape class
|
||||
* before calling this if they need to distinguish; alternatively use
|
||||
* `computeFitness` which handles routing).
|
||||
*
|
||||
* Boundary semantics (binding per Amendment 2 §8 test cases):
|
||||
* 1.49 → −0.05
|
||||
* 1.50 → 0.00 (exact lower threshold = zero band)
|
||||
* 1.99 → 0.00
|
||||
* 2.00 → +0.05 (exact upper threshold = bonus band)
|
||||
* 2.50 → +0.05
|
||||
*/
|
||||
export function computeRetrievalEngagementBonus(
|
||||
shape: ShapeName,
|
||||
meanRetrievalCallsPerTask: number,
|
||||
): number {
|
||||
if (!QWEN_TARGETED_SHAPES.has(shape)) {
|
||||
return 0.0;
|
||||
}
|
||||
if (meanRetrievalCallsPerTask >= RETRIEVAL_ENGAGEMENT_BANDS.upperThreshold) {
|
||||
return RETRIEVAL_ENGAGEMENT_BANDS.bonusPlus;
|
||||
}
|
||||
if (meanRetrievalCallsPerTask >= RETRIEVAL_ENGAGEMENT_BANDS.lowerThreshold) {
|
||||
return RETRIEVAL_ENGAGEMENT_BANDS.bonusZero;
|
||||
}
|
||||
return RETRIEVAL_ENGAGEMENT_BANDS.bonusMinus;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the cost penalty per brief §3.1.
|
||||
*
|
||||
* If the candidate's mean cost is at or below the per-shape baseline median,
|
||||
* penalty is 0. Otherwise, penalty = 0.5pp per $0.10 of overage, encoded as
|
||||
* a positive decimal (caller subtracts from fitness).
|
||||
*/
|
||||
export function computeCostPenalty(
|
||||
candidateMeanCostUsd: number,
|
||||
baselineMedianCostUsd: number,
|
||||
): number {
|
||||
const overageUsd = candidateMeanCostUsd - baselineMedianCostUsd;
|
||||
if (overageUsd <= 0) {
|
||||
return 0.0;
|
||||
}
|
||||
// 0.5 pp per $0.10 → 5 pp per $1.00 → encoded as 0.05 per $1.00 → 0.005 per $0.10
|
||||
const penaltyDecimal = (overageUsd / 0.10) * (COST_PENALTY_PP_PER_DOLLAR_10C / 100);
|
||||
return penaltyDecimal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the per-shape fitness for a candidate.
|
||||
*
|
||||
* Routes by shape class:
|
||||
* - Qwen-targeted (qwen-thinking, qwen-non-thinking): includes retrieval engagement bonus
|
||||
* - Non-Qwen (claude, gpt, generic-simple): no retrieval engagement weighting
|
||||
*
|
||||
* Returns full FitnessComponents for downstream audit + reporting per
|
||||
* launch decision §A.9 binding compliance requirement.
|
||||
*/
|
||||
export function computeFitness(inputs: FitnessInputs): FitnessComponents {
|
||||
const { candidate, baselineMedianCostUsd } = inputs;
|
||||
|
||||
const trioStrictPassRateII = candidate.trioStrictPassRateII;
|
||||
|
||||
const retrievalEngagementApplied = QWEN_TARGETED_SHAPES.has(candidate.shape);
|
||||
const retrievalEngagementBonus = retrievalEngagementApplied
|
||||
? computeRetrievalEngagementBonus(candidate.shape, candidate.meanRetrievalCallsPerTask)
|
||||
: 0.0;
|
||||
|
||||
const costPenalty = computeCostPenalty(candidate.meanCostUsd, baselineMedianCostUsd);
|
||||
|
||||
const fitness = trioStrictPassRateII + retrievalEngagementBonus - costPenalty;
|
||||
|
||||
return {
|
||||
trioStrictPassRateII,
|
||||
retrievalEngagementBonus,
|
||||
costPenalty,
|
||||
fitness,
|
||||
retrievalEngagementApplied,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Amendment 7 — tiered fitness function ─────────────────────────────────
|
||||
|
||||
/** Tier 2 cap per Amendment 7 §fitness_function_tiered.tier_2 (max bonus). */
|
||||
export const TIER_2_BONUS_CAP = 0.25;
|
||||
/** Tier 2 weight per percentage point of retrieval engagement above NULL baseline. */
|
||||
export const TIER_2_BONUS_PER_PP = 0.05;
|
||||
/** Tier 3 binary bonus when all 7 cell-semantic anchors invariant. */
|
||||
export const TIER_3_BONUS_FULL_INVARIANCE = 0.10;
|
||||
/** Total cell-semantic anchor count (types.ts + MULTI_STEP_ACTION_CONTRACT + 5 baseline shapes). */
|
||||
export const TIER_3_ANCHOR_COUNT_FULL = 7;
|
||||
|
||||
/**
|
||||
* Compute Tier 2 retrieval engagement bonus per Amendment 7 §fitness_function_tiered.tier_2.
|
||||
*
|
||||
* Continuous formula (supersedes Amendment 2 band bonus for tiered ranking):
|
||||
* bonus = clamp(0.05 × delta_pp, 0, 0.25)
|
||||
* where delta_pp = (candidate_mean - baseline_mean) × 100
|
||||
*
|
||||
* Cap reached at +5pp absolute increase in mean retrieval calls per task.
|
||||
* Floor 0 (no negative bonus from Tier 2 — negative-band penalty handled
|
||||
* by Amendment 2 §F.5 false-positive guard separately).
|
||||
*
|
||||
* Always returns 0 for non-Qwen shapes.
|
||||
*/
|
||||
export function computeTier2RetrievalBonus(
|
||||
shape: ShapeName,
|
||||
candidateMeanRetrievalCallsPerTask: number,
|
||||
baselineMeanRetrievalCallsPerTask: number,
|
||||
): number {
|
||||
if (!QWEN_TARGETED_SHAPES.has(shape)) return 0;
|
||||
const deltaAbsolute = candidateMeanRetrievalCallsPerTask - baselineMeanRetrievalCallsPerTask;
|
||||
if (deltaAbsolute <= 0) return 0;
|
||||
const deltaPP = deltaAbsolute * 100;
|
||||
return Math.min(TIER_2_BONUS_PER_PP * deltaPP, TIER_2_BONUS_CAP);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the per-shape tiered fitness per Amendment 7 §fitness_function_tiered.
|
||||
*
|
||||
* Routes tier roles by saturation:
|
||||
* - In saturated regime (≥4/5 shapes have NULL pass rate ≥75%):
|
||||
* PRIMARY differentiator = Tier 2 (Qwen-targeted retrieval engagement)
|
||||
* SECONDARY differentiator = Tier 3 (cell-semantic anchor invariance)
|
||||
* TIE-BREAKER = Tier 1 (NULL pass rate delta)
|
||||
* aggregateSaturatedRegime = tier_2 + tier_3
|
||||
* - Outside saturated regime: legacy Amendment 2 form (computeFitness) applies
|
||||
* and aggregateSaturatedRegime is reported but not the canonical fitness.
|
||||
*
|
||||
* Tier 1 also serves as the §F.1 acceptance gate (≥+5pp) — UNCHANGED from
|
||||
* Amendment 5 launch decision §F.1.
|
||||
*/
|
||||
export function computeTieredFitness(inputs: TieredFitnessInputs): TieredFitnessComponents {
|
||||
const {
|
||||
candidate,
|
||||
nullBaselinePassRateII,
|
||||
nullBaselineMeanRetrievalCallsPerTask,
|
||||
mutationValidatorPassed,
|
||||
saturatedRegime,
|
||||
} = inputs;
|
||||
|
||||
// Tier 1 — NULL pass rate delta (signed pp)
|
||||
const tier1DeltaPP = (candidate.trioStrictPassRateII - nullBaselinePassRateII) * 100;
|
||||
|
||||
// Tier 2 — continuous retrieval engagement bonus (Qwen-targeted only)
|
||||
const tier2RetrievalBonus = computeTier2RetrievalBonus(
|
||||
candidate.shape,
|
||||
candidate.meanRetrievalCallsPerTask,
|
||||
nullBaselineMeanRetrievalCallsPerTask,
|
||||
);
|
||||
|
||||
// Tier 3 — binary cell-semantic anchor invariance bonus
|
||||
const tier3CellSemanticInvarianceBonus = mutationValidatorPassed
|
||||
? TIER_3_BONUS_FULL_INVARIANCE
|
||||
: 0;
|
||||
// For Gen 1 candidates the mutation_validator gives binary (valid|invalid);
|
||||
// count semantic: pass = full 7-anchor invariance, fail = 0.
|
||||
const cellSemanticAnchorInvarianceCount = mutationValidatorPassed ? TIER_3_ANCHOR_COUNT_FULL : 0;
|
||||
|
||||
// Aggregate fitness in saturated regime: tier_2 + tier_3 only
|
||||
const aggregateSaturatedRegime = tier2RetrievalBonus + tier3CellSemanticInvarianceBonus;
|
||||
|
||||
return {
|
||||
tier1DeltaPP,
|
||||
tier2RetrievalBonus,
|
||||
tier3CellSemanticInvarianceBonus,
|
||||
aggregateSaturatedRegime,
|
||||
saturatedRegimeApplied: saturatedRegime,
|
||||
cellSemanticAnchorInvarianceCount,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Amendment 7 — pre-registered Δ-floor verdict ──────────────────────────
|
||||
|
||||
/** Δ-floor thresholds per Amendment 7 §gen_1_pre_registered_delta_floor. */
|
||||
export const DELTA_FLOOR_THRESHOLDS = {
|
||||
/** Threshold 1: aggregate Tier 1 delta in pp (≥+3pp passes). */
|
||||
threshold1AggregateTier1PP: 3,
|
||||
/** Threshold 2: max Qwen retrieval engagement delta absolute (≥+0.10 passes). */
|
||||
threshold2QwenRetrievalAbsolute: 0.10,
|
||||
/** Threshold 3a: aggregate Tier 1 delta in pp (≥0pp). */
|
||||
threshold3Tier1MinPP: 0,
|
||||
/** Threshold 3b: aggregate Tier 2 bonus across Qwen-targeted candidates (≥0.05). */
|
||||
threshold3Tier2MinBonus: 0.05,
|
||||
} as const;
|
||||
|
||||
/** Float tolerance for exact-boundary threshold comparisons (consistent with acceptance.ts EPSILON). */
|
||||
const DELTA_FLOOR_EPSILON = 1e-9;
|
||||
|
||||
/**
|
||||
* Compute the Δ-floor verdict per Amendment 7 §gen_1_pre_registered_delta_floor.
|
||||
*
|
||||
* Three OR-gated thresholds:
|
||||
* 1. aggregate Tier 1 delta ≥ +3pp absolute (loosened from §F.1 ≥+5pp)
|
||||
* 2. max Qwen-shape retrieval engagement delta ≥ +0.10 absolute above per-shape NULL baseline
|
||||
* 3. (aggregate Tier 1 delta ≥ 0pp) AND (aggregate Tier 2 bonus ≥ 0.05)
|
||||
*
|
||||
* If ANY ONE passes → PROCEED (continue past Checkpoint B subject to PM ratify).
|
||||
* If ALL THREE fail → HALT_INVESTIGATE (file Investigate report).
|
||||
*
|
||||
* Per Amendment 7 §3.2: "If Gen 1 fails ALL three thresholds → HALT before Gen 2,
|
||||
* file Investigate report. If Gen 1 passes any one → proceed to Gen 2."
|
||||
*/
|
||||
export function computeDeltaFloorVerdict(inputs: DeltaFloorInputs): DeltaFloorVerdict {
|
||||
// Threshold 1 — aggregate Tier 1 delta
|
||||
const threshold1ValuePP =
|
||||
(inputs.aggregateTrioStrictPassRateII - inputs.aggregateNullBaselinePassRateII) * 100;
|
||||
const threshold1Pass =
|
||||
threshold1ValuePP >= DELTA_FLOOR_THRESHOLDS.threshold1AggregateTier1PP - DELTA_FLOOR_EPSILON;
|
||||
|
||||
// Threshold 2 — max Qwen retrieval engagement delta absolute
|
||||
let threshold2MaxDeltaAbsolute: number | null = null;
|
||||
for (const [shapeKey, candidateMean] of Object.entries(inputs.qwenShapeRetrievalMeans)) {
|
||||
if (candidateMean === undefined) continue;
|
||||
const baselineMean = inputs.qwenShapeNullBaselineRetrievalMeans[shapeKey as ShapeName];
|
||||
if (baselineMean === undefined) continue;
|
||||
const delta = candidateMean - baselineMean;
|
||||
if (threshold2MaxDeltaAbsolute === null || delta > threshold2MaxDeltaAbsolute) {
|
||||
threshold2MaxDeltaAbsolute = delta;
|
||||
}
|
||||
}
|
||||
// No Qwen data: report 0 delta (informative neutral); threshold2 fails since 0 < 0.10
|
||||
if (threshold2MaxDeltaAbsolute === null) threshold2MaxDeltaAbsolute = 0;
|
||||
const threshold2Pass =
|
||||
threshold2MaxDeltaAbsolute >=
|
||||
DELTA_FLOOR_THRESHOLDS.threshold2QwenRetrievalAbsolute - DELTA_FLOOR_EPSILON;
|
||||
|
||||
// Threshold 3 — compound Tier 1 ≥0 AND Tier 2 ≥0.05
|
||||
const threshold3Tier1ValuePP = threshold1ValuePP;
|
||||
const threshold3Tier1Pass =
|
||||
threshold3Tier1ValuePP >= DELTA_FLOOR_THRESHOLDS.threshold3Tier1MinPP - DELTA_FLOOR_EPSILON;
|
||||
const threshold3Tier2Pass =
|
||||
inputs.qwenAggregateTier2Bonus >=
|
||||
DELTA_FLOOR_THRESHOLDS.threshold3Tier2MinBonus - DELTA_FLOOR_EPSILON;
|
||||
const threshold3Pass = threshold3Tier1Pass && threshold3Tier2Pass;
|
||||
|
||||
const overallVerdict: 'PROCEED' | 'HALT_INVESTIGATE' =
|
||||
threshold1Pass || threshold2Pass || threshold3Pass ? 'PROCEED' : 'HALT_INVESTIGATE';
|
||||
|
||||
return {
|
||||
threshold1AggregateTier1: threshold1Pass ? 'PASS' : 'FAIL',
|
||||
threshold1ValuePP,
|
||||
threshold2QwenRetrievalAbsolute: threshold2Pass ? 'PASS' : 'FAIL',
|
||||
threshold2MaxDeltaAbsolute,
|
||||
threshold3CompoundTier1PlusTier2: threshold3Pass ? 'PASS' : 'FAIL',
|
||||
threshold3Tier1ValuePP,
|
||||
threshold3Tier2Aggregate: inputs.qwenAggregateTier2Bonus,
|
||||
overallVerdict,
|
||||
};
|
||||
}
|
||||
24
benchmarks/gepa/src/faza-1/index.ts
Normal file
24
benchmarks/gepa/src/faza-1/index.ts
Normal file
@@ -0,0 +1,24 @@
|
||||
/**
|
||||
* GEPA Faza 1 — public API surface.
|
||||
*
|
||||
* Per launch decision §G + manifest v7 §gepa.
|
||||
*
|
||||
* Module map:
|
||||
* - types shared types + shape-class partitions
|
||||
* - fitness per-shape fitness function (Amendment 2 §3 fork)
|
||||
* - acceptance §F + §F.5 verdict per candidate
|
||||
* - mutation-validator cell-semantic preservation audit (boundary SHAs)
|
||||
* - kappa-audit drift band detection vs canonical 0.7878 ± 0.05
|
||||
* - cost-tracker super-linear governance + halt triggers
|
||||
* - selection top-1-per-shape + run-aggregate verdict
|
||||
* - mutation-oracle-fork Qwen vs non-Qwen template routing (Amendment 2 §4)
|
||||
*/
|
||||
|
||||
export * from './types.js';
|
||||
export * from './fitness.js';
|
||||
export * from './acceptance.js';
|
||||
export * from './mutation-validator.js';
|
||||
export * from './kappa-audit.js';
|
||||
export * from './cost-tracker.js';
|
||||
export * from './selection.js';
|
||||
export * from './mutation-oracle-fork.js';
|
||||
141
benchmarks/gepa/src/faza-1/kappa-audit.ts
Normal file
141
benchmarks/gepa/src/faza-1/kappa-audit.ts
Normal file
@@ -0,0 +1,141 @@
|
||||
/**
|
||||
* GEPA Faza 1 — κ drift audit utility.
|
||||
*
|
||||
* Per launch decision §F.3 + manifest v7 §canonical_kappa_anchor +
|
||||
* §faza_1_acceptance.condition_3.
|
||||
*
|
||||
* Canonical κ value: 0.7877758913412564 (rounds to 0.7878).
|
||||
* Source: benchmarks/calibration/v6-kappa-recal/_summary-v6-kappa.json
|
||||
* (SHA 657d4490bab28d35cf8a9c3ccea8a6b79e92835d700155184e51f3900836684c)
|
||||
* Drift threshold: ±0.05 (drift band [0.7378, 0.8378])
|
||||
*
|
||||
* Per κ_recalibration_v6 success_criteria + monitoring_runtime in manifest v6 §5.4:
|
||||
* - PASS: κ_trio ≥ 0.70
|
||||
* - BORDERLINE: κ_trio ∈ [0.60, 0.70]
|
||||
* - FAIL: κ_trio < 0.60
|
||||
*
|
||||
* Faza 1-specific drift acceptance (tighter than v6 baseline policy):
|
||||
* - PASS: κ_trio ∈ [0.7378, 0.8378] (canonical ±0.05)
|
||||
* - DRIFT_LOW: κ_trio < 0.7378
|
||||
* - DRIFT_HIGH: κ_trio > 0.8378 (also a fail signal — judge ensemble drifted upward)
|
||||
*/
|
||||
|
||||
/** Canonical κ baseline anchor pinned per manifest v7. */
|
||||
export const CANONICAL_KAPPA = 0.7877758913412564;
|
||||
|
||||
/** Drift threshold (per brief §4 condition 3). */
|
||||
export const KAPPA_DRIFT_THRESHOLD = 0.05;
|
||||
|
||||
/** Drift band lower bound (canonical − threshold). */
|
||||
export const KAPPA_DRIFT_BAND_LOW = CANONICAL_KAPPA - KAPPA_DRIFT_THRESHOLD;
|
||||
|
||||
/** Drift band upper bound (canonical + threshold). */
|
||||
export const KAPPA_DRIFT_BAND_HIGH = CANONICAL_KAPPA + KAPPA_DRIFT_THRESHOLD;
|
||||
|
||||
/** v6 policy floor for absolute κ pass (kept for cross-validation reporting). */
|
||||
export const V6_KAPPA_POLICY_FLOOR_PASS = 0.70;
|
||||
|
||||
/** v6 borderline lower bound. */
|
||||
export const V6_KAPPA_POLICY_FLOOR_BORDERLINE = 0.60;
|
||||
|
||||
export type KappaVerdict =
|
||||
| 'PASS_WITHIN_DRIFT_BAND'
|
||||
| 'DRIFT_LOW_BELOW_BAND'
|
||||
| 'DRIFT_HIGH_ABOVE_BAND';
|
||||
|
||||
export interface KappaPairwise {
|
||||
/** Cohen's κ between Opus and GPT verdict streams. */
|
||||
kOpusGpt: number;
|
||||
/** Cohen's κ between Opus and MiniMax verdict streams. */
|
||||
kOpusMinimax: number;
|
||||
/** Cohen's κ between GPT and MiniMax verdict streams. */
|
||||
kGptMinimax: number;
|
||||
}
|
||||
|
||||
export interface KappaAuditResult {
|
||||
/** Conservative trio κ = min of three pairwise κ values (per manifest v6 §5.4). */
|
||||
kConservativeTrio: number;
|
||||
/** Pairwise components (carried through for audit log). */
|
||||
pairwise: KappaPairwise;
|
||||
/** Drift band verdict per Faza 1 §F.3 acceptance. */
|
||||
verdict: KappaVerdict;
|
||||
/** Absolute drift from canonical (positive = above, negative = below). */
|
||||
driftFromCanonical: number;
|
||||
/** Whether the κ_conservative_trio passes v6 policy floor (≥ 0.70). */
|
||||
v6PolicyFloorPass: boolean;
|
||||
/** Audit log line for inclusion in checkpoint reports. */
|
||||
auditLogLine: string;
|
||||
}
|
||||
|
||||
/** Compute κ audit verdict from three pairwise Cohen's κ values. */
|
||||
export function auditKappa(pairwise: KappaPairwise): KappaAuditResult {
|
||||
const kConservativeTrio = Math.min(
|
||||
pairwise.kOpusGpt,
|
||||
pairwise.kOpusMinimax,
|
||||
pairwise.kGptMinimax,
|
||||
);
|
||||
|
||||
const driftFromCanonical = kConservativeTrio - CANONICAL_KAPPA;
|
||||
|
||||
let verdict: KappaVerdict;
|
||||
if (kConservativeTrio < KAPPA_DRIFT_BAND_LOW) {
|
||||
verdict = 'DRIFT_LOW_BELOW_BAND';
|
||||
} else if (kConservativeTrio > KAPPA_DRIFT_BAND_HIGH) {
|
||||
verdict = 'DRIFT_HIGH_ABOVE_BAND';
|
||||
} else {
|
||||
verdict = 'PASS_WITHIN_DRIFT_BAND';
|
||||
}
|
||||
|
||||
const v6PolicyFloorPass = kConservativeTrio >= V6_KAPPA_POLICY_FLOOR_PASS;
|
||||
|
||||
const auditLogLine = [
|
||||
`κ_conservative_trio=${kConservativeTrio.toFixed(4)}`,
|
||||
`canonical=${CANONICAL_KAPPA.toFixed(4)}`,
|
||||
`drift=${driftFromCanonical >= 0 ? '+' : ''}${driftFromCanonical.toFixed(4)}`,
|
||||
`verdict=${verdict}`,
|
||||
`v6_policy_floor=${v6PolicyFloorPass ? 'PASS' : 'FAIL'}`,
|
||||
].join(' | ');
|
||||
|
||||
return {
|
||||
kConservativeTrio,
|
||||
pairwise,
|
||||
verdict,
|
||||
driftFromCanonical,
|
||||
v6PolicyFloorPass,
|
||||
auditLogLine,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute Cohen's κ from a 2×2 confusion matrix.
|
||||
*
|
||||
* Convenience helper for callers that have raw verdict pair counts (the κ
|
||||
* recalibration script computes these for Phase 1 baseline; tests and
|
||||
* inline audits during Faza 1 reproduce the computation here).
|
||||
*
|
||||
* Returns NaN if total observations is zero (caller must handle).
|
||||
*/
|
||||
export function computeCohensKappa(confusion: {
|
||||
bothCorrect: number;
|
||||
bothIncorrect: number;
|
||||
firstCorrectSecondIncorrect: number;
|
||||
firstIncorrectSecondCorrect: number;
|
||||
}): number {
|
||||
const { bothCorrect, bothIncorrect, firstCorrectSecondIncorrect, firstIncorrectSecondCorrect } = confusion;
|
||||
const total = bothCorrect + bothIncorrect + firstCorrectSecondIncorrect + firstIncorrectSecondCorrect;
|
||||
if (total === 0) return NaN;
|
||||
|
||||
const observedAgreement = (bothCorrect + bothIncorrect) / total;
|
||||
|
||||
// Marginal probabilities for "correct" verdict per rater.
|
||||
const firstCorrectMarginal = (bothCorrect + firstCorrectSecondIncorrect) / total;
|
||||
const secondCorrectMarginal = (bothCorrect + firstIncorrectSecondCorrect) / total;
|
||||
|
||||
const expectedAgreement =
|
||||
firstCorrectMarginal * secondCorrectMarginal +
|
||||
(1 - firstCorrectMarginal) * (1 - secondCorrectMarginal);
|
||||
|
||||
if (expectedAgreement === 1) return 1.0; // perfect base rate, no variance → return κ=1 by convention
|
||||
|
||||
return (observedAgreement - expectedAgreement) / (1 - expectedAgreement);
|
||||
}
|
||||
85
benchmarks/gepa/src/faza-1/mutation-oracle-fork.ts
Normal file
85
benchmarks/gepa/src/faza-1/mutation-oracle-fork.ts
Normal file
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* GEPA Faza 1 — mutation oracle fork (Qwen vs non-Qwen template routing).
|
||||
*
|
||||
* Per Amendment 2 §4 + manifest v7 §mutation_oracle_design.
|
||||
*
|
||||
* Phase 4.5 finding requires Qwen-specific scaffolding to address mechanistic
|
||||
* under-engagement; uniform mutation guidance would not target the empirical
|
||||
* gap. Forked oracle prompts ensure Qwen mutations explore the engagement-
|
||||
* bonus reward landscape while non-Qwen mutations explore the broader scaffold
|
||||
* space.
|
||||
*
|
||||
* This module is responsible only for:
|
||||
* 1. Selecting the right template path per shape class
|
||||
* 2. Loading template content from disk
|
||||
* 3. Substituting baseline shape body + failure mode summary into template
|
||||
*
|
||||
* The actual LLM oracle call lives in the run orchestrator (out of scaffold scope).
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
import { type ShapeName, QWEN_TARGETED_SHAPES } from './types.js';
|
||||
|
||||
export type TemplateClass = 'qwen' | 'non-qwen';
|
||||
|
||||
/** Per-shape template class routing. */
|
||||
export function classifyShape(shape: ShapeName): TemplateClass {
|
||||
return QWEN_TARGETED_SHAPES.has(shape) ? 'qwen' : 'non-qwen';
|
||||
}
|
||||
|
||||
/** Resolve template path for a given shape class. */
|
||||
export function templatePathForShape(
|
||||
shape: ShapeName,
|
||||
oracleDir: string = path.resolve(__dirname, '../../oracle/faza-1'),
|
||||
): string {
|
||||
const cls = classifyShape(shape);
|
||||
const filename =
|
||||
cls === 'qwen'
|
||||
? 'mutation-prompt-template-qwen.md'
|
||||
: 'mutation-prompt-template-non-qwen.md';
|
||||
return path.join(oracleDir, filename);
|
||||
}
|
||||
|
||||
/** Load raw template content for a given shape. */
|
||||
export function loadTemplate(
|
||||
shape: ShapeName,
|
||||
oracleDir?: string,
|
||||
): string {
|
||||
const templatePath = templatePathForShape(shape, oracleDir);
|
||||
return fs.readFileSync(templatePath, 'utf-8');
|
||||
}
|
||||
|
||||
/** Inputs for assembling the final oracle prompt. */
|
||||
export interface OraclePromptInputs {
|
||||
shape: ShapeName;
|
||||
/** Baseline shape file content (the candidate to mutate). */
|
||||
baselineShapeContent: string;
|
||||
/**
|
||||
* Failure mode summary from Phase 4.3 verdict (top-3 T2 failures for this shape).
|
||||
* Per brief §3.3 mutation oracle prompt.
|
||||
*/
|
||||
failureModeSummary: string;
|
||||
oracleDir?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the complete oracle prompt by template substitution.
|
||||
*
|
||||
* Templates use these placeholders (must appear verbatim in template files):
|
||||
* ###BASELINE_SHAPE_CONTENT###
|
||||
* ###FAILURE_MODE_SUMMARY###
|
||||
* ###SHAPE_NAME###
|
||||
* ###TEMPLATE_CLASS###
|
||||
*/
|
||||
export function buildOraclePrompt(inputs: OraclePromptInputs): string {
|
||||
const template = loadTemplate(inputs.shape, inputs.oracleDir);
|
||||
const cls = classifyShape(inputs.shape);
|
||||
|
||||
return template
|
||||
.replace(/###BASELINE_SHAPE_CONTENT###/g, inputs.baselineShapeContent)
|
||||
.replace(/###FAILURE_MODE_SUMMARY###/g, inputs.failureModeSummary)
|
||||
.replace(/###SHAPE_NAME###/g, inputs.shape)
|
||||
.replace(/###TEMPLATE_CLASS###/g, cls);
|
||||
}
|
||||
194
benchmarks/gepa/src/faza-1/mutation-validator.ts
Normal file
194
benchmarks/gepa/src/faza-1/mutation-validator.ts
Normal file
@@ -0,0 +1,194 @@
|
||||
/**
|
||||
* GEPA Faza 1 — mutation validator (cell-semantic preservation audit).
|
||||
*
|
||||
* Per launch decision §A.4 (binding) + manifest v7 §gepa.mutation_validator.
|
||||
*
|
||||
* Boundary anchor: MULTI_STEP_ACTION_CONTRACT constant in
|
||||
* packages/agent/src/prompt-shapes/types.ts.
|
||||
* - whole-file SHA: 1a9fa329e4b66ed9f0abe8bc22cbbf0124e0c879e1e78ec806d557cab25bc94d
|
||||
* - constant body bytes SHA: 70a1701dfa126f8dc1df9c116f0a8469da005821ecadc59d9b8f348568e755ba (252 bytes)
|
||||
*
|
||||
* Any GEPA candidate that produces non-zero diff against either anchor → INVALID.
|
||||
*
|
||||
* Allowed diff targets (per manifest v7 §gepa.mutation_validator.valid_diff_targets):
|
||||
* - shape_file.systemPrompt method body (string-building only)
|
||||
* - shape_file.soloUserPrompt method body
|
||||
* - shape_file.multiStepKickoffUserPrompt method body
|
||||
* - shape_file.retrievalInjectionUserPrompt method body
|
||||
* - shape_file.metadata.evidence_link (MUST update to point to GEPA Gen 1 results)
|
||||
*
|
||||
* Invalid diff targets (LOCKED):
|
||||
* - types.ts (entire file)
|
||||
* - selector.ts (entire file)
|
||||
* - index.ts (entire file)
|
||||
* - shape_file.metadata.{description,modelClass,defaultThinking,defaultMaxTokens}
|
||||
* - shape_file.imports
|
||||
* - MULTI_STEP_ACTION_CONTRACT bytes
|
||||
*/
|
||||
|
||||
import * as crypto from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
|
||||
/**
|
||||
* Cell-semantic boundary SHAs pinned at substrate anchor commit c9bda3d.
|
||||
* These are the byte-level invariants that any GEPA candidate must preserve.
|
||||
*/
|
||||
export const BOUNDARY_SHAS = {
|
||||
/** Whole types.ts file SHA (must equal this exactly post-mutation). */
|
||||
typesFile: '1a9fa329e4b66ed9f0abe8bc22cbbf0124e0c879e1e78ec806d557cab25bc94d',
|
||||
/** MULTI_STEP_ACTION_CONTRACT constant body bytes SHA (252 bytes). */
|
||||
multiStepActionContract: '70a1701dfa126f8dc1df9c116f0a8469da005821ecadc59d9b8f348568e755ba',
|
||||
} as const;
|
||||
|
||||
/** Per-shape baseline SHAs pinned at substrate anchor commit c9bda3d. */
|
||||
export const BASELINE_SHAPE_SHAS: Readonly<Record<string, string>> = {
|
||||
'claude.ts': 'cbaf0c37b067b025a1fe97f2feeec11fae4070a8b3fcfaad1da8775dda451cc0',
|
||||
'qwen-thinking.ts': '848a4e4917baa5c7bbcc3bb35fb8cb4b4ac8f0ab537243f14cbef3a99197aacb',
|
||||
'qwen-non-thinking.ts': '35be379be9a8caafc2c419e32da5f63f92fc83f6f6d70d9df76029c1e8584572',
|
||||
'gpt.ts': '5dc6d750d52a68feb9d37ad8384b2bcd59d70962066122ff086b0e5888413576',
|
||||
'generic-simple.ts': '81189817f560e26a69394248d8bd9089cae72c7d40825323e2b7407e36026172',
|
||||
} as const;
|
||||
|
||||
/** Validator verdict for a single candidate. */
|
||||
export interface ValidatorVerdict {
|
||||
valid: boolean;
|
||||
/** List of violations (empty if valid). Each violation is a structured reason. */
|
||||
violations: ValidationViolation[];
|
||||
/** SHA of the candidate shape file (computed by validator). */
|
||||
candidateShapeFileSha: string;
|
||||
/** SHA of the cell-semantic types.ts file at validation time. */
|
||||
typesFileSha: string;
|
||||
/** SHA of the MULTI_STEP_ACTION_CONTRACT bytes at validation time. */
|
||||
multiStepActionContractSha: string;
|
||||
}
|
||||
|
||||
export interface ValidationViolation {
|
||||
category:
|
||||
| 'types_file_modified'
|
||||
| 'multi_step_action_contract_modified'
|
||||
| 'shape_file_unchanged_from_baseline'
|
||||
| 'shape_file_metadata_locked_field_modified'
|
||||
| 'shape_file_imports_modified';
|
||||
severity: 'invalid';
|
||||
detail: string;
|
||||
}
|
||||
|
||||
/** Compute SHA-256 of a string's UTF-8 bytes. */
|
||||
export function sha256(bytes: string | Buffer): string {
|
||||
const buf = typeof bytes === 'string' ? Buffer.from(bytes, 'utf-8') : bytes;
|
||||
return crypto.createHash('sha256').update(buf).digest('hex');
|
||||
}
|
||||
|
||||
/** Compute SHA-256 of a file's bytes. */
|
||||
export function sha256File(filepath: string): string {
|
||||
return sha256(fs.readFileSync(filepath));
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the bytes of MULTI_STEP_ACTION_CONTRACT from a types.ts file content.
|
||||
* Returns null if the constant is not found (which itself indicates a violation).
|
||||
*/
|
||||
export function extractMultiStepActionContractBytes(typesFileContent: string): string | null {
|
||||
// Match the template literal body between backticks.
|
||||
const match = typesFileContent.match(/export const MULTI_STEP_ACTION_CONTRACT = `([^`]+)`/);
|
||||
return match ? match[1] : null;
|
||||
}
|
||||
|
||||
/** Inputs for validating a GEPA candidate against cell-semantic boundaries. */
|
||||
export interface ValidatorInputs {
|
||||
/** Path to the candidate shape file (e.g., gepa-evolved/qwen-thinking-gen1-v0.ts). */
|
||||
candidateShapeFilePath: string;
|
||||
/** Shape file basename for baseline lookup (e.g., 'qwen-thinking.ts'). */
|
||||
baselineShapeName: keyof typeof BASELINE_SHAPE_SHAS;
|
||||
/** Path to the types.ts file (cell-semantic boundary anchor). */
|
||||
typesFilePath: string;
|
||||
/** Whether to require shape file to differ from baseline (true for Gen 1 mutations). */
|
||||
expectShapeDiff: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a GEPA candidate against the cell-semantic preservation invariants.
|
||||
*
|
||||
* Returns INVALID with structured violations if any boundary anchor is breached,
|
||||
* or if a Gen 1 mutation produces zero diff vs baseline (violates the
|
||||
* "every mutation must change something" implicit contract — Gen 0 baselines
|
||||
* use expectShapeDiff=false).
|
||||
*/
|
||||
export function validateCandidate(inputs: ValidatorInputs): ValidatorVerdict {
|
||||
const violations: ValidationViolation[] = [];
|
||||
|
||||
const typesFileSha = sha256File(inputs.typesFilePath);
|
||||
const typesContent = fs.readFileSync(inputs.typesFilePath, 'utf-8');
|
||||
const contractBytes = extractMultiStepActionContractBytes(typesContent);
|
||||
const multiStepActionContractSha = contractBytes ? sha256(contractBytes) : '';
|
||||
const candidateShapeFileSha = sha256File(inputs.candidateShapeFilePath);
|
||||
|
||||
if (typesFileSha !== BOUNDARY_SHAS.typesFile) {
|
||||
violations.push({
|
||||
category: 'types_file_modified',
|
||||
severity: 'invalid',
|
||||
detail: `types.ts SHA ${typesFileSha} != pinned baseline ${BOUNDARY_SHAS.typesFile}`,
|
||||
});
|
||||
}
|
||||
|
||||
if (multiStepActionContractSha !== BOUNDARY_SHAS.multiStepActionContract) {
|
||||
violations.push({
|
||||
category: 'multi_step_action_contract_modified',
|
||||
severity: 'invalid',
|
||||
detail:
|
||||
contractBytes === null
|
||||
? 'MULTI_STEP_ACTION_CONTRACT constant not found in types.ts'
|
||||
: `MULTI_STEP_ACTION_CONTRACT bytes SHA ${multiStepActionContractSha} != pinned baseline ${BOUNDARY_SHAS.multiStepActionContract}`,
|
||||
});
|
||||
}
|
||||
|
||||
const baselineSha = BASELINE_SHAPE_SHAS[inputs.baselineShapeName];
|
||||
if (inputs.expectShapeDiff && candidateShapeFileSha === baselineSha) {
|
||||
violations.push({
|
||||
category: 'shape_file_unchanged_from_baseline',
|
||||
severity: 'invalid',
|
||||
detail: `Gen 1 mutation expected to differ from baseline ${inputs.baselineShapeName} (SHA ${baselineSha}) but candidate produced identical bytes`,
|
||||
});
|
||||
}
|
||||
|
||||
// Optional shallow check: locked metadata fields must remain in the shape file.
|
||||
// GEPA mutations are allowed to update evidence_link only.
|
||||
const candidateContent = fs.readFileSync(inputs.candidateShapeFilePath, 'utf-8');
|
||||
const lockedMetadataFields = [
|
||||
'description:',
|
||||
'modelClass:',
|
||||
'defaultThinking:',
|
||||
'defaultMaxTokens:',
|
||||
];
|
||||
for (const field of lockedMetadataFields) {
|
||||
if (!candidateContent.includes(field)) {
|
||||
violations.push({
|
||||
category: 'shape_file_metadata_locked_field_modified',
|
||||
severity: 'invalid',
|
||||
detail: `Locked metadata field "${field}" missing from candidate shape file`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Check that imports block is still present (GEPA mutations cannot add/remove imports).
|
||||
// Accept either './types.js' (same-dir candidate) or '../types.js' (gepa-evolved/ subdir
|
||||
// candidate per manifest v7 §gepa.shape_scope.target_path) — the invariant is that
|
||||
// candidates must import from types.js, not the specific relative path.
|
||||
const importsTypes = candidateContent.includes("from './types.js'") ||
|
||||
candidateContent.includes("from '../types.js'");
|
||||
if (!importsTypes) {
|
||||
violations.push({
|
||||
category: 'shape_file_imports_modified',
|
||||
severity: 'invalid',
|
||||
detail: `Required import from "types.js" (either "./" or "../" relative) missing from candidate shape file`,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
valid: violations.length === 0,
|
||||
violations,
|
||||
candidateShapeFileSha,
|
||||
typesFileSha,
|
||||
multiStepActionContractSha,
|
||||
};
|
||||
}
|
||||
128
benchmarks/gepa/src/faza-1/selection.ts
Normal file
128
benchmarks/gepa/src/faza-1/selection.ts
Normal file
@@ -0,0 +1,128 @@
|
||||
/**
|
||||
* GEPA Faza 1 — top-1-per-shape selection.
|
||||
*
|
||||
* Per brief §2 selection metric ("best-per-shape-per-cell, ne aggregate") +
|
||||
* launch decision §F + §F.5 acceptance.
|
||||
*
|
||||
* Selection algorithm:
|
||||
* 1. For each shape, compute fitness for all candidates (per-shape fitness fork)
|
||||
* 2. Per shape, select candidate with highest fitness as "best"
|
||||
* 3. Apply acceptance verdict to each best-per-shape candidate
|
||||
* 4. Report shape-level + run-aggregate verdict
|
||||
*
|
||||
* Run-aggregate §F conditions:
|
||||
* §F.2: ≥3/5 shapes show positive trio_strict delta vs NULL-baseline
|
||||
*
|
||||
* Per-candidate §F.1 + §F.5 already evaluated by acceptance.evaluateCandidate.
|
||||
*/
|
||||
|
||||
import { computeFitness } from './fitness.js';
|
||||
import { evaluateCandidate } from './acceptance.js';
|
||||
import {
|
||||
type CandidateMetrics,
|
||||
type FitnessComponents,
|
||||
type AcceptanceVerdict,
|
||||
type ShapeName,
|
||||
} from './types.js';
|
||||
|
||||
/** Per-shape selection result. */
|
||||
export interface ShapeSelectionResult {
|
||||
shape: ShapeName;
|
||||
bestCandidate: CandidateMetrics;
|
||||
bestFitness: FitnessComponents;
|
||||
acceptance: AcceptanceVerdict;
|
||||
/** All candidates evaluated for this shape (for audit log). */
|
||||
allCandidatesRanked: Array<{ candidate: CandidateMetrics; fitness: FitnessComponents }>;
|
||||
}
|
||||
|
||||
/** Aggregate run verdict per launch decision §F.2. */
|
||||
export interface RunAggregateVerdict {
|
||||
/** Number of shapes with positive trio_strict delta vs NULL-baseline. */
|
||||
shapesWithPositiveDelta: number;
|
||||
/** Total shapes evaluated. */
|
||||
totalShapes: number;
|
||||
/** §F.2 condition: ≥3/5 shapes show positive delta. */
|
||||
condition2Pass: boolean;
|
||||
/** Number of shapes where best candidate was ACCEPTED (passes §F.1 + not §F.5). */
|
||||
shapesAccepted: number;
|
||||
}
|
||||
|
||||
/** Full selection report — per shape + run aggregate. */
|
||||
export interface SelectionReport {
|
||||
perShape: ShapeSelectionResult[];
|
||||
runAggregate: RunAggregateVerdict;
|
||||
}
|
||||
|
||||
/** Inputs: per-shape candidates + per-shape NULL-baseline metrics + cost baselines. */
|
||||
export interface SelectionInputs {
|
||||
/** All candidates grouped by shape. */
|
||||
candidatesPerShape: Map<ShapeName, CandidateMetrics[]>;
|
||||
/** NULL-baseline trio_strict_pass_rate (op. ii) per shape. */
|
||||
baselineTrioStrictPassRateII: Map<ShapeName, number>;
|
||||
/** NULL-baseline median cost (USD per evaluation) per shape. */
|
||||
baselineMedianCostUsd: Map<ShapeName, number>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run top-1-per-shape selection + apply acceptance verdicts.
|
||||
*
|
||||
* Returns a complete selection report including per-shape results and the
|
||||
* run-aggregate verdict.
|
||||
*
|
||||
* Throws if a shape in candidatesPerShape lacks a corresponding baseline entry.
|
||||
*/
|
||||
export function runSelection(inputs: SelectionInputs): SelectionReport {
|
||||
const perShape: ShapeSelectionResult[] = [];
|
||||
|
||||
for (const [shape, candidates] of inputs.candidatesPerShape.entries()) {
|
||||
if (candidates.length === 0) {
|
||||
continue; // skip shapes with no candidates
|
||||
}
|
||||
|
||||
const baselineRate = inputs.baselineTrioStrictPassRateII.get(shape);
|
||||
if (baselineRate === undefined) {
|
||||
throw new Error(`runSelection: missing baseline trio_strict rate for shape "${shape}"`);
|
||||
}
|
||||
const baselineCost = inputs.baselineMedianCostUsd.get(shape);
|
||||
if (baselineCost === undefined) {
|
||||
throw new Error(`runSelection: missing baseline median cost for shape "${shape}"`);
|
||||
}
|
||||
|
||||
// Compute fitness for all candidates of this shape
|
||||
const ranked = candidates
|
||||
.map(candidate => ({
|
||||
candidate,
|
||||
fitness: computeFitness({ candidate, baselineMedianCostUsd: baselineCost }),
|
||||
}))
|
||||
.sort((a, b) => b.fitness.fitness - a.fitness.fitness);
|
||||
|
||||
const top = ranked[0];
|
||||
const acceptance = evaluateCandidate({
|
||||
candidate: top.candidate,
|
||||
baselineTrioStrictPassRateII: baselineRate,
|
||||
});
|
||||
|
||||
perShape.push({
|
||||
shape,
|
||||
bestCandidate: top.candidate,
|
||||
bestFitness: top.fitness,
|
||||
acceptance,
|
||||
allCandidatesRanked: ranked,
|
||||
});
|
||||
}
|
||||
|
||||
const shapesWithPositiveDelta = perShape.filter(s => s.acceptance.trioStrictDeltaPP > 0).length;
|
||||
const shapesAccepted = perShape.filter(s => s.acceptance.accepted).length;
|
||||
const totalShapes = perShape.length;
|
||||
const condition2Pass = shapesWithPositiveDelta >= 3;
|
||||
|
||||
return {
|
||||
perShape,
|
||||
runAggregate: {
|
||||
shapesWithPositiveDelta,
|
||||
totalShapes,
|
||||
condition2Pass,
|
||||
shapesAccepted,
|
||||
},
|
||||
};
|
||||
}
|
||||
321
benchmarks/gepa/src/faza-1/types.ts
Normal file
321
benchmarks/gepa/src/faza-1/types.ts
Normal file
@@ -0,0 +1,321 @@
|
||||
/**
|
||||
* GEPA Faza 1 — shared type definitions.
|
||||
*
|
||||
* Per manifest v7 (SHA 583712dde139ffc87fb1ab21643f68d52c56469ded9e8090a624980b05969beb,
|
||||
* Amendment 2 supplemented) and launch decision §A inherited rules.
|
||||
*
|
||||
* Cell semantic boundary linchpin: MULTI_STEP_ACTION_CONTRACT in
|
||||
* packages/agent/src/prompt-shapes/types.ts (SHA 70a1701d...).
|
||||
*
|
||||
* DO NOT modify this file as part of GEPA candidate evolution. This file is
|
||||
* scaffold-only; the mutation surface is the prompt-shape body methods in
|
||||
* packages/agent/src/prompt-shapes/{claude,qwen-thinking,qwen-non-thinking,gpt,generic-simple}.ts.
|
||||
*/
|
||||
|
||||
/** The 5 shape names targeted by Faza 1 GEPA evolution. */
|
||||
export type ShapeName =
|
||||
| 'claude'
|
||||
| 'qwen-thinking'
|
||||
| 'qwen-non-thinking'
|
||||
| 'gpt'
|
||||
| 'generic-simple';
|
||||
|
||||
/**
|
||||
* Shapes that get retrieval-engagement weighting per Amendment 2 §3.
|
||||
* Phase 4.5 finding: only Qwen exhibits the under-engagement gap.
|
||||
*/
|
||||
export const QWEN_TARGETED_SHAPES: ReadonlySet<ShapeName> = new Set<ShapeName>([
|
||||
'qwen-thinking',
|
||||
'qwen-non-thinking',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Shapes that use baseline fitness (no retrieval-engagement weight).
|
||||
* Per Amendment 2 §3: these shapes don't have the gap, so applying the
|
||||
* bonus uniformly would distort their fitness measurement.
|
||||
*/
|
||||
export const NON_QWEN_SHAPES: ReadonlySet<ShapeName> = new Set<ShapeName>([
|
||||
'claude',
|
||||
'gpt',
|
||||
'generic-simple',
|
||||
]);
|
||||
|
||||
/** Per-task evaluation result for one candidate × one instance. */
|
||||
export interface EvaluationResult {
|
||||
/** Stable instance identifier (e.g., from corpus generator). */
|
||||
instanceId: string;
|
||||
|
||||
/** Trio judge mean (per pilot runner line 654 — arithmetic mean of valid judge means). */
|
||||
trioMean: number;
|
||||
|
||||
/**
|
||||
* Trio strict pass per metric_operationalization (ii) — primary acceptance.
|
||||
* `trioMean >= 4.0` per Amendment 1 Ask B ratification.
|
||||
*/
|
||||
trioStrictPassII: boolean;
|
||||
|
||||
/**
|
||||
* Trio strict pass per metric_operationalization (i) — supplementary.
|
||||
* `>= 2 of 3 judges with judge.mean >= 3.5` per pilot runner line 657.
|
||||
* Reported in parallel for cross-validation against pilot baseline.
|
||||
*/
|
||||
trioStrictPassI: boolean;
|
||||
|
||||
/** Number of retrieve actions issued during this evaluation (existing telemetry). */
|
||||
retrievalCalls: number;
|
||||
|
||||
/** Subject + judge cumulative cost (USD) for this evaluation. */
|
||||
costUsd: number;
|
||||
}
|
||||
|
||||
/** Aggregated per-candidate metrics across N=8 evaluations (per shape). */
|
||||
export interface CandidateMetrics {
|
||||
/** Stable candidate identifier (e.g., shape name + generation + variant). */
|
||||
candidateId: string;
|
||||
|
||||
/** Which shape this candidate belongs to. */
|
||||
shape: ShapeName;
|
||||
|
||||
/** All N=8 evaluation results for this candidate. */
|
||||
evaluations: EvaluationResult[];
|
||||
|
||||
/** Pass rate per operationalization (ii) — primary. Range [0, 1]. */
|
||||
trioStrictPassRateII: number;
|
||||
|
||||
/** Pass rate per operationalization (i) — supplementary. Range [0, 1]. */
|
||||
trioStrictPassRateI: number;
|
||||
|
||||
/** Mean retrieval_calls across evaluations (per task). */
|
||||
meanRetrievalCallsPerTask: number;
|
||||
|
||||
/** Mean cost (USD) across evaluations. */
|
||||
meanCostUsd: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-shape fitness components per Amendment 2 §3.
|
||||
*
|
||||
* For Qwen-targeted shapes:
|
||||
* fitness = trio_strict_pass_rate + retrieval_engagement_bonus - cost_penalty
|
||||
*
|
||||
* For non-Qwen shapes:
|
||||
* fitness = trio_strict_pass_rate - cost_penalty
|
||||
* (retrievalEngagementBonus is always 0 for non-Qwen — not added to fitness)
|
||||
*/
|
||||
export interface FitnessComponents {
|
||||
/** trio_strict_pass_rate per operationalization (ii). Range [0, 1]. */
|
||||
trioStrictPassRateII: number;
|
||||
|
||||
/**
|
||||
* Retrieval engagement bonus per Amendment 2 §3 bands:
|
||||
* +0.05 if mean retrieval_calls per task >= 2.0
|
||||
* 0.00 if mean retrieval_calls per task in [1.5, 2.0)
|
||||
* -0.05 if mean retrieval_calls per task < 1.5
|
||||
* Always 0.0 for non-Qwen shapes (these shapes don't have the gap).
|
||||
*/
|
||||
retrievalEngagementBonus: number;
|
||||
|
||||
/**
|
||||
* Cost penalty per brief §3.1 — −0.5pp per $0.10 above per-shape baseline median.
|
||||
* Encoded as decimal (e.g., 0.005 = 0.5pp).
|
||||
*/
|
||||
costPenalty: number;
|
||||
|
||||
/** Aggregate fitness (sum of above with sign convention: bonus +, penalty −). */
|
||||
fitness: number;
|
||||
|
||||
/**
|
||||
* Whether retrieval engagement was applied (true for Qwen-targeted shapes,
|
||||
* false otherwise). Useful for downstream auditing + report generation.
|
||||
*/
|
||||
retrievalEngagementApplied: boolean;
|
||||
}
|
||||
|
||||
/** Inputs for the per-shape fitness function. */
|
||||
export interface FitnessInputs {
|
||||
/** Aggregated metrics for this candidate. */
|
||||
candidate: CandidateMetrics;
|
||||
|
||||
/**
|
||||
* Per-shape baseline median cost (USD per evaluation). Used for cost penalty
|
||||
* computation. Typically the NULL-baseline median for the same shape.
|
||||
*/
|
||||
baselineMedianCostUsd: number;
|
||||
}
|
||||
|
||||
/** Acceptance verdict for a single candidate per §F + §F.5 of launch decision. */
|
||||
export interface AcceptanceVerdict {
|
||||
/** Whether candidate passes §F condition 1 (trio_strict delta + Qwen retrieval floor). */
|
||||
condition1Pass: boolean;
|
||||
|
||||
/**
|
||||
* §F.5 false-positive guard — REJECTED if Qwen candidate has +5pp trio delta
|
||||
* but mean retrieval_calls < 1.5 (Amendment 2 §5).
|
||||
*/
|
||||
condition5FalsePositiveGuardTriggered: boolean;
|
||||
|
||||
/** Overall acceptance for this candidate (must pass condition 1 AND not trigger §F.5). */
|
||||
accepted: boolean;
|
||||
|
||||
/** Detailed reason string for audit log. */
|
||||
reason: string;
|
||||
|
||||
/**
|
||||
* Computed delta vs NULL-baseline trio_strict_pass_rate (percentage points).
|
||||
* Positive = improvement.
|
||||
*/
|
||||
trioStrictDeltaPP: number;
|
||||
}
|
||||
|
||||
/** Inputs for the acceptance validator. */
|
||||
export interface AcceptanceInputs {
|
||||
/** Candidate under evaluation. */
|
||||
candidate: CandidateMetrics;
|
||||
|
||||
/** NULL-baseline trio_strict_pass_rate (op. (ii)) for the same shape. */
|
||||
baselineTrioStrictPassRateII: number;
|
||||
}
|
||||
|
||||
// ── Amendment 7 — tiered fitness + Δ-floor types ───────────────────────────
|
||||
|
||||
/**
|
||||
* Per-shape NULL-baseline anchors pinned from Checkpoint A v2 §B.2 (manifest
|
||||
* v7 Amendment 6 binding SHA 0b55d8e353...).
|
||||
*
|
||||
* These anchor:
|
||||
* - Tier 1 baseline (NULL pass rate per shape)
|
||||
* - Tier 2 baseline (NULL retrieval engagement per shape)
|
||||
* - §F.1 acceptance gate ≥+5pp delta basis
|
||||
* - Δ-floor threshold 2 per-shape comparison
|
||||
*
|
||||
* Source: real per-shape data from re-run NULL-baseline post Amendment 6
|
||||
* promptShapeOverride bug fix (run bhe0zwi91, 40/40 evals, 2026-04-28).
|
||||
*/
|
||||
export const NULL_BASELINE_PER_SHAPE: Readonly<
|
||||
Record<ShapeName, { trioStrictPassRateII: number; meanRetrievalCallsPerTask: number }>
|
||||
> = {
|
||||
claude: { trioStrictPassRateII: 0.875, meanRetrievalCallsPerTask: 1.12 },
|
||||
'qwen-thinking': { trioStrictPassRateII: 0.875, meanRetrievalCallsPerTask: 1.12 },
|
||||
'qwen-non-thinking': { trioStrictPassRateII: 1.000, meanRetrievalCallsPerTask: 1.25 },
|
||||
gpt: { trioStrictPassRateII: 0.750, meanRetrievalCallsPerTask: 1.00 },
|
||||
'generic-simple': { trioStrictPassRateII: 0.875, meanRetrievalCallsPerTask: 1.12 },
|
||||
} as const;
|
||||
|
||||
/** NULL-baseline aggregate across all 5 shapes (Checkpoint A v2 §B.2). */
|
||||
export const NULL_BASELINE_AGGREGATE = {
|
||||
trioStrictPassRateII: 0.875, // 35/40
|
||||
meanRetrievalCallsPerTask: 1.12,
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Tiered fitness components per Amendment 7 §fitness_function_tiered.
|
||||
*
|
||||
* Saturated regime (NULL pass rate ≥75% for ≥4/5 shapes — current Faza 1 state)
|
||||
* makes Tier 1 (NULL delta) noise-bound on N=8 binomial. Tier 2 + Tier 3 act as
|
||||
* primary differentiators; Tier 1 retained as TIE-BREAKER + acceptance gate.
|
||||
*/
|
||||
export interface TieredFitnessComponents {
|
||||
/**
|
||||
* Tier 1 — NULL pass rate delta (signed, percentage points).
|
||||
* Acceptance gate threshold: ≥+5pp (launch decision §F.1, UNCHANGED).
|
||||
* Role in saturated regime: TIE_BREAKER (noise-bound at N=8).
|
||||
*/
|
||||
tier1DeltaPP: number;
|
||||
|
||||
/**
|
||||
* Tier 2 — continuous retrieval engagement bonus (Qwen-targeted only).
|
||||
* Formula: clamp(0.05 × delta_pp, 0, 0.25) where delta_pp = (candidate − baseline) × 100.
|
||||
* Always 0 for non-Qwen shapes.
|
||||
* Role in saturated regime: PRIMARY differentiator for Qwen-targeted shapes.
|
||||
*/
|
||||
tier2RetrievalBonus: number;
|
||||
|
||||
/**
|
||||
* Tier 3 — binary cell-semantic anchor invariance bonus.
|
||||
* 0.10 if all 7 anchors invariant (mutation-validator passed); 0 otherwise.
|
||||
* Role in saturated regime: SECONDARY differentiator (substrate-preservation proxy).
|
||||
*/
|
||||
tier3CellSemanticInvarianceBonus: number;
|
||||
|
||||
/**
|
||||
* Aggregate fitness in saturated regime: tier_2 + tier_3.
|
||||
* Tier 1 reserved as tie-breaker (NOT in this aggregate).
|
||||
* Cost penalty per Amendment 2 NOT in tiered ranking aggregate (supplementary diagnostic).
|
||||
*/
|
||||
aggregateSaturatedRegime: number;
|
||||
|
||||
/** Whether saturated regime applies (per Amendment 7 §fitness_function_tiered.saturated_regime_definition). */
|
||||
saturatedRegimeApplied: boolean;
|
||||
|
||||
/** Cell-semantic anchor invariance count (0..7) for audit reporting. */
|
||||
cellSemanticAnchorInvarianceCount: number;
|
||||
}
|
||||
|
||||
/** Inputs for tiered fitness. */
|
||||
export interface TieredFitnessInputs {
|
||||
/** Aggregated metrics for this candidate. */
|
||||
candidate: CandidateMetrics;
|
||||
|
||||
/** Per-shape NULL-baseline pass rate (op. ii). Typically NULL_BASELINE_PER_SHAPE[shape].trioStrictPassRateII. */
|
||||
nullBaselinePassRateII: number;
|
||||
|
||||
/** Per-shape NULL-baseline mean retrieval calls per task. Typically NULL_BASELINE_PER_SHAPE[shape].meanRetrievalCallsPerTask. */
|
||||
nullBaselineMeanRetrievalCallsPerTask: number;
|
||||
|
||||
/** Whether the candidate's mutation-validator verdict was VALID (all 7 anchors invariant). */
|
||||
mutationValidatorPassed: boolean;
|
||||
|
||||
/**
|
||||
* Whether the saturated regime applies (≥4/5 shapes have NULL pass rate ≥75%).
|
||||
* Caller computes from Checkpoint A v2 data; default true for Faza 1.
|
||||
*/
|
||||
saturatedRegime: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Δ-floor verdict per Amendment 7 §gen_1_pre_registered_delta_floor.
|
||||
* Pre-registered Gen 1 floor for "evolution worked at all" — looser than
|
||||
* §F.1 acceptance gate. Three OR-gated thresholds.
|
||||
*/
|
||||
export interface DeltaFloorVerdict {
|
||||
/** Threshold 1: aggregate Tier 1 delta ≥+3pp. */
|
||||
threshold1AggregateTier1: 'PASS' | 'FAIL';
|
||||
/** Aggregate Tier 1 delta value (pp, signed) — for audit. */
|
||||
threshold1ValuePP: number;
|
||||
|
||||
/** Threshold 2: max Qwen-shape retrieval engagement delta ≥+0.10 absolute. */
|
||||
threshold2QwenRetrievalAbsolute: 'PASS' | 'FAIL';
|
||||
/** Max delta across Qwen shapes (absolute) — for audit. */
|
||||
threshold2MaxDeltaAbsolute: number;
|
||||
|
||||
/** Threshold 3: (Tier 1 ≥0pp) AND (Tier 2 ≥0.05). */
|
||||
threshold3CompoundTier1PlusTier2: 'PASS' | 'FAIL';
|
||||
/** Aggregate Tier 1 delta (pp, signed) — for audit. */
|
||||
threshold3Tier1ValuePP: number;
|
||||
/** Aggregate Tier 2 bonus across Qwen-targeted candidates — for audit. */
|
||||
threshold3Tier2Aggregate: number;
|
||||
|
||||
/** Overall verdict: PROCEED if ANY threshold passes; HALT_INVESTIGATE if ALL fail. */
|
||||
overallVerdict: 'PROCEED' | 'HALT_INVESTIGATE';
|
||||
}
|
||||
|
||||
/** Inputs for Δ-floor verdict computation. */
|
||||
export interface DeltaFloorInputs {
|
||||
/** Aggregate trio_strict_pass_rate_II across all evaluated candidates × all evals (in saturated regime, mean across 30 Checkpoint B evals). */
|
||||
aggregateTrioStrictPassRateII: number;
|
||||
|
||||
/** Aggregate NULL-baseline pass rate (e.g., 0.875 = NULL_BASELINE_AGGREGATE.trioStrictPassRateII). */
|
||||
aggregateNullBaselinePassRateII: number;
|
||||
|
||||
/**
|
||||
* Per-shape mean retrieval calls per task for Qwen-targeted candidates.
|
||||
* Map: shape → meanRetrievalCallsPerTask. Empty entries treated as no-data (skipped).
|
||||
*/
|
||||
qwenShapeRetrievalMeans: Partial<Record<ShapeName, number>>;
|
||||
|
||||
/** Per-shape NULL baseline retrieval means (typically NULL_BASELINE_PER_SHAPE[shape].meanRetrievalCallsPerTask). */
|
||||
qwenShapeNullBaselineRetrievalMeans: Partial<Record<ShapeName, number>>;
|
||||
|
||||
/** Aggregate Tier 2 retrieval engagement bonus across Qwen-targeted candidates (mean). */
|
||||
qwenAggregateTier2Bonus: number;
|
||||
}
|
||||
Reference in New Issue
Block a user