Files
waggle-os/benchmarks/gepa/src/faza-1/fitness.ts
Oleg Maslov 0c3e2ead3b
Some checks failed
Installer Smoke / installer-smoke (push) Has been cancelled
moving
2026-09-02 10:10:29 +02:00

299 lines
12 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* GEPA Faza 1 — per-shape fitness function.
*
* Per Amendment 2 §3 (manifest v7 §metric_operationalization.per_shape_fitness_formula
* + §metric_operationalization.retrieval_engagement_bonus).
*
* Phase 4.5 empirical signal: Qwen retrieves 1.33×/task vs Opus 2.33×/task.
* H4 score gap mechanistically traces to under-engagement. Mutation surface
* (prompt-shape body) is the lever to address this on Qwen-targeted shapes.
*
* Fitness function forks by shape class:
* Qwen-targeted: fitness = trio_strict_pass_rate + retrieval_engagement_bonus − cost_penalty
* Non-Qwen: fitness = trio_strict_pass_rate − cost_penalty
*
* Retrieval engagement bonus bands (Qwen-targeted shapes only):
* +0.05 if mean retrieval_calls per task ≥ 2.0 (Opus parity proxy)
* 0.00 if mean retrieval_calls per task ∈ [1.5, 2.0)
* −0.05 if mean retrieval_calls per task < 1.5 (Qwen baseline penalty)
*/
import {
type FitnessInputs,
type FitnessComponents,
type ShapeName,
type TieredFitnessInputs,
type TieredFitnessComponents,
type DeltaFloorInputs,
type DeltaFloorVerdict,
QWEN_TARGETED_SHAPES,
} from './types.js';
/** Cost penalty coefficient per brief §3.1 — 0.5pp per $0.10 above baseline median. */
const COST_PENALTY_PP_PER_DOLLAR_10C = 0.5;
/** Retrieval engagement thresholds per Amendment 2 §3 bands. */
export const RETRIEVAL_ENGAGEMENT_BANDS = {
/** ≥ this → +0.05 bonus (Opus parity proxy). */
upperThreshold: 2.0,
/** ≥ this and < upper → 0.00. */
lowerThreshold: 1.5,
/** < lower → −0.05 (Qwen baseline penalty). */
bonusPlus: 0.05,
bonusZero: 0.0,
bonusMinus: -0.05,
} as const;
/**
* Compute the retrieval engagement bonus for a Qwen-targeted shape.
*
* Returns 0.0 for non-Qwen shapes (callers should branch on shape class
* before calling this if they need to distinguish; alternatively use
* `computeFitness` which handles routing).
*
* Boundary semantics (binding per Amendment 2 §8 test cases):
* 1.49 → −0.05
* 1.50 → 0.00 (exact lower threshold = zero band)
* 1.99 → 0.00
* 2.00 → +0.05 (exact upper threshold = bonus band)
* 2.50 → +0.05
*/
export function computeRetrievalEngagementBonus(
shape: ShapeName,
meanRetrievalCallsPerTask: number,
): number {
if (!QWEN_TARGETED_SHAPES.has(shape)) {
return 0.0;
}
if (meanRetrievalCallsPerTask >= RETRIEVAL_ENGAGEMENT_BANDS.upperThreshold) {
return RETRIEVAL_ENGAGEMENT_BANDS.bonusPlus;
}
if (meanRetrievalCallsPerTask >= RETRIEVAL_ENGAGEMENT_BANDS.lowerThreshold) {
return RETRIEVAL_ENGAGEMENT_BANDS.bonusZero;
}
return RETRIEVAL_ENGAGEMENT_BANDS.bonusMinus;
}
/**
* Compute the cost penalty per brief §3.1.
*
* If the candidate's mean cost is at or below the per-shape baseline median,
* penalty is 0. Otherwise, penalty = 0.5pp per $0.10 of overage, encoded as
* a positive decimal (caller subtracts from fitness).
*/
export function computeCostPenalty(
candidateMeanCostUsd: number,
baselineMedianCostUsd: number,
): number {
const overageUsd = candidateMeanCostUsd - baselineMedianCostUsd;
if (overageUsd <= 0) {
return 0.0;
}
// 0.5 pp per $0.10 → 5 pp per $1.00 → encoded as 0.05 per $1.00 → 0.005 per $0.10
const penaltyDecimal = (overageUsd / 0.10) * (COST_PENALTY_PP_PER_DOLLAR_10C / 100);
return penaltyDecimal;
}
/**
* Compute the per-shape fitness for a candidate.
*
* Routes by shape class:
* - Qwen-targeted (qwen-thinking, qwen-non-thinking): includes retrieval engagement bonus
* - Non-Qwen (claude, gpt, generic-simple): no retrieval engagement weighting
*
* Returns full FitnessComponents for downstream audit + reporting per
* launch decision §A.9 binding compliance requirement.
*/
export function computeFitness(inputs: FitnessInputs): FitnessComponents {
const { candidate, baselineMedianCostUsd } = inputs;
const trioStrictPassRateII = candidate.trioStrictPassRateII;
const retrievalEngagementApplied = QWEN_TARGETED_SHAPES.has(candidate.shape);
const retrievalEngagementBonus = retrievalEngagementApplied
? computeRetrievalEngagementBonus(candidate.shape, candidate.meanRetrievalCallsPerTask)
: 0.0;
const costPenalty = computeCostPenalty(candidate.meanCostUsd, baselineMedianCostUsd);
const fitness = trioStrictPassRateII + retrievalEngagementBonus - costPenalty;
return {
trioStrictPassRateII,
retrievalEngagementBonus,
costPenalty,
fitness,
retrievalEngagementApplied,
};
}
// ── Amendment 7 — tiered fitness function ─────────────────────────────────
/** Tier 2 cap per Amendment 7 §fitness_function_tiered.tier_2 (max bonus). */
export const TIER_2_BONUS_CAP = 0.25;
/** Tier 2 weight per percentage point of retrieval engagement above NULL baseline. */
export const TIER_2_BONUS_PER_PP = 0.05;
/** Tier 3 binary bonus when all 7 cell-semantic anchors invariant. */
export const TIER_3_BONUS_FULL_INVARIANCE = 0.10;
/** Total cell-semantic anchor count (types.ts + MULTI_STEP_ACTION_CONTRACT + 5 baseline shapes). */
export const TIER_3_ANCHOR_COUNT_FULL = 7;
/**
* Compute Tier 2 retrieval engagement bonus per Amendment 7 §fitness_function_tiered.tier_2.
*
* Continuous formula (supersedes Amendment 2 band bonus for tiered ranking):
* bonus = clamp(0.05 × delta_pp, 0, 0.25)
* where delta_pp = (candidate_mean - baseline_mean) × 100
*
* Cap reached at +5pp absolute increase in mean retrieval calls per task.
* Floor 0 (no negative bonus from Tier 2 — negative-band penalty handled
* by Amendment 2 §F.5 false-positive guard separately).
*
* Always returns 0 for non-Qwen shapes.
*/
export function computeTier2RetrievalBonus(
shape: ShapeName,
candidateMeanRetrievalCallsPerTask: number,
baselineMeanRetrievalCallsPerTask: number,
): number {
if (!QWEN_TARGETED_SHAPES.has(shape)) return 0;
const deltaAbsolute = candidateMeanRetrievalCallsPerTask - baselineMeanRetrievalCallsPerTask;
if (deltaAbsolute <= 0) return 0;
const deltaPP = deltaAbsolute * 100;
return Math.min(TIER_2_BONUS_PER_PP * deltaPP, TIER_2_BONUS_CAP);
}
/**
* Compute the per-shape tiered fitness per Amendment 7 §fitness_function_tiered.
*
* Routes tier roles by saturation:
* - In saturated regime (≥4/5 shapes have NULL pass rate ≥75%):
* PRIMARY differentiator = Tier 2 (Qwen-targeted retrieval engagement)
* SECONDARY differentiator = Tier 3 (cell-semantic anchor invariance)
* TIE-BREAKER = Tier 1 (NULL pass rate delta)
* aggregateSaturatedRegime = tier_2 + tier_3
* - Outside saturated regime: legacy Amendment 2 form (computeFitness) applies
* and aggregateSaturatedRegime is reported but not the canonical fitness.
*
* Tier 1 also serves as the §F.1 acceptance gate (≥+5pp) — UNCHANGED from
* Amendment 5 launch decision §F.1.
*/
export function computeTieredFitness(inputs: TieredFitnessInputs): TieredFitnessComponents {
const {
candidate,
nullBaselinePassRateII,
nullBaselineMeanRetrievalCallsPerTask,
mutationValidatorPassed,
saturatedRegime,
} = inputs;
// Tier 1 — NULL pass rate delta (signed pp)
const tier1DeltaPP = (candidate.trioStrictPassRateII - nullBaselinePassRateII) * 100;
// Tier 2 — continuous retrieval engagement bonus (Qwen-targeted only)
const tier2RetrievalBonus = computeTier2RetrievalBonus(
candidate.shape,
candidate.meanRetrievalCallsPerTask,
nullBaselineMeanRetrievalCallsPerTask,
);
// Tier 3 — binary cell-semantic anchor invariance bonus
const tier3CellSemanticInvarianceBonus = mutationValidatorPassed
? TIER_3_BONUS_FULL_INVARIANCE
: 0;
// For Gen 1 candidates the mutation_validator gives binary (valid|invalid);
// count semantic: pass = full 7-anchor invariance, fail = 0.
const cellSemanticAnchorInvarianceCount = mutationValidatorPassed ? TIER_3_ANCHOR_COUNT_FULL : 0;
// Aggregate fitness in saturated regime: tier_2 + tier_3 only
const aggregateSaturatedRegime = tier2RetrievalBonus + tier3CellSemanticInvarianceBonus;
return {
tier1DeltaPP,
tier2RetrievalBonus,
tier3CellSemanticInvarianceBonus,
aggregateSaturatedRegime,
saturatedRegimeApplied: saturatedRegime,
cellSemanticAnchorInvarianceCount,
};
}
// ── Amendment 7 — pre-registered Δ-floor verdict ──────────────────────────
/** Δ-floor thresholds per Amendment 7 §gen_1_pre_registered_delta_floor. */
export const DELTA_FLOOR_THRESHOLDS = {
/** Threshold 1: aggregate Tier 1 delta in pp (≥+3pp passes). */
threshold1AggregateTier1PP: 3,
/** Threshold 2: max Qwen retrieval engagement delta absolute (≥+0.10 passes). */
threshold2QwenRetrievalAbsolute: 0.10,
/** Threshold 3a: aggregate Tier 1 delta in pp (≥0pp). */
threshold3Tier1MinPP: 0,
/** Threshold 3b: aggregate Tier 2 bonus across Qwen-targeted candidates (≥0.05). */
threshold3Tier2MinBonus: 0.05,
} as const;
/** Float tolerance for exact-boundary threshold comparisons (consistent with acceptance.ts EPSILON). */
const DELTA_FLOOR_EPSILON = 1e-9;
/**
* Compute the Δ-floor verdict per Amendment 7 §gen_1_pre_registered_delta_floor.
*
* Three OR-gated thresholds:
* 1. aggregate Tier 1 delta ≥ +3pp absolute (loosened from §F.1 ≥+5pp)
* 2. max Qwen-shape retrieval engagement delta ≥ +0.10 absolute above per-shape NULL baseline
* 3. (aggregate Tier 1 delta ≥ 0pp) AND (aggregate Tier 2 bonus ≥ 0.05)
*
* If ANY ONE passes → PROCEED (continue past Checkpoint B subject to PM ratify).
* If ALL THREE fail → HALT_INVESTIGATE (file Investigate report).
*
* Per Amendment 7 §3.2: "If Gen 1 fails ALL three thresholds → HALT before Gen 2,
* file Investigate report. If Gen 1 passes any one → proceed to Gen 2."
*/
export function computeDeltaFloorVerdict(inputs: DeltaFloorInputs): DeltaFloorVerdict {
// Threshold 1 — aggregate Tier 1 delta
const threshold1ValuePP =
(inputs.aggregateTrioStrictPassRateII - inputs.aggregateNullBaselinePassRateII) * 100;
const threshold1Pass =
threshold1ValuePP >= DELTA_FLOOR_THRESHOLDS.threshold1AggregateTier1PP - DELTA_FLOOR_EPSILON;
// Threshold 2 — max Qwen retrieval engagement delta absolute
let threshold2MaxDeltaAbsolute: number | null = null;
for (const [shapeKey, candidateMean] of Object.entries(inputs.qwenShapeRetrievalMeans)) {
if (candidateMean === undefined) continue;
const baselineMean = inputs.qwenShapeNullBaselineRetrievalMeans[shapeKey as ShapeName];
if (baselineMean === undefined) continue;
const delta = candidateMean - baselineMean;
if (threshold2MaxDeltaAbsolute === null || delta > threshold2MaxDeltaAbsolute) {
threshold2MaxDeltaAbsolute = delta;
}
}
// No Qwen data: report 0 delta (informative neutral); threshold2 fails since 0 < 0.10
if (threshold2MaxDeltaAbsolute === null) threshold2MaxDeltaAbsolute = 0;
const threshold2Pass =
threshold2MaxDeltaAbsolute >=
DELTA_FLOOR_THRESHOLDS.threshold2QwenRetrievalAbsolute - DELTA_FLOOR_EPSILON;
// Threshold 3 — compound Tier 1 ≥0 AND Tier 2 ≥0.05
const threshold3Tier1ValuePP = threshold1ValuePP;
const threshold3Tier1Pass =
threshold3Tier1ValuePP >= DELTA_FLOOR_THRESHOLDS.threshold3Tier1MinPP - DELTA_FLOOR_EPSILON;
const threshold3Tier2Pass =
inputs.qwenAggregateTier2Bonus >=
DELTA_FLOOR_THRESHOLDS.threshold3Tier2MinBonus - DELTA_FLOOR_EPSILON;
const threshold3Pass = threshold3Tier1Pass && threshold3Tier2Pass;
const overallVerdict: 'PROCEED' | 'HALT_INVESTIGATE' =
threshold1Pass || threshold2Pass || threshold3Pass ? 'PROCEED' : 'HALT_INVESTIGATE';
return {
threshold1AggregateTier1: threshold1Pass ? 'PASS' : 'FAIL',
threshold1ValuePP,
threshold2QwenRetrievalAbsolute: threshold2Pass ? 'PASS' : 'FAIL',
threshold2MaxDeltaAbsolute,
threshold3CompoundTier1PlusTier2: threshold3Pass ? 'PASS' : 'FAIL',
threshold3Tier1ValuePP,
threshold3Tier2Aggregate: inputs.qwenAggregateTier2Bonus,
overallVerdict,
};
}