299 lines
12 KiB
TypeScript
299 lines
12 KiB
TypeScript
/**
|
||
* GEPA Faza 1 — per-shape fitness function.
|
||
*
|
||
* Per Amendment 2 §3 (manifest v7 §metric_operationalization.per_shape_fitness_formula
|
||
* + §metric_operationalization.retrieval_engagement_bonus).
|
||
*
|
||
* Phase 4.5 empirical signal: Qwen retrieves 1.33×/task vs Opus 2.33×/task.
|
||
* H4 score gap mechanistically traces to under-engagement. Mutation surface
|
||
* (prompt-shape body) is the lever to address this on Qwen-targeted shapes.
|
||
*
|
||
* Fitness function forks by shape class:
|
||
* Qwen-targeted: fitness = trio_strict_pass_rate + retrieval_engagement_bonus − cost_penalty
|
||
* Non-Qwen: fitness = trio_strict_pass_rate − cost_penalty
|
||
*
|
||
* Retrieval engagement bonus bands (Qwen-targeted shapes only):
|
||
* +0.05 if mean retrieval_calls per task ≥ 2.0 (Opus parity proxy)
|
||
* 0.00 if mean retrieval_calls per task ∈ [1.5, 2.0)
|
||
* −0.05 if mean retrieval_calls per task < 1.5 (Qwen baseline penalty)
|
||
*/
|
||
|
||
import {
|
||
type FitnessInputs,
|
||
type FitnessComponents,
|
||
type ShapeName,
|
||
type TieredFitnessInputs,
|
||
type TieredFitnessComponents,
|
||
type DeltaFloorInputs,
|
||
type DeltaFloorVerdict,
|
||
QWEN_TARGETED_SHAPES,
|
||
} from './types.js';
|
||
|
||
/** Cost penalty coefficient per brief §3.1 — 0.5pp per $0.10 above baseline median. */
|
||
const COST_PENALTY_PP_PER_DOLLAR_10C = 0.5;
|
||
|
||
/** Retrieval engagement thresholds per Amendment 2 §3 bands. */
|
||
export const RETRIEVAL_ENGAGEMENT_BANDS = {
|
||
/** ≥ this → +0.05 bonus (Opus parity proxy). */
|
||
upperThreshold: 2.0,
|
||
/** ≥ this and < upper → 0.00. */
|
||
lowerThreshold: 1.5,
|
||
/** < lower → −0.05 (Qwen baseline penalty). */
|
||
bonusPlus: 0.05,
|
||
bonusZero: 0.0,
|
||
bonusMinus: -0.05,
|
||
} as const;
|
||
|
||
/**
|
||
* Compute the retrieval engagement bonus for a Qwen-targeted shape.
|
||
*
|
||
* Returns 0.0 for non-Qwen shapes (callers should branch on shape class
|
||
* before calling this if they need to distinguish; alternatively use
|
||
* `computeFitness` which handles routing).
|
||
*
|
||
* Boundary semantics (binding per Amendment 2 §8 test cases):
|
||
* 1.49 → −0.05
|
||
* 1.50 → 0.00 (exact lower threshold = zero band)
|
||
* 1.99 → 0.00
|
||
* 2.00 → +0.05 (exact upper threshold = bonus band)
|
||
* 2.50 → +0.05
|
||
*/
|
||
export function computeRetrievalEngagementBonus(
|
||
shape: ShapeName,
|
||
meanRetrievalCallsPerTask: number,
|
||
): number {
|
||
if (!QWEN_TARGETED_SHAPES.has(shape)) {
|
||
return 0.0;
|
||
}
|
||
if (meanRetrievalCallsPerTask >= RETRIEVAL_ENGAGEMENT_BANDS.upperThreshold) {
|
||
return RETRIEVAL_ENGAGEMENT_BANDS.bonusPlus;
|
||
}
|
||
if (meanRetrievalCallsPerTask >= RETRIEVAL_ENGAGEMENT_BANDS.lowerThreshold) {
|
||
return RETRIEVAL_ENGAGEMENT_BANDS.bonusZero;
|
||
}
|
||
return RETRIEVAL_ENGAGEMENT_BANDS.bonusMinus;
|
||
}
|
||
|
||
/**
|
||
* Compute the cost penalty per brief §3.1.
|
||
*
|
||
* If the candidate's mean cost is at or below the per-shape baseline median,
|
||
* penalty is 0. Otherwise, penalty = 0.5pp per $0.10 of overage, encoded as
|
||
* a positive decimal (caller subtracts from fitness).
|
||
*/
|
||
export function computeCostPenalty(
|
||
candidateMeanCostUsd: number,
|
||
baselineMedianCostUsd: number,
|
||
): number {
|
||
const overageUsd = candidateMeanCostUsd - baselineMedianCostUsd;
|
||
if (overageUsd <= 0) {
|
||
return 0.0;
|
||
}
|
||
// 0.5 pp per $0.10 → 5 pp per $1.00 → encoded as 0.05 per $1.00 → 0.005 per $0.10
|
||
const penaltyDecimal = (overageUsd / 0.10) * (COST_PENALTY_PP_PER_DOLLAR_10C / 100);
|
||
return penaltyDecimal;
|
||
}
|
||
|
||
/**
|
||
* Compute the per-shape fitness for a candidate.
|
||
*
|
||
* Routes by shape class:
|
||
* - Qwen-targeted (qwen-thinking, qwen-non-thinking): includes retrieval engagement bonus
|
||
* - Non-Qwen (claude, gpt, generic-simple): no retrieval engagement weighting
|
||
*
|
||
* Returns full FitnessComponents for downstream audit + reporting per
|
||
* launch decision §A.9 binding compliance requirement.
|
||
*/
|
||
export function computeFitness(inputs: FitnessInputs): FitnessComponents {
|
||
const { candidate, baselineMedianCostUsd } = inputs;
|
||
|
||
const trioStrictPassRateII = candidate.trioStrictPassRateII;
|
||
|
||
const retrievalEngagementApplied = QWEN_TARGETED_SHAPES.has(candidate.shape);
|
||
const retrievalEngagementBonus = retrievalEngagementApplied
|
||
? computeRetrievalEngagementBonus(candidate.shape, candidate.meanRetrievalCallsPerTask)
|
||
: 0.0;
|
||
|
||
const costPenalty = computeCostPenalty(candidate.meanCostUsd, baselineMedianCostUsd);
|
||
|
||
const fitness = trioStrictPassRateII + retrievalEngagementBonus - costPenalty;
|
||
|
||
return {
|
||
trioStrictPassRateII,
|
||
retrievalEngagementBonus,
|
||
costPenalty,
|
||
fitness,
|
||
retrievalEngagementApplied,
|
||
};
|
||
}
|
||
|
||
// ── Amendment 7 — tiered fitness function ─────────────────────────────────
|
||
|
||
/** Tier 2 cap per Amendment 7 §fitness_function_tiered.tier_2 (max bonus). */
|
||
export const TIER_2_BONUS_CAP = 0.25;
|
||
/** Tier 2 weight per percentage point of retrieval engagement above NULL baseline. */
|
||
export const TIER_2_BONUS_PER_PP = 0.05;
|
||
/** Tier 3 binary bonus when all 7 cell-semantic anchors invariant. */
|
||
export const TIER_3_BONUS_FULL_INVARIANCE = 0.10;
|
||
/** Total cell-semantic anchor count (types.ts + MULTI_STEP_ACTION_CONTRACT + 5 baseline shapes). */
|
||
export const TIER_3_ANCHOR_COUNT_FULL = 7;
|
||
|
||
/**
|
||
* Compute Tier 2 retrieval engagement bonus per Amendment 7 §fitness_function_tiered.tier_2.
|
||
*
|
||
* Continuous formula (supersedes Amendment 2 band bonus for tiered ranking):
|
||
* bonus = clamp(0.05 × delta_pp, 0, 0.25)
|
||
* where delta_pp = (candidate_mean - baseline_mean) × 100
|
||
*
|
||
* Cap reached at +5pp absolute increase in mean retrieval calls per task.
|
||
* Floor 0 (no negative bonus from Tier 2 — negative-band penalty handled
|
||
* by Amendment 2 §F.5 false-positive guard separately).
|
||
*
|
||
* Always returns 0 for non-Qwen shapes.
|
||
*/
|
||
export function computeTier2RetrievalBonus(
|
||
shape: ShapeName,
|
||
candidateMeanRetrievalCallsPerTask: number,
|
||
baselineMeanRetrievalCallsPerTask: number,
|
||
): number {
|
||
if (!QWEN_TARGETED_SHAPES.has(shape)) return 0;
|
||
const deltaAbsolute = candidateMeanRetrievalCallsPerTask - baselineMeanRetrievalCallsPerTask;
|
||
if (deltaAbsolute <= 0) return 0;
|
||
const deltaPP = deltaAbsolute * 100;
|
||
return Math.min(TIER_2_BONUS_PER_PP * deltaPP, TIER_2_BONUS_CAP);
|
||
}
|
||
|
||
/**
|
||
* Compute the per-shape tiered fitness per Amendment 7 §fitness_function_tiered.
|
||
*
|
||
* Routes tier roles by saturation:
|
||
* - In saturated regime (≥4/5 shapes have NULL pass rate ≥75%):
|
||
* PRIMARY differentiator = Tier 2 (Qwen-targeted retrieval engagement)
|
||
* SECONDARY differentiator = Tier 3 (cell-semantic anchor invariance)
|
||
* TIE-BREAKER = Tier 1 (NULL pass rate delta)
|
||
* aggregateSaturatedRegime = tier_2 + tier_3
|
||
* - Outside saturated regime: legacy Amendment 2 form (computeFitness) applies
|
||
* and aggregateSaturatedRegime is reported but not the canonical fitness.
|
||
*
|
||
* Tier 1 also serves as the §F.1 acceptance gate (≥+5pp) — UNCHANGED from
|
||
* Amendment 5 launch decision §F.1.
|
||
*/
|
||
export function computeTieredFitness(inputs: TieredFitnessInputs): TieredFitnessComponents {
|
||
const {
|
||
candidate,
|
||
nullBaselinePassRateII,
|
||
nullBaselineMeanRetrievalCallsPerTask,
|
||
mutationValidatorPassed,
|
||
saturatedRegime,
|
||
} = inputs;
|
||
|
||
// Tier 1 — NULL pass rate delta (signed pp)
|
||
const tier1DeltaPP = (candidate.trioStrictPassRateII - nullBaselinePassRateII) * 100;
|
||
|
||
// Tier 2 — continuous retrieval engagement bonus (Qwen-targeted only)
|
||
const tier2RetrievalBonus = computeTier2RetrievalBonus(
|
||
candidate.shape,
|
||
candidate.meanRetrievalCallsPerTask,
|
||
nullBaselineMeanRetrievalCallsPerTask,
|
||
);
|
||
|
||
// Tier 3 — binary cell-semantic anchor invariance bonus
|
||
const tier3CellSemanticInvarianceBonus = mutationValidatorPassed
|
||
? TIER_3_BONUS_FULL_INVARIANCE
|
||
: 0;
|
||
// For Gen 1 candidates the mutation_validator gives binary (valid|invalid);
|
||
// count semantic: pass = full 7-anchor invariance, fail = 0.
|
||
const cellSemanticAnchorInvarianceCount = mutationValidatorPassed ? TIER_3_ANCHOR_COUNT_FULL : 0;
|
||
|
||
// Aggregate fitness in saturated regime: tier_2 + tier_3 only
|
||
const aggregateSaturatedRegime = tier2RetrievalBonus + tier3CellSemanticInvarianceBonus;
|
||
|
||
return {
|
||
tier1DeltaPP,
|
||
tier2RetrievalBonus,
|
||
tier3CellSemanticInvarianceBonus,
|
||
aggregateSaturatedRegime,
|
||
saturatedRegimeApplied: saturatedRegime,
|
||
cellSemanticAnchorInvarianceCount,
|
||
};
|
||
}
|
||
|
||
// ── Amendment 7 — pre-registered Δ-floor verdict ──────────────────────────
|
||
|
||
/** Δ-floor thresholds per Amendment 7 §gen_1_pre_registered_delta_floor. */
|
||
export const DELTA_FLOOR_THRESHOLDS = {
|
||
/** Threshold 1: aggregate Tier 1 delta in pp (≥+3pp passes). */
|
||
threshold1AggregateTier1PP: 3,
|
||
/** Threshold 2: max Qwen retrieval engagement delta absolute (≥+0.10 passes). */
|
||
threshold2QwenRetrievalAbsolute: 0.10,
|
||
/** Threshold 3a: aggregate Tier 1 delta in pp (≥0pp). */
|
||
threshold3Tier1MinPP: 0,
|
||
/** Threshold 3b: aggregate Tier 2 bonus across Qwen-targeted candidates (≥0.05). */
|
||
threshold3Tier2MinBonus: 0.05,
|
||
} as const;
|
||
|
||
/** Float tolerance for exact-boundary threshold comparisons (consistent with acceptance.ts EPSILON). */
|
||
const DELTA_FLOOR_EPSILON = 1e-9;
|
||
|
||
/**
|
||
* Compute the Δ-floor verdict per Amendment 7 §gen_1_pre_registered_delta_floor.
|
||
*
|
||
* Three OR-gated thresholds:
|
||
* 1. aggregate Tier 1 delta ≥ +3pp absolute (loosened from §F.1 ≥+5pp)
|
||
* 2. max Qwen-shape retrieval engagement delta ≥ +0.10 absolute above per-shape NULL baseline
|
||
* 3. (aggregate Tier 1 delta ≥ 0pp) AND (aggregate Tier 2 bonus ≥ 0.05)
|
||
*
|
||
* If ANY ONE passes → PROCEED (continue past Checkpoint B subject to PM ratify).
|
||
* If ALL THREE fail → HALT_INVESTIGATE (file Investigate report).
|
||
*
|
||
* Per Amendment 7 §3.2: "If Gen 1 fails ALL three thresholds → HALT before Gen 2,
|
||
* file Investigate report. If Gen 1 passes any one → proceed to Gen 2."
|
||
*/
|
||
export function computeDeltaFloorVerdict(inputs: DeltaFloorInputs): DeltaFloorVerdict {
|
||
// Threshold 1 — aggregate Tier 1 delta
|
||
const threshold1ValuePP =
|
||
(inputs.aggregateTrioStrictPassRateII - inputs.aggregateNullBaselinePassRateII) * 100;
|
||
const threshold1Pass =
|
||
threshold1ValuePP >= DELTA_FLOOR_THRESHOLDS.threshold1AggregateTier1PP - DELTA_FLOOR_EPSILON;
|
||
|
||
// Threshold 2 — max Qwen retrieval engagement delta absolute
|
||
let threshold2MaxDeltaAbsolute: number | null = null;
|
||
for (const [shapeKey, candidateMean] of Object.entries(inputs.qwenShapeRetrievalMeans)) {
|
||
if (candidateMean === undefined) continue;
|
||
const baselineMean = inputs.qwenShapeNullBaselineRetrievalMeans[shapeKey as ShapeName];
|
||
if (baselineMean === undefined) continue;
|
||
const delta = candidateMean - baselineMean;
|
||
if (threshold2MaxDeltaAbsolute === null || delta > threshold2MaxDeltaAbsolute) {
|
||
threshold2MaxDeltaAbsolute = delta;
|
||
}
|
||
}
|
||
// No Qwen data: report 0 delta (informative neutral); threshold2 fails since 0 < 0.10
|
||
if (threshold2MaxDeltaAbsolute === null) threshold2MaxDeltaAbsolute = 0;
|
||
const threshold2Pass =
|
||
threshold2MaxDeltaAbsolute >=
|
||
DELTA_FLOOR_THRESHOLDS.threshold2QwenRetrievalAbsolute - DELTA_FLOOR_EPSILON;
|
||
|
||
// Threshold 3 — compound Tier 1 ≥0 AND Tier 2 ≥0.05
|
||
const threshold3Tier1ValuePP = threshold1ValuePP;
|
||
const threshold3Tier1Pass =
|
||
threshold3Tier1ValuePP >= DELTA_FLOOR_THRESHOLDS.threshold3Tier1MinPP - DELTA_FLOOR_EPSILON;
|
||
const threshold3Tier2Pass =
|
||
inputs.qwenAggregateTier2Bonus >=
|
||
DELTA_FLOOR_THRESHOLDS.threshold3Tier2MinBonus - DELTA_FLOOR_EPSILON;
|
||
const threshold3Pass = threshold3Tier1Pass && threshold3Tier2Pass;
|
||
|
||
const overallVerdict: 'PROCEED' | 'HALT_INVESTIGATE' =
|
||
threshold1Pass || threshold2Pass || threshold3Pass ? 'PROCEED' : 'HALT_INVESTIGATE';
|
||
|
||
return {
|
||
threshold1AggregateTier1: threshold1Pass ? 'PASS' : 'FAIL',
|
||
threshold1ValuePP,
|
||
threshold2QwenRetrievalAbsolute: threshold2Pass ? 'PASS' : 'FAIL',
|
||
threshold2MaxDeltaAbsolute,
|
||
threshold3CompoundTier1PlusTier2: threshold3Pass ? 'PASS' : 'FAIL',
|
||
threshold3Tier1ValuePP,
|
||
threshold3Tier2Aggregate: inputs.qwenAggregateTier2Bonus,
|
||
overallVerdict,
|
||
};
|
||
}
|