155 lines
5.7 KiB
TypeScript
155 lines
5.7 KiB
TypeScript
/**
|
|
* GEPA Faza 1 — acceptance validator.
|
|
*
|
|
* Per launch decision §F (4 must-hold conditions) + §F.5 (Amendment 2 §5
|
|
* false-positive evolution guard).
|
|
*
|
|
* Per-candidate verdict logic:
|
|
*
|
|
* §F condition 1 (third update — Amendment 2 §5):
|
|
* "Best GEPA candidate per shape beats NULL-baseline by ≥+5pp on
|
|
* trio_strict_pass rate (trio_mean ≥ 4.0). For Qwen-targeted shapes,
|
|
* additionally: best candidate must have mean retrieval_calls per task
|
|
* ≥ 1.7 (engagement gap closed by ≥50% relative to Qwen baseline 1.33)."
|
|
*
|
|
* §F.5 (NEW per Amendment 2 — false-positive guard):
|
|
* "If best Qwen-shape candidate achieves +5pp trio_strict delta WITHOUT
|
|
* closing retrieval engagement gap (mean retrieval_calls < 1.5), this
|
|
* signals false-positive evolution. Result: candidate REJECTED, shape
|
|
* marked FAIL even if other criteria pass."
|
|
*
|
|
* Conditions §F.2/§F.3/§F.4 are evaluated at the run-aggregate level (across
|
|
* shapes / κ stability across all evaluations / mutation validator log) and
|
|
* are NOT checked per-candidate here. See selection.ts + run-orchestrator
|
|
* for those.
|
|
*/
|
|
|
|
import {
|
|
type AcceptanceInputs,
|
|
type AcceptanceVerdict,
|
|
QWEN_TARGETED_SHAPES,
|
|
} from './types.js';
|
|
|
|
/** §F condition 1 trio_strict delta threshold (percentage points). */
|
|
export const TRIO_STRICT_DELTA_THRESHOLD_PP = 5;
|
|
|
|
/**
|
|
* §F condition 1 Qwen-only retrieval engagement floor (mean retrieval_calls
|
|
* per task). 1.7 = 50% gap closure between Qwen baseline 1.33 and Opus
|
|
* parity 2.33 per Amendment 2 §5.
|
|
*/
|
|
export const QWEN_RETRIEVAL_ENGAGEMENT_FLOOR = 1.7;
|
|
|
|
/**
|
|
* §F.5 false-positive guard threshold (mean retrieval_calls per task).
|
|
* If a Qwen candidate achieves trio_strict delta but stays below this floor,
|
|
* it is REJECTED as false-positive evolution per Amendment 2 §5.
|
|
*/
|
|
export const QWEN_FALSE_POSITIVE_RETRIEVAL_FLOOR = 1.5;
|
|
|
|
/**
|
|
* Floating-point tolerance for threshold comparisons. 1e-9 is well below
|
|
* any signal magnitude in the +/-0.05 fitness band (which is itself ~9
|
|
* orders of magnitude larger). Required because IEEE 754 makes
|
|
* `(0.25 - 0.20) * 100 = 4.999999999999999` rather than exact 5.0.
|
|
*/
|
|
const EPSILON = 1e-9;
|
|
|
|
/**
|
|
* Compute acceptance verdict for a single candidate per launch decision §F + §F.5.
|
|
*
|
|
* Caller is responsible for §F.2 (≥3/5 shapes positive delta), §F.3 (κ stability),
|
|
* and §F.4 (zero cell semantic violations) at the run-aggregate level.
|
|
*/
|
|
export function evaluateCandidate(inputs: AcceptanceInputs): AcceptanceVerdict {
|
|
const { candidate, baselineTrioStrictPassRateII } = inputs;
|
|
|
|
// Compute delta in percentage points (scale 0..100)
|
|
const trioStrictDeltaPP =
|
|
(candidate.trioStrictPassRateII - baselineTrioStrictPassRateII) * 100;
|
|
|
|
const isQwenTargeted = QWEN_TARGETED_SHAPES.has(candidate.shape);
|
|
|
|
// §F.5 false-positive guard — applies ONLY to Qwen-targeted shapes
|
|
// and ONLY when trio_strict delta meets the +5pp threshold.
|
|
// Per Amendment 2 §5: if delta ≥ +5pp AND retrieval_calls < 1.5 → REJECT.
|
|
// EPSILON tolerance handles IEEE 754 precision on exact-boundary deltas.
|
|
let condition5FalsePositiveGuardTriggered = false;
|
|
if (
|
|
isQwenTargeted &&
|
|
trioStrictDeltaPP >= TRIO_STRICT_DELTA_THRESHOLD_PP - EPSILON &&
|
|
candidate.meanRetrievalCallsPerTask < QWEN_FALSE_POSITIVE_RETRIEVAL_FLOOR - EPSILON
|
|
) {
|
|
condition5FalsePositiveGuardTriggered = true;
|
|
}
|
|
|
|
// §F condition 1 — trio_strict delta + (Qwen only) retrieval engagement floor
|
|
let condition1Pass = trioStrictDeltaPP >= TRIO_STRICT_DELTA_THRESHOLD_PP - EPSILON;
|
|
if (condition1Pass && isQwenTargeted) {
|
|
// Qwen sub-criterion: mean retrieval_calls ≥ 1.7
|
|
if (candidate.meanRetrievalCallsPerTask < QWEN_RETRIEVAL_ENGAGEMENT_FLOOR - EPSILON) {
|
|
condition1Pass = false;
|
|
}
|
|
}
|
|
|
|
// Overall acceptance: condition 1 must pass AND §F.5 must not trigger
|
|
const accepted = condition1Pass && !condition5FalsePositiveGuardTriggered;
|
|
|
|
// Reason string for audit log
|
|
const reason = buildReason({
|
|
candidate,
|
|
trioStrictDeltaPP,
|
|
condition1Pass,
|
|
condition5FalsePositiveGuardTriggered,
|
|
isQwenTargeted,
|
|
accepted,
|
|
});
|
|
|
|
return {
|
|
condition1Pass,
|
|
condition5FalsePositiveGuardTriggered,
|
|
accepted,
|
|
reason,
|
|
trioStrictDeltaPP,
|
|
};
|
|
}
|
|
|
|
interface BuildReasonInputs {
|
|
candidate: { shape: string; meanRetrievalCallsPerTask: number };
|
|
trioStrictDeltaPP: number;
|
|
condition1Pass: boolean;
|
|
condition5FalsePositiveGuardTriggered: boolean;
|
|
isQwenTargeted: boolean;
|
|
accepted: boolean;
|
|
}
|
|
|
|
function buildReason(i: BuildReasonInputs): string {
|
|
const parts: string[] = [];
|
|
parts.push(`shape=${i.candidate.shape}`);
|
|
parts.push(`trio_strict_delta=${i.trioStrictDeltaPP.toFixed(2)}pp`);
|
|
parts.push(`mean_retrieval_calls=${i.candidate.meanRetrievalCallsPerTask.toFixed(2)}`);
|
|
|
|
if (i.condition5FalsePositiveGuardTriggered) {
|
|
parts.push(
|
|
`REJECTED §F.5 false-positive guard: delta ≥ +${TRIO_STRICT_DELTA_THRESHOLD_PP}pp AND retrieval < ${QWEN_FALSE_POSITIVE_RETRIEVAL_FLOOR}`,
|
|
);
|
|
} else if (!i.condition1Pass) {
|
|
// Use same EPSILON tolerance as condition1Pass evaluation to keep root-cause
|
|
// attribution consistent with the gating logic on exact-boundary deltas.
|
|
if (i.trioStrictDeltaPP < TRIO_STRICT_DELTA_THRESHOLD_PP - EPSILON) {
|
|
parts.push(
|
|
`FAIL §F.1 trio_strict delta: ${i.trioStrictDeltaPP.toFixed(2)}pp < ${TRIO_STRICT_DELTA_THRESHOLD_PP}pp`,
|
|
);
|
|
} else if (i.isQwenTargeted) {
|
|
parts.push(
|
|
`FAIL §F.1 Qwen retrieval floor: ${i.candidate.meanRetrievalCallsPerTask.toFixed(2)} < ${QWEN_RETRIEVAL_ENGAGEMENT_FLOOR}`,
|
|
);
|
|
}
|
|
} else {
|
|
parts.push(`PASS §F.1${i.isQwenTargeted ? ' (Qwen retrieval floor met)' : ''}`);
|
|
}
|
|
|
|
parts.push(`accepted=${i.accepted}`);
|
|
return parts.join(' | ');
|
|
}
|