This commit is contained in:
154
benchmarks/gepa/src/faza-1/acceptance.ts
Normal file
154
benchmarks/gepa/src/faza-1/acceptance.ts
Normal file
@@ -0,0 +1,154 @@
|
||||
/**
|
||||
* GEPA Faza 1 — acceptance validator.
|
||||
*
|
||||
* Per launch decision §F (4 must-hold conditions) + §F.5 (Amendment 2 §5
|
||||
* false-positive evolution guard).
|
||||
*
|
||||
* Per-candidate verdict logic:
|
||||
*
|
||||
* §F condition 1 (third update — Amendment 2 §5):
|
||||
* "Best GEPA candidate per shape beats NULL-baseline by ≥+5pp on
|
||||
* trio_strict_pass rate (trio_mean ≥ 4.0). For Qwen-targeted shapes,
|
||||
* additionally: best candidate must have mean retrieval_calls per task
|
||||
* ≥ 1.7 (engagement gap closed by ≥50% relative to Qwen baseline 1.33)."
|
||||
*
|
||||
* §F.5 (NEW per Amendment 2 — false-positive guard):
|
||||
* "If best Qwen-shape candidate achieves +5pp trio_strict delta WITHOUT
|
||||
* closing retrieval engagement gap (mean retrieval_calls < 1.5), this
|
||||
* signals false-positive evolution. Result: candidate REJECTED, shape
|
||||
* marked FAIL even if other criteria pass."
|
||||
*
|
||||
* Conditions §F.2/§F.3/§F.4 are evaluated at the run-aggregate level (across
|
||||
* shapes / κ stability across all evaluations / mutation validator log) and
|
||||
* are NOT checked per-candidate here. See selection.ts + run-orchestrator
|
||||
* for those.
|
||||
*/
|
||||
|
||||
import {
|
||||
type AcceptanceInputs,
|
||||
type AcceptanceVerdict,
|
||||
QWEN_TARGETED_SHAPES,
|
||||
} from './types.js';
|
||||
|
||||
/** §F condition 1 trio_strict delta threshold (percentage points). */
|
||||
export const TRIO_STRICT_DELTA_THRESHOLD_PP = 5;
|
||||
|
||||
/**
|
||||
* §F condition 1 Qwen-only retrieval engagement floor (mean retrieval_calls
|
||||
* per task). 1.7 = 50% gap closure between Qwen baseline 1.33 and Opus
|
||||
* parity 2.33 per Amendment 2 §5.
|
||||
*/
|
||||
export const QWEN_RETRIEVAL_ENGAGEMENT_FLOOR = 1.7;
|
||||
|
||||
/**
|
||||
* §F.5 false-positive guard threshold (mean retrieval_calls per task).
|
||||
* If a Qwen candidate achieves trio_strict delta but stays below this floor,
|
||||
* it is REJECTED as false-positive evolution per Amendment 2 §5.
|
||||
*/
|
||||
export const QWEN_FALSE_POSITIVE_RETRIEVAL_FLOOR = 1.5;
|
||||
|
||||
/**
|
||||
* Floating-point tolerance for threshold comparisons. 1e-9 is well below
|
||||
* any signal magnitude in the +/-0.05 fitness band (which is itself ~9
|
||||
* orders of magnitude larger). Required because IEEE 754 makes
|
||||
* `(0.25 - 0.20) * 100 = 4.999999999999999` rather than exact 5.0.
|
||||
*/
|
||||
const EPSILON = 1e-9;
|
||||
|
||||
/**
|
||||
* Compute acceptance verdict for a single candidate per launch decision §F + §F.5.
|
||||
*
|
||||
* Caller is responsible for §F.2 (≥3/5 shapes positive delta), §F.3 (κ stability),
|
||||
* and §F.4 (zero cell semantic violations) at the run-aggregate level.
|
||||
*/
|
||||
export function evaluateCandidate(inputs: AcceptanceInputs): AcceptanceVerdict {
|
||||
const { candidate, baselineTrioStrictPassRateII } = inputs;
|
||||
|
||||
// Compute delta in percentage points (scale 0..100)
|
||||
const trioStrictDeltaPP =
|
||||
(candidate.trioStrictPassRateII - baselineTrioStrictPassRateII) * 100;
|
||||
|
||||
const isQwenTargeted = QWEN_TARGETED_SHAPES.has(candidate.shape);
|
||||
|
||||
// §F.5 false-positive guard — applies ONLY to Qwen-targeted shapes
|
||||
// and ONLY when trio_strict delta meets the +5pp threshold.
|
||||
// Per Amendment 2 §5: if delta ≥ +5pp AND retrieval_calls < 1.5 → REJECT.
|
||||
// EPSILON tolerance handles IEEE 754 precision on exact-boundary deltas.
|
||||
let condition5FalsePositiveGuardTriggered = false;
|
||||
if (
|
||||
isQwenTargeted &&
|
||||
trioStrictDeltaPP >= TRIO_STRICT_DELTA_THRESHOLD_PP - EPSILON &&
|
||||
candidate.meanRetrievalCallsPerTask < QWEN_FALSE_POSITIVE_RETRIEVAL_FLOOR - EPSILON
|
||||
) {
|
||||
condition5FalsePositiveGuardTriggered = true;
|
||||
}
|
||||
|
||||
// §F condition 1 — trio_strict delta + (Qwen only) retrieval engagement floor
|
||||
let condition1Pass = trioStrictDeltaPP >= TRIO_STRICT_DELTA_THRESHOLD_PP - EPSILON;
|
||||
if (condition1Pass && isQwenTargeted) {
|
||||
// Qwen sub-criterion: mean retrieval_calls ≥ 1.7
|
||||
if (candidate.meanRetrievalCallsPerTask < QWEN_RETRIEVAL_ENGAGEMENT_FLOOR - EPSILON) {
|
||||
condition1Pass = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Overall acceptance: condition 1 must pass AND §F.5 must not trigger
|
||||
const accepted = condition1Pass && !condition5FalsePositiveGuardTriggered;
|
||||
|
||||
// Reason string for audit log
|
||||
const reason = buildReason({
|
||||
candidate,
|
||||
trioStrictDeltaPP,
|
||||
condition1Pass,
|
||||
condition5FalsePositiveGuardTriggered,
|
||||
isQwenTargeted,
|
||||
accepted,
|
||||
});
|
||||
|
||||
return {
|
||||
condition1Pass,
|
||||
condition5FalsePositiveGuardTriggered,
|
||||
accepted,
|
||||
reason,
|
||||
trioStrictDeltaPP,
|
||||
};
|
||||
}
|
||||
|
||||
interface BuildReasonInputs {
|
||||
candidate: { shape: string; meanRetrievalCallsPerTask: number };
|
||||
trioStrictDeltaPP: number;
|
||||
condition1Pass: boolean;
|
||||
condition5FalsePositiveGuardTriggered: boolean;
|
||||
isQwenTargeted: boolean;
|
||||
accepted: boolean;
|
||||
}
|
||||
|
||||
function buildReason(i: BuildReasonInputs): string {
|
||||
const parts: string[] = [];
|
||||
parts.push(`shape=${i.candidate.shape}`);
|
||||
parts.push(`trio_strict_delta=${i.trioStrictDeltaPP.toFixed(2)}pp`);
|
||||
parts.push(`mean_retrieval_calls=${i.candidate.meanRetrievalCallsPerTask.toFixed(2)}`);
|
||||
|
||||
if (i.condition5FalsePositiveGuardTriggered) {
|
||||
parts.push(
|
||||
`REJECTED §F.5 false-positive guard: delta ≥ +${TRIO_STRICT_DELTA_THRESHOLD_PP}pp AND retrieval < ${QWEN_FALSE_POSITIVE_RETRIEVAL_FLOOR}`,
|
||||
);
|
||||
} else if (!i.condition1Pass) {
|
||||
// Use same EPSILON tolerance as condition1Pass evaluation to keep root-cause
|
||||
// attribution consistent with the gating logic on exact-boundary deltas.
|
||||
if (i.trioStrictDeltaPP < TRIO_STRICT_DELTA_THRESHOLD_PP - EPSILON) {
|
||||
parts.push(
|
||||
`FAIL §F.1 trio_strict delta: ${i.trioStrictDeltaPP.toFixed(2)}pp < ${TRIO_STRICT_DELTA_THRESHOLD_PP}pp`,
|
||||
);
|
||||
} else if (i.isQwenTargeted) {
|
||||
parts.push(
|
||||
`FAIL §F.1 Qwen retrieval floor: ${i.candidate.meanRetrievalCallsPerTask.toFixed(2)} < ${QWEN_RETRIEVAL_ENGAGEMENT_FLOOR}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
parts.push(`PASS §F.1${i.isQwenTargeted ? ' (Qwen retrieval floor met)' : ''}`);
|
||||
}
|
||||
|
||||
parts.push(`accepted=${i.accepted}`);
|
||||
return parts.join(' | ');
|
||||
}
|
||||
Reference in New Issue
Block a user