moving

benchmarks/gepa/scripts/faza-1/probe-registry-injection.ts

@@ -0,0 +1,161 @@
+#!/usr/bin/env tsx
+/**
+ * GEPA Faza 1 — REGISTRY-injection root-cause diagnostic probe.
+ *
+ * Per investigate-report.md §C.3, this probe distinguishes:
+ *   H1 — ESM module-identity mismatch (deep relative path vs package import)
+ *   H2 — registry mutation timing / silent freeze
+ *   H3 — other (tsx loader, vite-node interop, etc.)
+ *
+ * Method: import REGISTRY via BOTH paths the failing runner uses, then
+ * mutate via the script's path and probe via both. Also exercise the
+ * exact selectShape() call site the agent-loop uses.
+ *
+ * Cost: ~$0 (no LLM calls).
+ *
+ * Verdicts:
+ *   - "Same object?" === true  → mutations propagate; H2/H3 candidates
+ *   - "Same object?" === false → H1 confirmed (module-identity mismatch)
+ *   - selectShape from package returns the mutated shape → fix is unnecessary
+ *   - selectShape from package throws → bug is on the read path
+ */
+
+// ── Path A: deep relative import (matches the failing runner's import) ──
+import { REGISTRY as RegistryFromScriptDeepPath } from '../../../../packages/agent/src/prompt-shapes/selector.js';
+
+// ── Path B: package import (matches what runRetrievalAgentLoop's call to
+//           selectShape() goes through internally) ─────────────────────────
+import {
+  REGISTRY as RegistryFromPackage,
+  selectShape as selectShapeFromPackage,
+} from '@waggle/agent';
+
+// ── Path C: package-internal prompt-shapes export (parallel re-export check) ──
+import { REGISTRY as RegistryFromPromptShapes } from '@waggle/agent';
+
+const PROBE_SHAPE_NAME = 'claude-gen1-v1-probe';
+
+// Build a minimal valid PromptShape stub matching the type contract.
+// We only care that it gets registered + retrieved; method bodies are not
+// invoked in this probe.
+const probeShape = {
+  name: PROBE_SHAPE_NAME,
+  metadata: {
+    description: 'Probe shape for REGISTRY-injection diagnostic',
+    modelClass: 'probe',
+    defaultThinking: false,
+    defaultMaxTokens: 100,
+    evidence_link: 'benchmarks/results/gepa-faza1/gen-1/investigate-report.md',
+  },
+  systemPrompt: () => 'probe',
+  soloUserPrompt: () => 'probe',
+  multiStepKickoffUserPrompt: () => 'probe',
+  retrievalInjectionUserPrompt: () => 'probe',
+};
+
+function log(line: string): void {
+  process.stdout.write(line + '\n');
+}
+
+function header(t: string): void {
+  log('');
+  log('━'.repeat(76));
+  log(`  ${t}`);
+  log('━'.repeat(76));
+}
+
+header('GEPA Faza 1 — REGISTRY-injection diagnostic probe');
+
+log(`Probe shape name: ${PROBE_SHAPE_NAME}`);
+log(`Node version:     ${process.version}`);
+log(`tsx              : (running via tsx if argv[0] is node + script)`);
+
+// ── Step 1 — pre-mutation snapshot ────────────────────────────────────────
+
+header('STEP 1 — Pre-mutation snapshot (3 import paths)');
+
+const keysScriptPath = Object.keys(RegistryFromScriptDeepPath).sort();
+const keysPackagePath = Object.keys(RegistryFromPackage).sort();
+const keysPromptShapesPath = Object.keys(RegistryFromPromptShapes).sort();
+
+log(`A) Script deep-relative-path REGISTRY: ${keysScriptPath.length} keys: [${keysScriptPath.join(', ')}]`);
+log(`B) Package @waggle/agent REGISTRY:     ${keysPackagePath.length} keys: [${keysPackagePath.join(', ')}]`);
+log(`C) Package @waggle/agent (2nd import): ${keysPromptShapesPath.length} keys: [${keysPromptShapesPath.join(', ')}]`);
+log('');
+log(`Object identity A === B: ${RegistryFromScriptDeepPath === RegistryFromPackage}`);
+log(`Object identity A === C: ${RegistryFromScriptDeepPath === RegistryFromPromptShapes}`);
+log(`Object identity B === C: ${RegistryFromPackage === RegistryFromPromptShapes}`);
+
+// ── Step 2 — mutate via script's path (the failing pattern) ───────────────
+
+header('STEP 2 — Mutate REGISTRY via script deep-relative-path');
+
+(RegistryFromScriptDeepPath as any)[PROBE_SHAPE_NAME] = probeShape;
+
+log(`Mutation via Path A: REGISTRY[${PROBE_SHAPE_NAME}] = probeShape`);
+
+// ── Step 3 — read-back from all 3 paths ───────────────────────────────────
+
+header('STEP 3 — Read-back probe-shape via each path');
+
+const seenInA = RegistryFromScriptDeepPath[PROBE_SHAPE_NAME] !== undefined;
+const seenInB = RegistryFromPackage[PROBE_SHAPE_NAME] !== undefined;
+const seenInC = RegistryFromPromptShapes[PROBE_SHAPE_NAME] !== undefined;
+
+log(`A) Script-import sees probe-shape:  ${seenInA}`);
+log(`B) Package-import sees probe-shape: ${seenInB}`);
+log(`C) Re-import sees probe-shape:      ${seenInC}`);
+
+// ── Step 4 — call selectShape via package (the agent-loop path) ───────────
+
+header('STEP 4 — selectShape({override}) via @waggle/agent (agent-loop path)');
+
+let selectShapeVerdict: 'FOUND' | 'NOT_FOUND' | 'OTHER_ERROR';
+let selectShapeError: string | null = null;
+
+try {
+  const found = selectShapeFromPackage('any-alias-not-relevant', { override: PROBE_SHAPE_NAME });
+  log(`selectShape returned shape with name="${(found as any).name ?? '<missing>'}"`);
+  selectShapeVerdict = 'FOUND';
+} catch (e) {
+  selectShapeError = (e as Error).message;
+  log(`selectShape THREW: ${selectShapeError}`);
+  selectShapeVerdict = selectShapeError.includes('not in REGISTRY') ? 'NOT_FOUND' : 'OTHER_ERROR';
+}
+
+// ── Step 5 — verdict ───────────────────────────────────────────────────────
+
+header('STEP 5 — VERDICT');
+
+const sameObjectAB = RegistryFromScriptDeepPath === RegistryFromPackage;
+
+if (!sameObjectAB) {
+  log('VERDICT: H1 CONFIRMED — ESM module-identity mismatch');
+  log('  Script deep-relative-path REGISTRY and @waggle/agent REGISTRY are');
+  log('  separate object instances. Mutations to one do not propagate.');
+  log('');
+  log('Fix: add registerShape(name, shape) API in selector.ts, called from');
+  log('     a single canonical entry point. Avoid direct REGISTRY mutation.');
+} else if (sameObjectAB && seenInA && !seenInB) {
+  log('VERDICT: H2/H3 — Same object but read mismatch');
+  log('  This should not happen: identical objects with different key sets.');
+  log('  Investigate JS engine optimization, hidden Proxy, or freeze-on-read.');
+} else if (sameObjectAB && seenInA && seenInB && selectShapeVerdict === 'FOUND') {
+  log('VERDICT: NOT REPRODUCED — REGISTRY mutation works in this probe.');
+  log('  The runner-time failure must be due to a different cause (timing,');
+  log('  loader, dynamic-import side effect on candidate shape). Investigate');
+  log('  loadCandidates() dynamic import return shape vs static import.');
+} else if (sameObjectAB && seenInA && seenInB && selectShapeVerdict !== 'FOUND') {
+  log('VERDICT: H2/H3 — Read found but selectShape rejected');
+  log(`  Direct read sees probe-shape, but selectShape() error: ${selectShapeError}`);
+  log('  Investigate selectShape() implementation for hidden state or guard.');
+} else {
+  log('VERDICT: UNKNOWN');
+  log(`  sameObjectAB=${sameObjectAB} seenInA=${seenInA} seenInB=${seenInB} seenInC=${seenInC} selectShape=${selectShapeVerdict}`);
+}
+
+log('');
+log('Probe complete. Cost: $0 (no LLM calls). Halt-and-PM with verdict above.');
+
+// Exit 0 in all cases — we want PM to read the full output regardless of verdict
+process.exit(0);