/** * Vision-harness JUDGE phase (Option C hybrid) — run via the Workflow tool: * Workflow({ scriptPath: "tests/vision/judge-workflow.mjs", args: { captures: [...] } }) * * Each capture is graded for MEANING by an independent vision-judge subagent * (it Reads the PNG — that IS the vision step), then a JS reducer cross-checks * the vision verdict against the objective console signal the capture phase * recorded: a vision-PASS that carries a real console error is downgraded to * FAIL (the "objective floor" so a plausible-looking screenshot can't pass). * * args.captures: [{ png, expectation, surface?, theme?, consoleErrors?[] }] * png absolute or repo-relative path to the screenshot * expectation one-line description of what the surface SHOULD show * consoleErrors objective signal from the capture driver (page.on('console')) * * Produces tests/vision/artifacts/vision-report.md + returns a summary. * Capture phase: tests/vision/capture.spec.ts (writes PNG + sidecar JSON). */ export const meta = { name: 'vision-e2e-judge', description: 'Grade captured Waggle screenshots for meaning via per-screenshot vision-judge agents + objective-signal reducer', phases: [ { title: 'Judge', detail: 'one vision-judge subagent per screenshot' }, { title: 'Report', detail: 'reduce verdicts + objective signals into one report' }, ], } const dim = { type: 'object', required: ['pass', 'note'], additionalProperties: false, properties: { pass: { type: 'boolean' }, note: { type: 'string', description: 'cite what you SEE' } }, } const VERDICT_SCHEMA = { type: 'object', required: ['verdict', 'confidence', 'dimensions'], additionalProperties: false, properties: { verdict: { type: 'string', enum: ['PASS', 'FAIL', 'WARN'] }, confidence: { type: 'number', description: '0-1 confidence in the overall verdict' }, dimensions: { type: 'object', required: ['renders_correctly', 'no_error_state', 'flow_completes', 'theme_legible'], additionalProperties: false, properties: { renders_correctly: dim, no_error_state: dim, flow_completes: dim, theme_legible: dim, }, }, }, } // args may arrive as a structured object OR a JSON string (depending on how // the Workflow caller passes it) — accept both. let parsedArgs = args if (typeof parsedArgs === 'string') { try { parsedArgs = JSON.parse(parsedArgs) } catch { parsedArgs = {} } } const captures = Array.isArray(parsedArgs?.captures) ? parsedArgs.captures : [] if (captures.length === 0) { log('No captures supplied. Pass args.captures = [{png, expectation, consoleErrors}].') return { error: 'no-captures', pass: 0, fail: 0, warn: 0 } } log(`Judging ${captures.length} captured surface(s) for meaning...`) phase('Judge') const VISION_DIMS = ['renders_correctly', 'no_error_state', 'flow_completes', 'theme_legible'] const judged = await parallel( captures.map((c) => () => agent( `You are a meticulous UI QA reviewer grading a single screenshot of the Waggle OS desktop app. Use the Read tool to VIEW the screenshot at this path, then judge what you actually see: ${c.png} This surface is expected to show: ${c.expectation} Grade each rubric dimension as pass=true/false with a one-line note citing what you SEE (not what you assume): - renders_correctly: content is laid out and visible — NOT blank, half-rendered, overlapping, or a bare skeleton. - no_error_state: no red error banner, no "Something went wrong", no stack trace, no infinite spinner, no empty white void where the app should be. - flow_completes: the expected end-state described above is actually visible on screen. - theme_legible: adequate text/background contrast — no dark-text-on-dark or white-text-on-white, nothing illegible. Set verdict=FAIL if any dimension fails and you are confident (>=0.7). verdict=WARN if you are unsure (0.4-0.7). verdict=PASS only if all four clearly hold. confidence = your certainty in that overall verdict. (An empty/clean "no data yet" state with clear UI chrome is a PASS for renders/no_error — judge whether the SHELL is healthy, not whether data exists.) Return ONLY the structured verdict.`, { label: `judge:${c.surface || c.theme || c.png}`, phase: 'Judge', schema: VERDICT_SCHEMA }, ).then((v) => ({ surface: c.surface || c.png, png: c.png, consoleErrors: Array.isArray(c.consoleErrors) ? c.consoleErrors : [], vision: v, })), ), ) // ── Reducer (objective floor): vision-PASS + real console error → FAIL ── const graded = judged.filter(Boolean).map((g) => { const v = g.vision || {} const dims = v.dimensions || {} const visionFailed = VISION_DIMS.some((d) => dims[d] && dims[d].pass === false) const hardSignal = g.consoleErrors.length > 0 let verdict = v.verdict || (visionFailed ? 'FAIL' : 'PASS') let downgraded = false if (verdict === 'PASS' && hardSignal) { verdict = 'FAIL' downgraded = true } return { surface: g.surface, png: g.png, verdict, confidence: typeof v.confidence === 'number' ? v.confidence : 0, downgradedByConsole: downgraded, failingDimensions: VISION_DIMS.filter((d) => dims[d] && dims[d].pass === false), notes: Object.fromEntries(VISION_DIMS.map((d) => [d, dims[d] ? dims[d].note : ''])), consoleErrors: g.consoleErrors, } }) const pass = graded.filter((g) => g.verdict === 'PASS').length const fail = graded.filter((g) => g.verdict === 'FAIL').length const warn = graded.filter((g) => g.verdict === 'WARN').length log(`Verdicts: ${pass} PASS / ${fail} FAIL / ${warn} WARN`) phase('Report') const REPORT_SCHEMA = { type: 'object', required: ['pass', 'fail', 'warn', 'reportPath'], additionalProperties: true, properties: { pass: { type: 'number' }, fail: { type: 'number' }, warn: { type: 'number' }, reportPath: { type: 'string' }, }, } const report = await agent( `Write a vision-E2E verdict report (Markdown) to the repo-relative path: tests/vision/artifacts/vision-report.md Use the Write tool. Base it ONLY on this graded data (already reduced — verdicts with downgradedByConsole=true were vision-PASS but had a real console error, so the objective floor flipped them to FAIL): ${JSON.stringify({ summary: { pass, fail, warn, total: graded.length }, graded }, null, 2)} The report must contain: 1. A "# Vision-E2E Report" heading + one-line summary: "${pass} PASS / ${fail} FAIL / ${warn} WARN of ${graded.length} surfaces". 2. A results table: Surface | Verdict | Confidence | Failing dimensions | Console errors | Downgraded?. 3. A "## Failures & Warnings" section — for every FAIL/WARN, the surface, the evidence PNG path, the failing dimensions with the judge's notes, and any console errors. (If none, write "All surfaces passed.") 4. A "## How this was graded" footer: each screenshot judged for meaning by an independent vision agent; a vision-PASS carrying a real console error is downgraded to FAIL (objective floor). Then return { pass, fail, warn, reportPath: "tests/vision/artifacts/vision-report.md" }.`, { label: 'report', phase: 'Report', schema: REPORT_SCHEMA }, ) return { summary: { pass, fail, warn, total: graded.length }, graded, report }