208 lines
10 KiB
TypeScript
208 lines
10 KiB
TypeScript
/**
|
|
* Vision-harness CAPTURE phase (Option C hybrid, per
|
|
* docs/audits/2026-06-01-vision-e2e-harness-design.md).
|
|
*
|
|
* Drives the real desktop-OS shell deterministically and writes, per surface:
|
|
* artifacts/<surface>-<theme>.png — the screenshot the vision model grades
|
|
* artifacts/<surface>-<theme>.json — { surface, theme, expectation, nav,
|
|
* consoleErrors[], networkFailures[] }
|
|
*
|
|
* The JUDGE phase (scripts/vision-judge.mjs Workflow) reads these and grades
|
|
* meaning; the reducer cross-checks vision verdicts against the objective
|
|
* console/network signals captured here.
|
|
*
|
|
* Chat round-trip is graded on Path 2 (REAL LLM) per the product decision —
|
|
* the capture server must run WITHOUT --skip-litellm (a real provider key in
|
|
* the vault). Run: see scripts/vision-run.md.
|
|
*/
|
|
import { test, expect } from '@playwright/test';
|
|
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
import { join } from 'node:path';
|
|
import {
|
|
BASE, attachConsoleCapture, gotoDesktop, openAppViaDock, setTheme, pressShortcut,
|
|
type ConsoleCapture,
|
|
} from './_helpers';
|
|
|
|
const ARTIFACTS = join(process.cwd(), 'tests', 'vision', 'artifacts');
|
|
mkdirSync(ARTIFACTS, { recursive: true });
|
|
|
|
/** Expectation strings for non-static surfaces (memory/overlays/flows), set in
|
|
* their test bodies and read by write(). Declared up top to avoid TDZ. */
|
|
const FLOW_EXPECT: Record<string, string> = {};
|
|
|
|
const THEMES = ['dark', 'light'] as const;
|
|
type Theme = (typeof THEMES)[number];
|
|
|
|
interface Surface {
|
|
key: string;
|
|
/** Dock label (direct or zone tray) — or 'memory'/'home' special-cased. */
|
|
label: string;
|
|
/** One-line expectation handed to the vision judge. */
|
|
expectation: string;
|
|
}
|
|
|
|
/** Static surface matrix — real labels verified from full-product-audit.spec.ts. */
|
|
const SURFACES: Surface[] = [
|
|
{ key: 'chat', label: 'Chat', expectation: 'AI chat: a persona/model header, a message thread area, and a message input box at the bottom.' },
|
|
{ key: 'room', label: 'Room', expectation: 'The Room: a canvas for running agents, or a clean empty state ("no agents running").' },
|
|
{ key: 'agents', label: 'Agents', expectation: 'Agents: a list of agents with status badges and category tabs, or an empty "no agents yet" state.' },
|
|
{ key: 'files', label: 'Files', expectation: 'Files: a workspace file/folder browser, or an empty state.' },
|
|
{ key: 'approvals', label: 'Approvals', expectation: 'Approvals inbox: pending approval requests or a clean "no pending approvals" state.' },
|
|
{ key: 'cockpit', label: 'Mission Control', expectation: 'Mission Control / cockpit: KPI cards for health, cost, and activity.' },
|
|
{ key: 'timeline', label: 'Timeline', expectation: 'Timeline: a chronological activity feed, or an empty "no activity" state.' },
|
|
{ key: 'telemetry', label: 'Usage & Cost', expectation: 'Usage & Cost: token/cost telemetry charts or numbers.' },
|
|
// Backup left the dock (P23 — it lives in Settings → Backup), so it has no
|
|
// dock-driven surface here anymore.
|
|
{ key: 'events', label: 'Events & Logs', expectation: 'Events & Logs: a filterable list of agent steps/events.' },
|
|
{ key: 'governance', label: 'Team Governance', expectation: 'Governance: team roles, permissions, or policy controls.' },
|
|
{ key: 'capabilities', label: 'Skills Hub', expectation: 'Skills Hub: installed skills and a marketplace/starter affordance.' },
|
|
{ key: 'connectors', label: 'Connector Hub', expectation: 'Connector Hub: a catalog of services/integrations to connect, with status badges.' },
|
|
{ key: 'mcp-hub', label: 'MCP Hub', expectation: 'MCP Hub: installed MCP servers with state badges, or an empty installed state, plus Catalog/Custom tabs.' },
|
|
{ key: 'marketplace', label: 'Marketplace', expectation: 'Marketplace: a faceted extension browser (skills/agents/connectors/MCPs/models/templates) with install or open-in affordances.' },
|
|
{ key: 'settings', label: 'Settings', expectation: 'Settings: tabbed config (General/Models/Vault/Permissions/Team/Advanced).' },
|
|
{ key: 'vault', label: 'Vault', expectation: 'Vault / API Keys: per-provider key management rows.' },
|
|
{ key: 'dashboard', label: 'Home', expectation: 'Home/dashboard: workspace overview, welcome, or create-workspace affordance.' },
|
|
];
|
|
|
|
function write(surface: string, theme: Theme, nav: boolean, cap: ConsoleCapture) {
|
|
writeFileSync(
|
|
join(ARTIFACTS, `${surface}-${theme}.json`),
|
|
JSON.stringify(
|
|
{
|
|
surface, theme, nav,
|
|
expectation: SURFACES.find((s) => s.key === surface)?.expectation ?? FLOW_EXPECT[surface] ?? '',
|
|
consoleErrors: cap.critical(),
|
|
networkFailures: cap.networkFailures,
|
|
pageErrors: cap.pageErrors,
|
|
},
|
|
null, 2,
|
|
),
|
|
);
|
|
}
|
|
|
|
test.use({ viewport: { width: 1440, height: 900 } });
|
|
test.describe.configure({ mode: 'serial' });
|
|
|
|
// ── Static surface matrix ────────────────────────────────────────────────
|
|
for (const theme of THEMES) {
|
|
test.describe(`surfaces:${theme}`, () => {
|
|
for (const s of SURFACES) {
|
|
test(`${s.key}:${theme}`, async ({ page }) => {
|
|
const cap = attachConsoleCapture(page);
|
|
await gotoDesktop(page);
|
|
await setTheme(page, theme);
|
|
// Memory is reachable by aria-label or the Ctrl+Shift+5 shortcut.
|
|
let nav = await openAppViaDock(page, s.label);
|
|
if (!nav && s.key === 'dashboard') nav = await openAppViaDock(page, 'Home');
|
|
await page.waitForTimeout(800);
|
|
await page.screenshot({ path: join(ARTIFACTS, `${s.key}-${theme}.png`) });
|
|
write(s.key, theme, nav, cap);
|
|
expect(nav, `dock nav to "${s.label}"`).toBeTruthy();
|
|
});
|
|
}
|
|
|
|
// Memory (special-cased trigger)
|
|
test(`memory:${theme}`, async ({ page }) => {
|
|
const cap = attachConsoleCapture(page);
|
|
await gotoDesktop(page);
|
|
await setTheme(page, theme);
|
|
let nav = await openAppViaDock(page, 'Memory');
|
|
if (!nav) {
|
|
await pressShortcut(page, { key: '5', code: 'Digit5', ctrl: true, shift: true });
|
|
nav = true;
|
|
}
|
|
await page.waitForTimeout(800);
|
|
await page.screenshot({ path: join(ARTIFACTS, `memory-${theme}.png`) });
|
|
FLOW_EXPECT['memory'] = 'Memory: a searchable list of memory frames, or a clean empty state.';
|
|
write('memory', theme, nav, cap);
|
|
});
|
|
});
|
|
}
|
|
|
|
// ── Overlays (dark only — overlays inherit theme; cheap, best-effort) ──────
|
|
test.describe('overlays', () => {
|
|
test('global-search', async ({ page }) => {
|
|
const cap = attachConsoleCapture(page);
|
|
await gotoDesktop(page);
|
|
await pressShortcut(page, { key: 'k', code: 'KeyK', ctrl: true });
|
|
FLOW_EXPECT['global-search'] = 'Global search palette (Ctrl+K): a search input with results/commands.';
|
|
await page.screenshot({ path: join(ARTIFACTS, 'global-search-dark.png') });
|
|
write('global-search', 'dark', true, cap);
|
|
});
|
|
|
|
test('spawn-agent', async ({ page }) => {
|
|
const cap = attachConsoleCapture(page);
|
|
await gotoDesktop(page);
|
|
const btn = page.locator('[data-testid="nav-spawn-agent"]');
|
|
const nav = await btn.isVisible({ timeout: 1500 }).catch(() => false);
|
|
if (nav) { await btn.click(); await page.waitForTimeout(900); }
|
|
FLOW_EXPECT['spawn-agent'] = 'Spawn-agent dialog: a persona picker + model selector + confirm button.';
|
|
await page.screenshot({ path: join(ARTIFACTS, 'spawn-agent-dark.png') });
|
|
write('spawn-agent', 'dark', nav, cap);
|
|
});
|
|
});
|
|
|
|
// ── Flows (end-state graded) ──────────────────────────────────────────────
|
|
test.describe('flows', () => {
|
|
// Chat round-trip — Path 2 (REAL LLM): a coherent assistant reply must render.
|
|
test('flow:chat-roundtrip', async ({ page }) => {
|
|
const cap = attachConsoleCapture(page);
|
|
await gotoDesktop(page);
|
|
await openAppViaDock(page, 'Chat');
|
|
await page.waitForTimeout(1200);
|
|
const box = page.locator('textarea').first();
|
|
const placeholderInput = page.getByPlaceholder(/message|ask|waggle/i).first();
|
|
const target = (await box.isVisible({ timeout: 2000 }).catch(() => false))
|
|
? box
|
|
: placeholderInput;
|
|
await target.fill('In one short sentence, what is Waggle OS?');
|
|
await target.press('Enter');
|
|
// Real LLM: wait for an assistant reply to stream in (best-effort up to 35s).
|
|
await page.waitForTimeout(2000);
|
|
await page.waitForFunction(
|
|
() => document.body.innerText.length > 400,
|
|
{ timeout: 35_000 },
|
|
).catch(() => { /* capture whatever state exists; judge decides */ });
|
|
await page.waitForTimeout(1500);
|
|
FLOW_EXPECT['flow-chat'] =
|
|
'Chat round-trip (real LLM): the user question and a coherent assistant reply are both visible in the thread. NOT a blank thread, error banner, or "configure API key" prompt.';
|
|
await page.screenshot({ path: join(ARTIFACTS, 'flow-chat-dark.png') });
|
|
write('flow-chat', 'dark', true, cap);
|
|
});
|
|
|
|
// Settings tab walk — each tab renders.
|
|
test('flow:settings-tabs', async ({ page }) => {
|
|
const cap = attachConsoleCapture(page);
|
|
await gotoDesktop(page);
|
|
await openAppViaDock(page, 'Settings');
|
|
await page.waitForTimeout(1000);
|
|
FLOW_EXPECT['flow-settings'] =
|
|
'Settings opened: a tabbed settings panel (General/Models/Vault/Permissions/Team/Advanced) rendering content, no error state.';
|
|
await page.screenshot({ path: join(ARTIFACTS, 'flow-settings-dark.png') });
|
|
write('flow-settings', 'dark', true, cap);
|
|
});
|
|
|
|
// Marketplace browse.
|
|
test('flow:marketplace', async ({ page }) => {
|
|
const cap = attachConsoleCapture(page);
|
|
await gotoDesktop(page);
|
|
await openAppViaDock(page, 'Marketplace');
|
|
await page.waitForTimeout(1200);
|
|
FLOW_EXPECT['flow-marketplace'] =
|
|
'Marketplace browse: packs/items listed with install affordances, or a clean empty/loading state — not an error.';
|
|
await page.screenshot({ path: join(ARTIFACTS, 'flow-marketplace-dark.png') });
|
|
write('flow-marketplace', 'dark', true, cap);
|
|
});
|
|
|
|
// Onboarding wizard (forceWizard).
|
|
test('flow:onboarding', async ({ page }) => {
|
|
const cap = attachConsoleCapture(page);
|
|
await page.goto(`${BASE}/?forceWizard=true&skipBoot=true`, { waitUntil: 'domcontentloaded' });
|
|
await page.waitForTimeout(1500);
|
|
FLOW_EXPECT['flow-onboarding'] =
|
|
'Onboarding wizard: a welcome/setup step with a clear primary action to proceed.';
|
|
await page.screenshot({ path: join(ARTIFACTS, 'flow-onboarding-dark.png') });
|
|
write('flow-onboarding', 'dark', true, cap);
|
|
});
|
|
});
|