1611 lines
68 KiB
TypeScript
1611 lines
68 KiB
TypeScript
import { expect, type APIRequestContext, type Page, test, type TestInfo } from '@playwright/test';
|
|
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
import { dirname } from 'node:path';
|
|
|
|
const BASE = process.env.WAGGLE_E2E_BASE_URL ?? 'http://127.0.0.1:3333';
|
|
const SKIP = 'skipOnboarding=true&skipBoot=true&tier=power&skipBriefing=true';
|
|
const API_PROBE_TIMEOUT_MS = 1_000;
|
|
const MEMORY_POLL_TIMEOUT_MS = 1_500;
|
|
const MEMORY_POLL_INTERVAL_MS = 250;
|
|
|
|
type DimensionId =
|
|
| 'onboarding'
|
|
| 'timeToValue'
|
|
| 'memory'
|
|
| 'workflowCoverage'
|
|
| 'competitiveAdvantage'
|
|
| 'addiction';
|
|
|
|
interface PersonaDefinition {
|
|
id: string;
|
|
name: string;
|
|
role: string;
|
|
currentDefault: string[];
|
|
jobToBeDone: string;
|
|
oneToolCriterion: string;
|
|
memoryAnchor: string;
|
|
memoryQuery: string;
|
|
expectedPersonaIds: string[];
|
|
expectedSkillTerms: string[];
|
|
expectedConnectorTerms: string[];
|
|
externalTriggerNeed: string;
|
|
internalTriggerNeed: string;
|
|
competitorBaseline: string;
|
|
}
|
|
|
|
interface DimensionScore {
|
|
id: DimensionId;
|
|
label: string;
|
|
max: number;
|
|
score: number;
|
|
evidence: string[];
|
|
gaps: string[];
|
|
}
|
|
|
|
interface ColdStartScore {
|
|
score: number;
|
|
grade: string;
|
|
evidence: string[];
|
|
gaps: string[];
|
|
}
|
|
|
|
interface WorkflowCompletion {
|
|
completed: boolean;
|
|
evidence: string[];
|
|
}
|
|
|
|
interface PersonaScore {
|
|
id: string;
|
|
name: string;
|
|
role: string;
|
|
total: number;
|
|
grade: string;
|
|
positioning: string;
|
|
currentDefault: string[];
|
|
competitorBaseline: string;
|
|
oneToolCriterion: string;
|
|
dimensions: DimensionScore[];
|
|
improvementAreas: string[];
|
|
}
|
|
|
|
interface ImprovementArea {
|
|
priority: number;
|
|
personaId: string;
|
|
personaName: string;
|
|
dimension: string;
|
|
impact: number;
|
|
recommendation: string;
|
|
evidence: string[];
|
|
}
|
|
|
|
interface AuditResult {
|
|
generatedAt: string;
|
|
overall: {
|
|
score: number;
|
|
grade: string;
|
|
coldStartScore: number;
|
|
coldStartGrade: string;
|
|
positioningVerdict: string;
|
|
};
|
|
coldStart: ColdStartScore;
|
|
addictionLevel: string;
|
|
personas: PersonaScore[];
|
|
improvementAreas: ImprovementArea[];
|
|
artifacts: {
|
|
markdownPath: string;
|
|
jsonPath: string;
|
|
};
|
|
}
|
|
|
|
interface ProbeContext {
|
|
shellLoaded: boolean;
|
|
shellText: string;
|
|
consoleErrors: string[];
|
|
coldHealth: { status: number; ok: boolean; ms: number; body: unknown; error?: string };
|
|
authEvidence?: string[];
|
|
api: Record<string, { status: number; ok: boolean; ms: number; body: unknown; error?: string }>;
|
|
personaIds: string[];
|
|
personaText: string;
|
|
skillsText: string;
|
|
connectorsText: string;
|
|
marketplaceText: string;
|
|
memory: Record<string, { saved: boolean; recalled: boolean; isolated: boolean; evidence: string[] }>;
|
|
workflowCompletion: Record<string, WorkflowCompletion>;
|
|
}
|
|
|
|
interface ProbeAuth {
|
|
headers: Record<string, string>;
|
|
evidence: string[];
|
|
}
|
|
|
|
interface ProbeWorkspace {
|
|
available: boolean;
|
|
id: string;
|
|
evidence: string[];
|
|
}
|
|
|
|
interface ProbeWorkspacePool {
|
|
primary: ProbeWorkspace;
|
|
comparison: ProbeWorkspace;
|
|
}
|
|
|
|
const PERSONAS: PersonaDefinition[] = [
|
|
{
|
|
id: 'sofia-operator',
|
|
name: 'Sofia',
|
|
role: 'Small business operator',
|
|
currentDefault: ['ChatGPT', 'Gmail', 'Canva'],
|
|
jobToBeDone: 'Draft customer replies, campaign ideas, and supplier follow-ups.',
|
|
oneToolCriterion: 'Daily communications and decisions happen in Waggle.',
|
|
memoryAnchor: 'Sofia runs a neighborhood studio and prefers short, warm customer replies with clear next steps.',
|
|
memoryQuery: 'short warm customer replies next steps',
|
|
expectedPersonaIds: ['support-agent', 'marketer', 'executive-assistant'],
|
|
expectedSkillTerms: ['email', 'marketing', 'document'],
|
|
expectedConnectorTerms: ['gmail', 'google', 'slack'],
|
|
externalTriggerNeed: 'daily brief or reply reminder',
|
|
internalTriggerNeed: 'I need to answer customers without sounding generic',
|
|
competitorBaseline: 'ChatGPT is fast for drafting but does not own her customer context or operating rhythm.',
|
|
},
|
|
{
|
|
id: 'mara-writer',
|
|
name: 'Mara',
|
|
role: 'Marketing writer',
|
|
currentDefault: ['ChatGPT', 'Claude', 'Notion AI'],
|
|
jobToBeDone: 'Turn notes and research into branded copy.',
|
|
oneToolCriterion: 'Voice, drafts, and campaign memory compound in Waggle.',
|
|
memoryAnchor: 'Mara writes in a crisp, specific brand voice and tracks campaign decisions by launch.',
|
|
memoryQuery: 'brand voice campaign decisions launch',
|
|
expectedPersonaIds: ['writer', 'marketer', 'creative-director'],
|
|
expectedSkillTerms: ['writing', 'brand', 'markdown'],
|
|
expectedConnectorTerms: ['notion', 'google', 'slack'],
|
|
externalTriggerNeed: 'weekly wins digest or draft reminder',
|
|
internalTriggerNeed: 'I need the AI to remember my voice and the campaign angle',
|
|
competitorBaseline: 'Claude is excellent at prose but does not act as a durable operating workspace.',
|
|
},
|
|
{
|
|
id: 'imran-consultant',
|
|
name: 'Imran',
|
|
role: 'Independent consultant',
|
|
currentDefault: ['Claude', 'ChatGPT', 'Gamma'],
|
|
jobToBeDone: 'Convert calls and notes into frameworks, briefs, and follow-ups.',
|
|
oneToolCriterion: 'Client context and recurring strategy work live in Waggle.',
|
|
memoryAnchor: 'Imran uses 2x2 frameworks and wants every client decision remembered by account.',
|
|
memoryQuery: '2x2 framework client decision account',
|
|
expectedPersonaIds: ['consultant', 'researcher', 'analyst'],
|
|
expectedSkillTerms: ['presentation', 'document', 'research'],
|
|
expectedConnectorTerms: ['calendar', 'google', 'slack'],
|
|
externalTriggerNeed: 'client follow-up reminder',
|
|
internalTriggerNeed: 'What did we decide for this client last time?',
|
|
competitorBaseline: 'Claude and Gamma help produce artifacts, but the client memory loop is fragmented.',
|
|
},
|
|
{
|
|
id: 'daniel-finance',
|
|
name: 'Daniel',
|
|
role: 'Finance and operations analyst',
|
|
currentDefault: ['Excel Copilot', 'ChatGPT', 'Looker'],
|
|
jobToBeDone: 'Explain variance, summarize metrics, and prepare board commentary.',
|
|
oneToolCriterion: 'Data commentary and recurring monthly memory live in Waggle.',
|
|
memoryAnchor: 'Daniel prepares monthly board commentary and cares about variance drivers, ARR, NPS, and burn.',
|
|
memoryQuery: 'monthly board commentary variance ARR NPS burn',
|
|
expectedPersonaIds: ['finance-owner', 'analyst', 'ops-manager'],
|
|
expectedSkillTerms: ['spreadsheet', 'csv', 'analysis'],
|
|
expectedConnectorTerms: ['excel', 'google', 'microsoft'],
|
|
externalTriggerNeed: 'monthly reporting reminder',
|
|
internalTriggerNeed: 'I need a variance explanation I can defend',
|
|
competitorBaseline: 'Excel Copilot is close to the data but weak as cross-month memory and agent workspace.',
|
|
},
|
|
{
|
|
id: 'priya-power-user',
|
|
name: 'Priya',
|
|
role: 'AI power user',
|
|
currentDefault: ['Claude Code', 'Codex', 'Hermes', 'OpenClaw'],
|
|
jobToBeDone: 'Coordinate AI workflows, skills, connectors, and memory.',
|
|
oneToolCriterion: 'Waggle is the front door for non-coding agent work.',
|
|
memoryAnchor: 'Priya wants a non-coding AI command center with skills, connectors, memory, and agent coordination.',
|
|
memoryQuery: 'non coding command center skills connectors agent coordination',
|
|
expectedPersonaIds: ['coordinator', 'planner', 'verifier', 'coder'],
|
|
expectedSkillTerms: ['skill', 'automation', 'agent'],
|
|
expectedConnectorTerms: ['github', 'mcp', 'webhook'],
|
|
externalTriggerNeed: 'OS hotkey, launcher, or scheduled automation',
|
|
internalTriggerNeed: 'I need one control surface for all my AI work',
|
|
competitorBaseline: 'Developer tools are powerful for code but do not give a non-coding personal AI workspace.',
|
|
},
|
|
];
|
|
|
|
function clampScore(score: number, max: number): number {
|
|
return Math.max(0, Math.min(max, Math.round(score)));
|
|
}
|
|
|
|
function gradeFor(score: number): string {
|
|
if (score >= 90) return 'Strong personal AI workspace position';
|
|
if (score >= 75) return 'Strong niche AI workspace fit';
|
|
if (score >= 60) return 'Promising but still competitor-dependent';
|
|
if (score >= 40) return 'Plausible positioning, weak product proof';
|
|
return 'Likely perceived as another AI chat/tool wrapper';
|
|
}
|
|
|
|
function addictionLevelFor(score: number): string {
|
|
if (score >= 85) return 'very strong';
|
|
if (score >= 70) return 'strong';
|
|
if (score >= 50) return 'emerging';
|
|
return 'weak';
|
|
}
|
|
|
|
function coldStartGradeFor(score: number): string {
|
|
if (score >= 90) return 'Fast first-session readiness';
|
|
if (score >= 75) return 'Usable but monitor first-session speed';
|
|
if (score >= 55) return 'Noticeable cold-start drag';
|
|
return 'Cold-start risk';
|
|
}
|
|
|
|
function textIncludesAny(haystack: string, needles: string[]): boolean {
|
|
const lower = haystack.toLowerCase();
|
|
return needles.some((needle) => lower.includes(needle.toLowerCase()));
|
|
}
|
|
|
|
function hasPersonalWorkAwarePositioning(shellText: string): boolean {
|
|
const personalSignal = /knows you|remembers you|personal|how you work|work style|stays yours|yours/i.test(shellText);
|
|
const workSignal = /knows your work|projects?|decisions?|context|work-in-progress|work itself|workspace/i.test(shellText);
|
|
const guidanceSignal = /runs the AI|right AI|AI underneath|guides? the next step|evolves?|routing|models?|AI workspace/i.test(shellText);
|
|
return personalSignal && workSignal && guidanceSignal;
|
|
}
|
|
|
|
function renderMarkdown(audit: AuditResult): string {
|
|
const lines: string[] = [];
|
|
lines.push('# Waggle Personal AI Workspace Positioning Audit');
|
|
lines.push('');
|
|
lines.push(`Generated: ${audit.generatedAt}`);
|
|
lines.push(`Overall score: ${audit.overall.score}/100`);
|
|
lines.push(`Grade: ${audit.overall.grade}`);
|
|
lines.push(`Cold-start readiness: ${audit.overall.coldStartScore}/100 (${audit.overall.coldStartGrade})`);
|
|
lines.push(`Positioning verdict: ${audit.overall.positioningVerdict}`);
|
|
lines.push(`Addiction level: ${audit.addictionLevel}`);
|
|
lines.push('');
|
|
lines.push('## Cold-Start Readiness');
|
|
lines.push('');
|
|
lines.push(`Score: ${audit.coldStart.score}/100`);
|
|
for (const evidence of audit.coldStart.evidence) {
|
|
lines.push(`- ${evidence}`);
|
|
}
|
|
for (const gap of audit.coldStart.gaps) {
|
|
lines.push(`- Gap: ${gap}`);
|
|
}
|
|
lines.push('');
|
|
lines.push('## Persona Scores');
|
|
lines.push('');
|
|
lines.push('| Persona | Role | Score | Grade | Current default |');
|
|
lines.push('|---|---|---:|---|---|');
|
|
for (const persona of audit.personas) {
|
|
lines.push(`| ${persona.name} | ${persona.role} | ${persona.total} | ${persona.grade} | ${persona.currentDefault.join(', ')} |`);
|
|
}
|
|
lines.push('');
|
|
lines.push('## Improvement Areas');
|
|
lines.push('');
|
|
for (const item of audit.improvementAreas) {
|
|
lines.push(`${item.priority}. **${item.personaName} - ${item.dimension}** (${item.impact} pts): ${item.recommendation}`);
|
|
for (const evidence of item.evidence.slice(0, 2)) {
|
|
lines.push(` - Evidence: ${evidence}`);
|
|
}
|
|
}
|
|
lines.push('');
|
|
lines.push('## Persona Detail');
|
|
for (const persona of audit.personas) {
|
|
lines.push('');
|
|
lines.push(`### ${persona.name} - ${persona.role}`);
|
|
lines.push('');
|
|
lines.push(`Score: ${persona.total}/100`);
|
|
lines.push(`Positioning: ${persona.positioning}`);
|
|
lines.push(`One-tool criterion: ${persona.oneToolCriterion}`);
|
|
lines.push(`Competitor baseline: ${persona.competitorBaseline}`);
|
|
lines.push('');
|
|
lines.push('| Dimension | Score | Evidence | Gaps |');
|
|
lines.push('|---|---:|---|---|');
|
|
for (const dim of persona.dimensions) {
|
|
lines.push(`| ${dim.label} | ${dim.score}/${dim.max} | ${dim.evidence.join('<br>')} | ${dim.gaps.join('<br>')} |`);
|
|
}
|
|
}
|
|
lines.push('');
|
|
return `${lines.join('\n')}\n`;
|
|
}
|
|
|
|
function requestErrorSummary(error: unknown): string {
|
|
return error instanceof Error ? error.message : String(error);
|
|
}
|
|
|
|
async function bootstrapProbeAuth(request: APIRequestContext): Promise<ProbeAuth> {
|
|
const response = await request.get(`${BASE}/api/auth/session-token`, { timeout: API_PROBE_TIMEOUT_MS }).catch((error: unknown) => {
|
|
return { error: requestErrorSummary(error) };
|
|
});
|
|
if ('error' in response) {
|
|
return {
|
|
headers: {},
|
|
evidence: [`Session-token bootstrap failed within ${API_PROBE_TIMEOUT_MS}ms: ${response.error}.`],
|
|
};
|
|
}
|
|
|
|
const body = await readResponseBody(response);
|
|
const token = typeof asRecord(body).token === 'string' ? String(asRecord(body).token) : '';
|
|
if (response.ok() && token) {
|
|
return {
|
|
headers: { Authorization: `Bearer ${token}` },
|
|
evidence: [`Session-token bootstrap returned ${response.status()} and protected probes used bearer auth.`],
|
|
};
|
|
}
|
|
|
|
return {
|
|
headers: {},
|
|
evidence: [`Session-token bootstrap returned ${response.status()} with ${bodySummary(body)}; protected probes continued without bearer auth.`],
|
|
};
|
|
}
|
|
|
|
async function timedGet(request: APIRequestContext, path: string, auth?: ProbeAuth) {
|
|
const started = Date.now();
|
|
const response = await request.get(`${BASE}${path}`, { timeout: API_PROBE_TIMEOUT_MS, headers: auth?.headers }).catch((error: unknown) => {
|
|
return { error: requestErrorSummary(error) };
|
|
});
|
|
const ms = Date.now() - started;
|
|
if ('error' in response) {
|
|
return { status: 0, ok: false, ms, body: { error: response.error }, error: response.error };
|
|
}
|
|
const body = await response.json().catch(async () => response.text().catch(() => null));
|
|
return { status: response.status(), ok: response.ok(), ms, body };
|
|
}
|
|
|
|
function bodySummary(body: unknown): string {
|
|
if (body === null || body === undefined) return 'empty body';
|
|
const raw = typeof body === 'string' ? body : JSON.stringify(body);
|
|
return raw.length > 240 ? `${raw.slice(0, 240)}...` : raw;
|
|
}
|
|
|
|
function asRecord(value: unknown): Record<string, unknown> {
|
|
return typeof value === 'object' && value !== null ? value as Record<string, unknown> : {};
|
|
}
|
|
|
|
function isValidFrameId(value: unknown): boolean {
|
|
if (typeof value === 'number') return Number.isFinite(value) && value > 0;
|
|
if (typeof value === 'string') return value.trim().length > 0;
|
|
return false;
|
|
}
|
|
|
|
function hasSavedFrameReference(record: Record<string, unknown>): boolean {
|
|
const frame = asRecord(record.frame);
|
|
const memory = asRecord(record.memory);
|
|
const data = asRecord(record.data);
|
|
return [
|
|
record.frameId,
|
|
record.id,
|
|
frame.frameId,
|
|
frame.id,
|
|
memory.frameId,
|
|
memory.id,
|
|
data.frameId,
|
|
data.id,
|
|
].some(isValidFrameId);
|
|
}
|
|
|
|
function meaningfulTerms(...texts: string[]): string[] {
|
|
const stopWords = new Set([
|
|
'about',
|
|
'across',
|
|
'agent',
|
|
'center',
|
|
'clear',
|
|
'coding',
|
|
'context',
|
|
'memory',
|
|
'normal',
|
|
'persona',
|
|
'personal',
|
|
'result',
|
|
'results',
|
|
'source',
|
|
'their',
|
|
'there',
|
|
'wants',
|
|
'workspace',
|
|
'workspaces',
|
|
]);
|
|
const terms = texts
|
|
.join(' ')
|
|
.toLowerCase()
|
|
.match(/[a-z0-9]+/g) ?? [];
|
|
return [...new Set(terms.filter((term) => term.length >= 3 && !stopWords.has(term)))];
|
|
}
|
|
|
|
async function readResponseBody(response: Awaited<ReturnType<APIRequestContext['get']>>): Promise<unknown> {
|
|
return response.json().catch(async () => response.text().catch(() => null));
|
|
}
|
|
|
|
async function wait(ms: number): Promise<void> {
|
|
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
async function ensureProbeWorkspace(request: APIRequestContext, workspaceName: string, auth?: ProbeAuth): Promise<ProbeWorkspace> {
|
|
const evidence: string[] = [];
|
|
const response = await request.post(`${BASE}/api/workspaces`, {
|
|
timeout: API_PROBE_TIMEOUT_MS,
|
|
headers: auth?.headers,
|
|
data: {
|
|
name: workspaceName,
|
|
group: 'Positioning Audit',
|
|
icon: 'sparkles',
|
|
storageType: 'virtual',
|
|
},
|
|
}).catch((error: unknown) => {
|
|
evidence.push(`Workspace create failed within ${API_PROBE_TIMEOUT_MS}ms: ${requestErrorSummary(error)}.`);
|
|
return null;
|
|
});
|
|
|
|
if (response) {
|
|
const body = await readResponseBody(response);
|
|
const record = asRecord(body);
|
|
const id = typeof record.id === 'string' ? record.id : '';
|
|
evidence.push(`Workspace create returned ${response.status()} with ${bodySummary(body)}.`);
|
|
if (response.ok() && id) {
|
|
return { available: true, id, evidence };
|
|
}
|
|
} else {
|
|
evidence.push('Workspace create did not return a response.');
|
|
}
|
|
|
|
const listResponse = await request.get(`${BASE}/api/workspaces`, { timeout: API_PROBE_TIMEOUT_MS, headers: auth?.headers }).catch((error: unknown) => {
|
|
evidence.push(`Workspace confirmation list failed within ${API_PROBE_TIMEOUT_MS}ms: ${requestErrorSummary(error)}.`);
|
|
return null;
|
|
});
|
|
if (!listResponse) {
|
|
evidence.push('Workspace confirmation list did not return a response.');
|
|
return { available: false, id: workspaceName, evidence };
|
|
}
|
|
|
|
const listBody = await readResponseBody(listResponse);
|
|
evidence.push(`Workspace confirmation list returned ${listResponse.status()} with ${bodySummary(listBody)}.`);
|
|
if (!listResponse.ok()) {
|
|
return { available: false, id: workspaceName, evidence };
|
|
}
|
|
|
|
const workspaces = Array.isArray(listBody) ? listBody : [];
|
|
const match = workspaces
|
|
.map(asRecord)
|
|
.find((workspace) => workspace.id === workspaceName || workspace.name === workspaceName);
|
|
const id = typeof match?.id === 'string' ? match.id : '';
|
|
if (!id) {
|
|
evidence.push('Probe workspace was not found in the workspace list.');
|
|
return { available: false, id: workspaceName, evidence };
|
|
}
|
|
|
|
evidence.push(`Probe workspace confirmed as ${id}.`);
|
|
return { available: true, id, evidence };
|
|
}
|
|
|
|
async function saveMemory(request: APIRequestContext, workspace: string, content: string, auth?: ProbeAuth): Promise<{ saved: boolean; evidence: string[] }> {
|
|
const response = await request.post(`${BASE}/api/memory/frames`, {
|
|
timeout: API_PROBE_TIMEOUT_MS,
|
|
headers: auth?.headers,
|
|
data: { content, workspace, source: 'user_stated', importance: 'normal' },
|
|
}).catch((error: unknown) => {
|
|
return { error: requestErrorSummary(error) };
|
|
});
|
|
if (response && 'error' in response) {
|
|
return { saved: false, evidence: [`Memory save failed within ${API_PROBE_TIMEOUT_MS}ms: ${response.error}.`] };
|
|
}
|
|
if (!response) {
|
|
return { saved: false, evidence: ['Memory save did not return a response.'] };
|
|
}
|
|
|
|
const body = await readResponseBody(response);
|
|
const record = asRecord(body);
|
|
const acceptedBody = record.saved === true || record.duplicate === true || hasSavedFrameReference(record);
|
|
return {
|
|
saved: response.ok() && acceptedBody,
|
|
evidence: [`Memory save returned ${response.status()} with ${bodySummary(body)}.`],
|
|
};
|
|
}
|
|
|
|
async function searchMemory(request: APIRequestContext, workspace: string, query: string, expectedContent = query, auth?: ProbeAuth): Promise<{ found: boolean; checked: boolean; evidence: string[] }> {
|
|
const response = await request.get(
|
|
`${BASE}/api/memory/search?q=${encodeURIComponent(query)}&workspace=${encodeURIComponent(workspace)}&scope=workspace&limit=5`,
|
|
{ timeout: API_PROBE_TIMEOUT_MS, headers: auth?.headers },
|
|
).catch((error: unknown) => {
|
|
return { error: requestErrorSummary(error) };
|
|
});
|
|
if (response && 'error' in response) {
|
|
return { found: false, checked: false, evidence: [`Memory search failed within ${API_PROBE_TIMEOUT_MS}ms: ${response.error}.`] };
|
|
}
|
|
if (!response) return { found: false, checked: false, evidence: ['Memory search did not return a response.'] };
|
|
const body = await readResponseBody(response);
|
|
if (!response.ok()) return { found: false, checked: false, evidence: [`Memory search returned ${response.status()} with ${bodySummary(body)}.`] };
|
|
const raw = JSON.stringify(body);
|
|
const record = asRecord(body);
|
|
const results = Array.isArray(body)
|
|
? body
|
|
: Array.isArray(record.results)
|
|
? record.results
|
|
: Array.isArray(record.recalled)
|
|
? record.recalled
|
|
: [];
|
|
const expectedTerms = meaningfulTerms(query, expectedContent);
|
|
const matchedTerms = expectedTerms.filter((term) => raw.toLowerCase().includes(term));
|
|
const requiredMatches = Math.max(1, Math.min(3, Math.ceil(expectedTerms.length * 0.4)));
|
|
const found = results.length > 0 && matchedTerms.length >= requiredMatches;
|
|
return {
|
|
found,
|
|
checked: true,
|
|
evidence: [
|
|
`Memory search returned ${response.status()} with ${results.length} result(s); matched ${matchedTerms.length}/${expectedTerms.length} expected term(s): ${matchedTerms.slice(0, 6).join(', ') || 'none'}.`,
|
|
`Memory search body: ${bodySummary(body)}.`,
|
|
],
|
|
};
|
|
}
|
|
|
|
async function pollMemorySearch(
|
|
request: APIRequestContext,
|
|
workspace: string,
|
|
query: string,
|
|
expectedContent: string,
|
|
label: string,
|
|
auth?: ProbeAuth,
|
|
): Promise<{ found: boolean; checked: boolean; evidence: string[] }> {
|
|
const started = Date.now();
|
|
let attempts = 0;
|
|
let lastResult: { found: boolean; checked: boolean; evidence: string[] } = {
|
|
found: false,
|
|
checked: false,
|
|
evidence: [`${label} memory search has not run yet.`],
|
|
};
|
|
|
|
while (Date.now() - started <= MEMORY_POLL_TIMEOUT_MS) {
|
|
attempts += 1;
|
|
lastResult = await searchMemory(request, workspace, query, expectedContent, auth);
|
|
if (lastResult.found) {
|
|
return {
|
|
found: true,
|
|
checked: true,
|
|
evidence: [`${label} memory search matched after ${attempts} attempt(s).`, ...lastResult.evidence],
|
|
};
|
|
}
|
|
|
|
const remaining = MEMORY_POLL_TIMEOUT_MS - (Date.now() - started);
|
|
if (remaining <= 0) break;
|
|
await wait(Math.min(MEMORY_POLL_INTERVAL_MS, remaining));
|
|
}
|
|
|
|
return {
|
|
found: false,
|
|
checked: lastResult.checked,
|
|
evidence: [`${label} memory search did not match within ${MEMORY_POLL_TIMEOUT_MS}ms after ${attempts} attempt(s).`, ...lastResult.evidence],
|
|
};
|
|
}
|
|
|
|
async function probeMemory(request: APIRequestContext, persona: PersonaDefinition, auth?: ProbeAuth, pool?: ProbeWorkspacePool, runId = '') {
|
|
const workspace = `ai-os-audit-${persona.id}-${Date.now()}`;
|
|
const otherWorkspace = `${workspace}-isolation`;
|
|
const primary = pool?.primary ?? await ensureProbeWorkspace(request, workspace, auth);
|
|
|
|
if (!primary.available) {
|
|
return {
|
|
saved: false,
|
|
recalled: false,
|
|
isolated: false,
|
|
evidence: [
|
|
...primary.evidence,
|
|
'Primary probe workspace was unavailable, so memory save/recall/isolation were not claimed.',
|
|
],
|
|
};
|
|
}
|
|
|
|
const comparison = pool?.comparison ?? await ensureProbeWorkspace(request, otherWorkspace, auth);
|
|
const memoryAnchor = runId ? `${persona.memoryAnchor} Audit run ${runId}.` : persona.memoryAnchor;
|
|
const memoryQuery = runId ? `${persona.memoryQuery} ${runId}` : persona.memoryQuery;
|
|
const save = await saveMemory(request, primary.id, memoryAnchor, auth);
|
|
const recall = save.saved
|
|
? await pollMemorySearch(request, primary.id, memoryQuery, memoryAnchor, 'Recall', auth)
|
|
: { found: false, checked: false, evidence: ['Memory recall skipped because the memory save did not succeed.'] };
|
|
const isolation = comparison.available
|
|
? await pollMemorySearch(request, comparison.id, memoryAnchor, memoryAnchor, 'Isolation', auth)
|
|
: { found: false, checked: false, evidence: ['Isolation search skipped because the comparison workspace was unavailable.'] };
|
|
const isolationProven = comparison.available && isolation.checked && !isolation.found;
|
|
return {
|
|
saved: save.saved,
|
|
recalled: save.saved && recall.found,
|
|
isolated: isolationProven,
|
|
evidence: [
|
|
...primary.evidence,
|
|
...comparison.evidence,
|
|
...save.evidence,
|
|
save.saved ? 'Persona memory anchor saved.' : 'Persona memory anchor could not be saved.',
|
|
...recall.evidence,
|
|
...isolation.evidence,
|
|
comparison.available && isolation.found
|
|
? 'Potential cross-workspace memory leakage detected.'
|
|
: isolationProven
|
|
? 'No cross-workspace recall detected for the persona anchor within the bounded polling window.'
|
|
: 'Cross-workspace isolation was not proven by the bounded isolation search.',
|
|
],
|
|
};
|
|
}
|
|
|
|
async function probeWorkflowCompletion(request: APIRequestContext, persona: PersonaDefinition, auth?: ProbeAuth, probeWorkspace?: ProbeWorkspace, runId = ''): Promise<WorkflowCompletion> {
|
|
const workspaceName = `ai-os-flow-${persona.id}-${Date.now()}`;
|
|
const workspace = probeWorkspace ?? await ensureProbeWorkspace(request, workspaceName, auth);
|
|
const evidence = [...workspace.evidence];
|
|
if (!workspace.available) {
|
|
return {
|
|
completed: false,
|
|
evidence: [...evidence, 'Workflow proof skipped because the probe workspace was unavailable.'],
|
|
};
|
|
}
|
|
|
|
try {
|
|
const title = `${persona.name}: prove ${persona.oneToolCriterion}${runId ? ` (${runId})` : ''}`;
|
|
const createTask = await request.post(`${BASE}/api/workspaces/${encodeURIComponent(workspace.id)}/tasks`, {
|
|
timeout: API_PROBE_TIMEOUT_MS,
|
|
headers: auth?.headers,
|
|
data: {
|
|
title,
|
|
creatorName: 'Positioning audit',
|
|
assigneeName: persona.name,
|
|
},
|
|
});
|
|
const createBody = await readResponseBody(createTask);
|
|
const created = asRecord(createBody);
|
|
const taskId = typeof created.id === 'string' ? created.id : '';
|
|
evidence.push(`Workflow task create returned ${createTask.status()} with ${bodySummary(createBody)}.`);
|
|
if (!createTask.ok() || !taskId) {
|
|
return { completed: false, evidence: [...evidence, 'Workflow task was not created with a readable task id.'] };
|
|
}
|
|
|
|
const completeTask = await request.patch(`${BASE}/api/workspaces/${encodeURIComponent(workspace.id)}/tasks/${encodeURIComponent(taskId)}`, {
|
|
timeout: API_PROBE_TIMEOUT_MS,
|
|
headers: auth?.headers,
|
|
data: { status: 'done' },
|
|
});
|
|
const completeBody = await readResponseBody(completeTask);
|
|
evidence.push(`Workflow task completion returned ${completeTask.status()} with ${bodySummary(completeBody)}.`);
|
|
if (!completeTask.ok()) {
|
|
return { completed: false, evidence };
|
|
}
|
|
|
|
const listDone = await request.get(`${BASE}/api/workspaces/${encodeURIComponent(workspace.id)}/tasks?status=done`, {
|
|
timeout: API_PROBE_TIMEOUT_MS,
|
|
headers: auth?.headers,
|
|
});
|
|
const listBody = await readResponseBody(listDone);
|
|
const rows = Array.isArray(asRecord(listBody).tasks) ? asRecord(listBody).tasks as unknown[] : [];
|
|
const listedDone = rows.map(asRecord).some((task) => task.id === taskId && task.status === 'done');
|
|
evidence.push(`Workflow done-task list returned ${listDone.status()} with ${rows.length} done task(s).`);
|
|
return {
|
|
completed: listDone.ok() && listedDone,
|
|
evidence: [
|
|
...evidence,
|
|
listedDone
|
|
? 'Workflow proof completed: task was created, marked done, and found in the done list.'
|
|
: 'Workflow proof did not find the completed task in the done list.',
|
|
],
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
completed: false,
|
|
evidence: [...evidence, `Workflow proof failed: ${requestErrorSummary(error)}.`],
|
|
};
|
|
}
|
|
}
|
|
|
|
async function collectProbeContext(page: Page): Promise<ProbeContext> {
|
|
const consoleErrors: string[] = [];
|
|
page.on('console', (message) => {
|
|
if (message.type() === 'error') consoleErrors.push(message.text());
|
|
});
|
|
|
|
const coldHealth = await timedGet(page.request, '/health');
|
|
let shellText = '';
|
|
let shellLoaded = false;
|
|
const shellResponse = await page.goto(`${BASE}/home?${SKIP}`, { waitUntil: 'domcontentloaded' }).catch((error: unknown) => {
|
|
shellText = `Shell navigation failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
return null;
|
|
});
|
|
if (shellResponse) {
|
|
if (!shellResponse.ok()) {
|
|
shellText = `Shell navigation returned ${shellResponse.status()} ${shellResponse.statusText()}.`;
|
|
}
|
|
await page.waitForSelector('.waggle-sidebar, [role="navigation"], main', { timeout: 15_000 }).catch(() => {});
|
|
await page.waitForTimeout(600);
|
|
const bodyText = await page.locator('body').innerText().catch(() => '');
|
|
shellText = bodyText || shellText;
|
|
shellLoaded = shellResponse.ok() && shellText.length > 80 && !/error boundary|something went wrong/i.test(shellText);
|
|
}
|
|
|
|
const auth = await bootstrapProbeAuth(page.request);
|
|
const endpoints = {
|
|
health: '/health',
|
|
personas: '/api/personas',
|
|
skills: '/api/skills',
|
|
connectors: '/api/connectors',
|
|
marketplace: '/api/marketplace/search?query=&limit=10',
|
|
workspaces: '/api/workspaces',
|
|
hooks: '/api/hooks',
|
|
fleet: '/api/fleet',
|
|
events: '/api/events?limit=3',
|
|
tier: '/api/tier',
|
|
};
|
|
|
|
const apiEntries = await Promise.all(
|
|
Object.entries(endpoints).map(async ([key, path]) => [key, await timedGet(page.request, path, auth)] as const),
|
|
);
|
|
const api = Object.fromEntries(apiEntries);
|
|
const personaBody = api.personas?.body as { personas?: Array<{ id?: string; name?: string; description?: string }> } | Array<{ id?: string }>;
|
|
const personaRows = Array.isArray(personaBody) ? personaBody : Array.isArray(personaBody?.personas) ? personaBody.personas : [];
|
|
const personaIds = personaRows.map((persona) => String(persona.id ?? ''));
|
|
const runId = `audit-${Date.now()}`;
|
|
const workspacePool: ProbeWorkspacePool = {
|
|
primary: await ensureProbeWorkspace(page.request, `ai-os-audit-primary-${runId}`, auth),
|
|
comparison: await ensureProbeWorkspace(page.request, `ai-os-audit-isolation-${runId}`, auth),
|
|
};
|
|
|
|
const memory: ProbeContext['memory'] = {};
|
|
const memoryResults = await Promise.allSettled(
|
|
PERSONAS.map(async (persona) => ({
|
|
persona,
|
|
result: await probeMemory(page.request, persona, auth, workspacePool, runId),
|
|
})),
|
|
);
|
|
for (let index = 0; index < memoryResults.length; index += 1) {
|
|
const result = memoryResults[index];
|
|
const persona = PERSONAS[index];
|
|
if (result.status === 'fulfilled') {
|
|
memory[result.value.persona.id] = result.value.result;
|
|
} else {
|
|
memory[persona.id] = {
|
|
saved: false,
|
|
recalled: false,
|
|
isolated: false,
|
|
evidence: [`Memory probe failed for ${persona.name}: ${requestErrorSummary(result.reason)}.`],
|
|
};
|
|
}
|
|
}
|
|
|
|
const workflowCompletion: ProbeContext['workflowCompletion'] = {};
|
|
const workflowResults = await Promise.allSettled(
|
|
PERSONAS.map(async (persona) => ({
|
|
persona,
|
|
result: await probeWorkflowCompletion(page.request, persona, auth, workspacePool.primary, runId),
|
|
})),
|
|
);
|
|
for (let index = 0; index < workflowResults.length; index += 1) {
|
|
const result = workflowResults[index];
|
|
const persona = PERSONAS[index];
|
|
if (result.status === 'fulfilled') {
|
|
workflowCompletion[result.value.persona.id] = result.value.result;
|
|
} else {
|
|
workflowCompletion[persona.id] = {
|
|
completed: false,
|
|
evidence: [`Workflow probe failed for ${persona.name}: ${requestErrorSummary(result.reason)}.`],
|
|
};
|
|
}
|
|
}
|
|
|
|
return {
|
|
shellLoaded,
|
|
shellText,
|
|
consoleErrors,
|
|
coldHealth,
|
|
authEvidence: auth.evidence,
|
|
api,
|
|
personaIds,
|
|
personaText: JSON.stringify(api.personas?.body ?? ''),
|
|
skillsText: JSON.stringify(api.skills?.body ?? ''),
|
|
connectorsText: JSON.stringify(api.connectors?.body ?? ''),
|
|
marketplaceText: JSON.stringify(api.marketplace?.body ?? ''),
|
|
memory,
|
|
workflowCompletion,
|
|
};
|
|
}
|
|
|
|
function scorePersona(persona: PersonaDefinition, context: ProbeContext): PersonaScore {
|
|
const memory = context.memory[persona.id];
|
|
const workflowCompletion = context.workflowCompletion?.[persona.id];
|
|
const coreApiKeys = ['health', 'personas', 'workspaces'];
|
|
const fastCoreApis = coreApiKeys.filter((key) => context.api[key]?.ok && context.api[key].ms < 800);
|
|
const coreApiEvidence = coreApiKeys.map((key) => {
|
|
const result = context.api[key];
|
|
if (!result) return `${key} was not probed.`;
|
|
if (result.ok && result.ms < 800) return `${key} responded in ${result.ms}ms.`;
|
|
if (result.ok) return `${key} responded in ${result.ms}ms, slower than the 800ms target.`;
|
|
return `${key} returned ${result.status || 'no status'} in ${result.ms}ms with ${bodySummary(result.body)}.`;
|
|
});
|
|
const completedWorkflow = workflowCompletion?.completed === true;
|
|
const relevantPersona = persona.expectedPersonaIds.some((id) => context.personaIds.includes(id));
|
|
const relevantSkills = textIncludesAny(`${context.skillsText} ${context.marketplaceText}`, persona.expectedSkillTerms);
|
|
const relevantConnectors = textIncludesAny(context.connectorsText, persona.expectedConnectorTerms);
|
|
const osSurfaces = ['hooks', 'fleet', 'events', 'tier'].filter((key) => context.api[key]?.ok || [403, 404].includes(context.api[key]?.status ?? 0));
|
|
const hasBasicShellPositioning = /Waggle|workspace|AI/i.test(context.shellText);
|
|
const hasPersonalPositioning = hasPersonalWorkAwarePositioning(context.shellText);
|
|
|
|
const dimensions: DimensionScore[] = [
|
|
{
|
|
id: 'onboarding',
|
|
label: 'Onboarding clarity',
|
|
max: 15,
|
|
score: clampScore(
|
|
(context.shellLoaded ? 10 : 0)
|
|
+ (context.consoleErrors.length === 0 ? 3 : 0)
|
|
+ (hasBasicShellPositioning ? 1 : 0)
|
|
+ (hasPersonalPositioning ? 1 : 0),
|
|
15,
|
|
),
|
|
evidence: [
|
|
context.shellLoaded ? 'App shell loaded meaningful content.' : 'App shell did not load meaningful content.',
|
|
`${context.consoleErrors.length} console error(s) captured on first load.`,
|
|
hasPersonalPositioning ? 'Loaded shell clearly positions Waggle as a personal, work-aware AI workspace.' : 'Loaded shell does not clearly position Waggle as personal, work-aware, and guided by AI.',
|
|
],
|
|
gaps: [
|
|
...(context.shellLoaded ? [] : ['Make first-load shell resilient and clearly explain what Waggle is.']),
|
|
...(hasPersonalPositioning ? [] : ['Personal work-aware positioning is not explicit in the loaded shell text.']),
|
|
],
|
|
},
|
|
{
|
|
id: 'timeToValue',
|
|
label: 'Time to first value',
|
|
max: 15,
|
|
score: clampScore(fastCoreApis.length * 4 + (context.api.marketplace?.ok ? 3 : 0), 15),
|
|
evidence: [...(context.authEvidence ?? []), ...coreApiEvidence],
|
|
gaps: fastCoreApis.length >= 3 ? [] : ['Core first-value APIs should respond quickly and consistently.'],
|
|
},
|
|
{
|
|
id: 'memory',
|
|
label: 'Memory and continuity',
|
|
max: 20,
|
|
score: clampScore((memory?.saved ? 7 : 0) + (memory?.recalled ? 8 : 0) + (memory?.isolated ? 5 : 0), 20),
|
|
evidence: memory?.evidence ?? ['Memory probe did not run.'],
|
|
gaps: [
|
|
...(memory?.saved ? [] : ['Memory anchor save failed.']),
|
|
...(memory?.recalled ? [] : ['Saved memory was not confidently recalled.']),
|
|
...(memory?.isolated ? [] : ['Workspace isolation was not proven by this audit.']),
|
|
],
|
|
},
|
|
{
|
|
id: 'workflowCoverage',
|
|
label: 'Workflow coverage',
|
|
max: 15,
|
|
score: clampScore((relevantPersona ? 4 : 0) + (relevantSkills ? 4 : 0) + (relevantConnectors ? 4 : 0) + (completedWorkflow ? 3 : 0), 15),
|
|
evidence: [
|
|
relevantPersona ? 'Relevant persona is present.' : `Missing obvious persona match from ${persona.expectedPersonaIds.join(', ')}.`,
|
|
relevantSkills ? 'Relevant skill or marketplace language found.' : `No clear skill match for ${persona.expectedSkillTerms.join(', ')}.`,
|
|
relevantConnectors ? 'Relevant connector language found.' : `No clear connector match for ${persona.expectedConnectorTerms.join(', ')}.`,
|
|
completedWorkflow ? 'A real workflow was created, completed, and listed.' : 'No completed workflow proof was recorded.',
|
|
...(workflowCompletion?.evidence.slice(0, 3) ?? []),
|
|
],
|
|
gaps: [
|
|
...(relevantPersona ? [] : ['Add or surface a persona that matches this workflow.']),
|
|
...(relevantSkills ? [] : ['Improve skill/template coverage for this workflow.']),
|
|
...(relevantConnectors ? [] : ['Improve connector coverage or setup guidance for this workflow.']),
|
|
...(completedWorkflow ? [] : ['Actual workflow completion was not proven by this audit.']),
|
|
],
|
|
},
|
|
{
|
|
id: 'competitiveAdvantage',
|
|
label: 'Competitive advantage',
|
|
max: 15,
|
|
score: clampScore((memory?.recalled ? 5 : 0) + (osSurfaces.length >= 3 ? 5 : 0) + (relevantPersona && relevantSkills ? 5 : 0), 15),
|
|
evidence: [
|
|
persona.competitorBaseline,
|
|
`${osSurfaces.length}/4 OS-like surfaces responded or degraded gracefully.`,
|
|
],
|
|
gaps: osSurfaces.length >= 3 && memory?.recalled ? [] : ['Make the advantage over the current default more visible and more provable.'],
|
|
},
|
|
{
|
|
id: 'addiction',
|
|
label: 'Addiction/return signal',
|
|
max: 20,
|
|
score: clampScore((memory?.recalled ? 7 : 0) + (context.api.events?.ok ? 3 : 0) + (context.api.hooks?.ok ? 4 : 0) + (context.api.workspaces?.ok ? 3 : 0) + (context.api.tier?.ok ? 3 : 0), 20),
|
|
evidence: [
|
|
`External trigger need: ${persona.externalTriggerNeed}.`,
|
|
`Internal trigger need: ${persona.internalTriggerNeed}.`,
|
|
],
|
|
gaps: [
|
|
...(context.api.hooks?.ok ? [] : ['Durable external trigger surface is weak or not reachable.']),
|
|
...(memory?.recalled ? [] : ['Stored value is not strong enough to create a return habit.']),
|
|
],
|
|
},
|
|
];
|
|
|
|
const total = dimensions.reduce((sum, dimension) => sum + dimension.score, 0);
|
|
return {
|
|
id: persona.id,
|
|
name: persona.name,
|
|
role: persona.role,
|
|
total,
|
|
grade: gradeFor(total),
|
|
positioning: total >= 75 ? 'Can credibly position Waggle as a personal AI workspace for this persona.' : 'Needs sharper proof before personal AI workspace positioning will feel earned.',
|
|
currentDefault: persona.currentDefault,
|
|
competitorBaseline: persona.competitorBaseline,
|
|
oneToolCriterion: persona.oneToolCriterion,
|
|
dimensions,
|
|
improvementAreas: dimensions.flatMap((dimension) => dimension.gaps.map((gap) => `${dimension.label}: ${gap}`)),
|
|
};
|
|
}
|
|
|
|
function collectImprovementAreas(personas: PersonaScore[]): ImprovementArea[] {
|
|
const areas: ImprovementArea[] = [];
|
|
for (const persona of personas) {
|
|
for (const dimension of persona.dimensions) {
|
|
const impact = dimension.max - dimension.score;
|
|
if (impact <= 0) continue;
|
|
areas.push({
|
|
priority: 0,
|
|
personaId: persona.id,
|
|
personaName: persona.name,
|
|
dimension: dimension.label,
|
|
impact,
|
|
recommendation: dimension.gaps[0] ?? `Improve ${dimension.label.toLowerCase()} for ${persona.name}.`,
|
|
evidence: dimension.evidence,
|
|
});
|
|
}
|
|
}
|
|
const sorted = areas.sort((a, b) => b.impact - a.impact || a.dimension.localeCompare(b.dimension) || a.personaName.localeCompare(b.personaName));
|
|
const buckets = new Map<number, ImprovementArea[]>();
|
|
for (const area of sorted) {
|
|
buckets.set(area.impact, [...(buckets.get(area.impact) ?? []), area]);
|
|
}
|
|
|
|
const selected: ImprovementArea[] = [];
|
|
const selectedKeys = new Set<string>();
|
|
const addArea = (area: ImprovementArea) => {
|
|
const key = `${area.personaId}:${area.dimension}`;
|
|
if (selectedKeys.has(key) || selected.length >= 12) return;
|
|
selected.push(area);
|
|
selectedKeys.add(key);
|
|
};
|
|
|
|
for (const impact of [...buckets.keys()].sort((a, b) => b - a)) {
|
|
const remaining = [...(buckets.get(impact) ?? [])];
|
|
const seenDimensions = new Set<string>();
|
|
while (remaining.length > 0 && selected.length < 12) {
|
|
let nextIndex = remaining.findIndex((area) => !seenDimensions.has(area.dimension));
|
|
if (nextIndex === -1) {
|
|
seenDimensions.clear();
|
|
nextIndex = 0;
|
|
}
|
|
const [area] = remaining.splice(nextIndex, 1);
|
|
seenDimensions.add(area.dimension);
|
|
addArea(area);
|
|
}
|
|
}
|
|
|
|
return selected.map((area, index) => ({ ...area, priority: index + 1 }));
|
|
}
|
|
|
|
function hasPersonaDimensionBelowMax(personas: PersonaScore[]): boolean {
|
|
return personas.some((persona) => persona.dimensions.some((dimension) => dimension.score < dimension.max));
|
|
}
|
|
|
|
function positioningVerdict(score: number): string {
|
|
if (score >= 85) return 'Waggle can lead with personal AI workspace positioning now, with persona-specific proof.';
|
|
if (score >= 70) return 'Waggle has credible personal AI workspace positioning for selected niches, but first-session proof must sharpen.';
|
|
if (score >= 55) return 'Waggle should position as a memory-native AI workspace before making broader command-center claims.';
|
|
return 'Waggle should fix core value proof before making broad market claims.';
|
|
}
|
|
|
|
function scoreColdStart(context: ProbeContext): ColdStartScore {
|
|
const warmHealth = context.api.health;
|
|
const cold = context.coldHealth;
|
|
const coldLatency = cold.ok
|
|
? cold.ms < 800
|
|
? 35
|
|
: cold.ms < 1500
|
|
? 22
|
|
: 10
|
|
: 0;
|
|
const warmLatency = warmHealth?.ok
|
|
? warmHealth.ms < 800
|
|
? 25
|
|
: warmHealth.ms < 1500
|
|
? 15
|
|
: 5
|
|
: 0;
|
|
const score = clampScore(
|
|
coldLatency
|
|
+ warmLatency
|
|
+ (context.shellLoaded ? 25 : 0)
|
|
+ (context.consoleErrors.length === 0 ? 15 : 0),
|
|
100,
|
|
);
|
|
const gaps = [
|
|
...(cold.ok ? [] : ['Cold health probe did not respond successfully.']),
|
|
...(cold.ok && cold.ms < 800 ? [] : ['Cold health probe should return under 800ms.']),
|
|
...(warmHealth?.ok && warmHealth.ms < 800 ? [] : ['Warm health probe should return under 800ms.']),
|
|
...(context.shellLoaded ? [] : ['First meaningful shell did not load.']),
|
|
...(context.consoleErrors.length === 0 ? [] : ['Console errors appeared during first load.']),
|
|
];
|
|
return {
|
|
score,
|
|
grade: coldStartGradeFor(score),
|
|
evidence: [
|
|
`Cold health returned ${cold.status || 'no status'} in ${cold.ms}ms with ${bodySummary(cold.body)}.`,
|
|
warmHealth
|
|
? `Warm health returned ${warmHealth.status || 'no status'} in ${warmHealth.ms}ms with ${bodySummary(warmHealth.body)}.`
|
|
: 'Warm health was not probed.',
|
|
context.shellLoaded ? 'First meaningful shell loaded.' : 'First meaningful shell did not load.',
|
|
`${context.consoleErrors.length} console error(s) captured on first load.`,
|
|
],
|
|
gaps,
|
|
};
|
|
}
|
|
|
|
async function runAiOsPositioningAudit(page: Page, testInfo: TestInfo): Promise<AuditResult> {
|
|
const context = await collectProbeContext(page);
|
|
const personas = PERSONAS.map((persona) => scorePersona(persona, context));
|
|
const overallScore = Math.round(personas.reduce((sum, persona) => sum + persona.total, 0) / personas.length);
|
|
const coldStart = scoreColdStart(context);
|
|
const generatedAt = new Date().toISOString();
|
|
const markdownPath = testInfo.outputPath('ai-os-positioning-audit.md');
|
|
const jsonPath = testInfo.outputPath('ai-os-positioning-audit.json');
|
|
const addictionScore = Math.round(
|
|
personas.reduce((sum, persona) => {
|
|
const addiction = persona.dimensions.find((dimension) => dimension.id === 'addiction');
|
|
return sum + (addiction ? (addiction.score / addiction.max) * 100 : 0);
|
|
}, 0) / personas.length,
|
|
);
|
|
const audit: AuditResult = {
|
|
generatedAt,
|
|
overall: {
|
|
score: overallScore,
|
|
grade: gradeFor(overallScore),
|
|
coldStartScore: coldStart.score,
|
|
coldStartGrade: coldStart.grade,
|
|
positioningVerdict: positioningVerdict(overallScore),
|
|
},
|
|
coldStart,
|
|
addictionLevel: addictionLevelFor(addictionScore),
|
|
personas,
|
|
improvementAreas: collectImprovementAreas(personas),
|
|
artifacts: {
|
|
markdownPath,
|
|
jsonPath,
|
|
},
|
|
};
|
|
|
|
const markdown = renderMarkdown(audit);
|
|
await mkdir(dirname(markdownPath), { recursive: true });
|
|
await writeFile(markdownPath, markdown, 'utf8');
|
|
await writeFile(jsonPath, JSON.stringify(audit, null, 2), 'utf8');
|
|
await testInfo.attach('ai-os-positioning-audit.md', { path: markdownPath, contentType: 'text/markdown' });
|
|
await testInfo.attach('ai-os-positioning-audit.json', { path: jsonPath, contentType: 'application/json' });
|
|
return audit;
|
|
}
|
|
|
|
test.describe('personal AI workspace positioning audit', () => {
|
|
test('awards onboarding clarity for personal work-aware positioning without literal AI OS language', () => {
|
|
const persona = PERSONAS[0];
|
|
const context: ProbeContext = {
|
|
shellLoaded: true,
|
|
shellText: 'Waggle is the personal AI workspace that remembers you, knows your projects, evolves with your work, and guides the next step while it runs the AI underneath.',
|
|
consoleErrors: [],
|
|
coldHealth: { status: 200, ok: true, ms: 100, body: {} },
|
|
api: {},
|
|
personaIds: [],
|
|
personaText: '',
|
|
skillsText: '',
|
|
connectorsText: '',
|
|
marketplaceText: '',
|
|
memory: {},
|
|
workflowCompletion: {},
|
|
};
|
|
|
|
const score = scorePersona(persona, context);
|
|
const onboarding = score.dimensions.find((dimension) => dimension.id === 'onboarding');
|
|
|
|
expect(context.shellText).not.toMatch(/AI OS|operating system/i);
|
|
expect(onboarding?.score).toBe(15);
|
|
expect(onboarding?.gaps).not.toContain('Personal work-aware positioning is not explicit in the loaded shell text.');
|
|
expect(onboarding?.evidence).toContain('Loaded shell clearly positions Waggle as a personal, work-aware AI workspace.');
|
|
});
|
|
|
|
test('keeps scoring evidence internally consistent and diverse', () => {
|
|
const persona = PERSONAS[0];
|
|
const context: ProbeContext = {
|
|
shellLoaded: true,
|
|
shellText: 'Waggle workspace AI for customer follow-ups.',
|
|
consoleErrors: [],
|
|
api: {
|
|
health: { status: 200, ok: true, ms: 100, body: {} },
|
|
personas: { status: 200, ok: true, ms: 950, body: {} },
|
|
workspaces: { status: 503, ok: false, ms: 120, body: { error: 'down' } },
|
|
marketplace: { status: 200, ok: true, ms: 100, body: {} },
|
|
hooks: { status: 200, ok: true, ms: 100, body: {} },
|
|
fleet: { status: 200, ok: true, ms: 100, body: {} },
|
|
events: { status: 200, ok: true, ms: 100, body: {} },
|
|
tier: { status: 200, ok: true, ms: 100, body: {} },
|
|
},
|
|
personaIds: persona.expectedPersonaIds,
|
|
personaText: '',
|
|
skillsText: persona.expectedSkillTerms.join(' '),
|
|
connectorsText: persona.expectedConnectorTerms.join(' '),
|
|
marketplaceText: '',
|
|
memory: {
|
|
[persona.id]: { saved: true, recalled: true, isolated: true, evidence: ['Memory proof exists.'] },
|
|
},
|
|
};
|
|
|
|
const score = scorePersona(persona, context);
|
|
const onboarding = score.dimensions.find((dimension) => dimension.id === 'onboarding');
|
|
const timeToValue = score.dimensions.find((dimension) => dimension.id === 'timeToValue');
|
|
|
|
expect(onboarding?.gaps).toContain('Personal work-aware positioning is not explicit in the loaded shell text.');
|
|
expect(onboarding?.score).toBeLessThan(onboarding?.max ?? 0);
|
|
expect(timeToValue?.evidence.join('\n')).toContain('personas responded in 950ms');
|
|
expect(timeToValue?.evidence.join('\n')).toContain('workspaces returned 503');
|
|
|
|
const crowdedPersonas: PersonaScore[] = PERSONAS.map((definition) => ({
|
|
id: definition.id,
|
|
name: definition.name,
|
|
role: definition.role,
|
|
total: 0,
|
|
grade: 'test',
|
|
positioning: 'test',
|
|
currentDefault: definition.currentDefault,
|
|
competitorBaseline: definition.competitorBaseline,
|
|
oneToolCriterion: definition.oneToolCriterion,
|
|
improvementAreas: [],
|
|
dimensions: [
|
|
{ id: 'memory', label: 'Memory and continuity', max: 20, score: 0, evidence: [`${definition.name} memory`], gaps: ['Memory gap.'] },
|
|
{ id: 'addiction', label: 'Addiction/return signal', max: 20, score: 0, evidence: [`${definition.name} addiction`], gaps: ['Addiction gap.'] },
|
|
{ id: 'timeToValue', label: 'Time to first value', max: 15, score: 5, evidence: [`${definition.name} time`], gaps: ['Time gap.'] },
|
|
{ id: 'onboarding', label: 'Onboarding clarity', max: 15, score: 14, evidence: [`${definition.name} onboarding`], gaps: ['Small onboarding gap.'] },
|
|
],
|
|
}));
|
|
|
|
const orderedAreas = collectImprovementAreas(crowdedPersonas);
|
|
for (let index = 1; index < orderedAreas.length; index += 1) {
|
|
expect(orderedAreas[index - 1].impact).toBeGreaterThanOrEqual(orderedAreas[index].impact);
|
|
}
|
|
const topFiveDimensions = new Set(orderedAreas.slice(0, 5).map((area) => area.dimension));
|
|
expect(topFiveDimensions.size).toBeGreaterThan(1);
|
|
});
|
|
|
|
test('allows perfect-score audits to report zero improvement areas', () => {
|
|
const perfectDimensions: DimensionScore[] = [
|
|
{ id: 'onboarding', label: 'Onboarding clarity', max: 15, score: 15, evidence: ['Perfect onboarding.'], gaps: [] },
|
|
{ id: 'timeToValue', label: 'Time to first value', max: 15, score: 15, evidence: ['Perfect time to value.'], gaps: [] },
|
|
{ id: 'memory', label: 'Memory and continuity', max: 20, score: 20, evidence: ['Perfect memory.'], gaps: [] },
|
|
{ id: 'workflowCoverage', label: 'Workflow coverage', max: 15, score: 15, evidence: ['Perfect workflow coverage.'], gaps: [] },
|
|
{ id: 'competitiveAdvantage', label: 'Competitive advantage', max: 15, score: 15, evidence: ['Perfect advantage.'], gaps: [] },
|
|
{ id: 'addiction', label: 'Addiction/return signal', max: 20, score: 20, evidence: ['Perfect return signal.'], gaps: [] },
|
|
];
|
|
const perfectPersonas: PersonaScore[] = PERSONAS.map((definition) => ({
|
|
id: definition.id,
|
|
name: definition.name,
|
|
role: definition.role,
|
|
total: 100,
|
|
grade: gradeFor(100),
|
|
positioning: 'Can credibly position Waggle as a personal AI workspace for this persona.',
|
|
currentDefault: definition.currentDefault,
|
|
competitorBaseline: definition.competitorBaseline,
|
|
oneToolCriterion: definition.oneToolCriterion,
|
|
dimensions: perfectDimensions.map((dimension) => ({ ...dimension })),
|
|
improvementAreas: [],
|
|
}));
|
|
const improvementAreas = collectImprovementAreas(perfectPersonas);
|
|
const audit: AuditResult = {
|
|
generatedAt: '2026-06-28T00:00:00.000Z',
|
|
overall: {
|
|
score: 100,
|
|
grade: gradeFor(100),
|
|
coldStartScore: 100,
|
|
coldStartGrade: coldStartGradeFor(100),
|
|
positioningVerdict: positioningVerdict(100),
|
|
},
|
|
coldStart: {
|
|
score: 100,
|
|
grade: coldStartGradeFor(100),
|
|
evidence: ['Perfect cold start.'],
|
|
gaps: [],
|
|
},
|
|
addictionLevel: addictionLevelFor(100),
|
|
personas: perfectPersonas,
|
|
improvementAreas,
|
|
artifacts: {
|
|
markdownPath: 'ai-os-positioning-audit.md',
|
|
jsonPath: 'ai-os-positioning-audit.json',
|
|
},
|
|
};
|
|
|
|
expect(hasPersonaDimensionBelowMax(perfectPersonas)).toBe(false);
|
|
expect(improvementAreas).toHaveLength(0);
|
|
expect(renderMarkdown(audit)).toContain('## Improvement Areas');
|
|
});
|
|
|
|
test('bounds probe requests and polls delayed memory recall', async () => {
|
|
const calls: Array<{ method: 'get' | 'post'; url: string; options?: { timeout?: number; data?: Record<string, unknown> } }> = [];
|
|
const response = (status: number, body: unknown) => ({
|
|
ok: () => status >= 200 && status < 300,
|
|
status: () => status,
|
|
statusText: () => String(status),
|
|
json: async () => body,
|
|
text: async () => JSON.stringify(body),
|
|
});
|
|
const timeoutRequest = {
|
|
get: async (url: string, options?: { timeout?: number }) => {
|
|
calls.push({ method: 'get', url, options });
|
|
return response(200, { results: [{ content: 'query content' }] });
|
|
},
|
|
post: async (url: string, options?: { timeout?: number; data?: Record<string, unknown> }) => {
|
|
calls.push({ method: 'post', url, options });
|
|
return response(200, { id: options?.data?.name ?? 'workspace', saved: true, frameId: 'frame-1' });
|
|
},
|
|
} as unknown as APIRequestContext;
|
|
|
|
await timedGet(timeoutRequest, '/health');
|
|
await ensureProbeWorkspace(timeoutRequest, 'workspace');
|
|
await saveMemory(timeoutRequest, 'workspace', 'content');
|
|
await searchMemory(timeoutRequest, 'workspace', 'query', 'query content');
|
|
|
|
expect(calls.every((call) => typeof call.options?.timeout === 'number' && call.options.timeout > 0)).toBe(true);
|
|
|
|
const memorySaveResult = async (body: unknown) => saveMemory({
|
|
post: async () => response(200, body),
|
|
} as unknown as APIRequestContext, 'workspace', 'content');
|
|
await expect(memorySaveResult({})).resolves.toMatchObject({ saved: false });
|
|
await expect(memorySaveResult({ saved: false })).resolves.toMatchObject({ saved: false });
|
|
await expect(memorySaveResult({ saved: true })).resolves.toMatchObject({ saved: true });
|
|
await expect(memorySaveResult({ duplicate: true })).resolves.toMatchObject({ saved: true });
|
|
await expect(memorySaveResult({ frameId: 42 })).resolves.toMatchObject({ saved: true });
|
|
await expect(memorySaveResult({ frame: { id: 'frame-42' } })).resolves.toMatchObject({ saved: true });
|
|
|
|
let recallSearches = 0;
|
|
const persona = PERSONAS[0];
|
|
const pollingRequest = {
|
|
post: async (url: string, options?: { timeout?: number; data?: Record<string, unknown> }) => {
|
|
calls.push({ method: 'post', url, options });
|
|
if (url.includes('/api/workspaces')) return response(200, { id: options?.data?.name });
|
|
return response(200, { saved: true, frameId: 'frame-1' });
|
|
},
|
|
get: async (url: string, options?: { timeout?: number }) => {
|
|
calls.push({ method: 'get', url, options });
|
|
const decodedUrl = decodeURIComponent(url);
|
|
if (decodedUrl.includes('-isolation')) return response(200, { results: [] });
|
|
recallSearches += 1;
|
|
if (recallSearches === 1) return response(200, { results: [] });
|
|
return response(200, { results: [{ content: persona.memoryAnchor }] });
|
|
},
|
|
} as unknown as APIRequestContext;
|
|
|
|
const memory = await probeMemory(pollingRequest, persona);
|
|
expect(recallSearches).toBeGreaterThan(1);
|
|
expect(memory.recalled).toBe(true);
|
|
});
|
|
|
|
test('collects persona memory probes concurrently and records per-persona failures as evidence', async () => {
|
|
const response = (status: number, body: unknown) => ({
|
|
ok: () => status >= 200 && status < 300,
|
|
status: () => status,
|
|
statusText: () => String(status),
|
|
json: async () => body,
|
|
text: async () => JSON.stringify(body),
|
|
});
|
|
let activeSaves = 0;
|
|
let maxActiveSaves = 0;
|
|
const request = {
|
|
get: async (url: string) => {
|
|
if (url.includes('/api/personas')) {
|
|
return response(200, { personas: [...new Set(PERSONAS.flatMap((persona) => persona.expectedPersonaIds))].map((id) => ({ id })) });
|
|
}
|
|
if (url.includes('/api/memory/search')) {
|
|
const parsed = new URL(url);
|
|
const workspace = parsed.searchParams.get('workspace') ?? '';
|
|
return response(200, {
|
|
results: workspace.includes('isolation') ? [] : [{ content: PERSONAS.map((definition) => definition.memoryAnchor).join(' ') }],
|
|
});
|
|
}
|
|
return response(200, {});
|
|
},
|
|
post: (url: string, options?: { timeout?: number; data?: Record<string, unknown> }) => {
|
|
if (url.includes('/api/workspaces')) {
|
|
const workspaceName = String(options?.data?.name ?? '');
|
|
if (workspaceName.endsWith('-isolation')) return Promise.resolve(response(500, { error: 'comparison unavailable' }));
|
|
return Promise.resolve(response(200, { id: workspaceName }));
|
|
}
|
|
if (url.includes('/api/memory/frames')) {
|
|
const content = String(options?.data?.content ?? '');
|
|
if (content.includes(PERSONAS[0].memoryAnchor)) {
|
|
throw new Error('sync memory probe failure');
|
|
}
|
|
activeSaves += 1;
|
|
maxActiveSaves = Math.max(maxActiveSaves, activeSaves);
|
|
return new Promise<ReturnType<typeof response>>((resolve) => {
|
|
setTimeout(() => {
|
|
activeSaves -= 1;
|
|
resolve(response(200, { saved: true, frameId: `frame-${activeSaves}` }));
|
|
}, 25);
|
|
});
|
|
}
|
|
return Promise.resolve(response(200, {}));
|
|
},
|
|
} as unknown as APIRequestContext;
|
|
const page = {
|
|
on: () => undefined,
|
|
goto: async () => response(200, {}),
|
|
waitForSelector: async () => undefined,
|
|
waitForTimeout: async () => undefined,
|
|
locator: () => ({
|
|
innerText: async () => 'Waggle personal AI workspace with meaningful app shell content for the audit.',
|
|
}),
|
|
request,
|
|
} as unknown as Page;
|
|
|
|
const context = await collectProbeContext(page);
|
|
const failedProbe = context.memory[PERSONAS[0].id];
|
|
|
|
expect(Object.keys(context.memory)).toHaveLength(PERSONAS.length);
|
|
expect(failedProbe.saved).toBe(false);
|
|
expect(failedProbe.evidence.join('\n')).toContain('sync memory probe failure');
|
|
expect(maxActiveSaves).toBeGreaterThan(1);
|
|
});
|
|
|
|
test('bootstraps session token before probing protected product APIs', async () => {
|
|
const response = (status: number, body: unknown) => ({
|
|
ok: () => status >= 200 && status < 300,
|
|
status: () => status,
|
|
statusText: () => String(status),
|
|
json: async () => body,
|
|
text: async () => JSON.stringify(body),
|
|
});
|
|
const hasAuth = (options?: { headers?: Record<string, string> }) => options?.headers?.Authorization === 'Bearer audit-token';
|
|
let tokenCalls = 0;
|
|
const protectedProbeUrls: string[] = [];
|
|
const request = {
|
|
get: async (url: string, options?: { timeout?: number; headers?: Record<string, string> }) => {
|
|
if (url.includes('/api/auth/session-token')) {
|
|
tokenCalls += 1;
|
|
return response(200, { token: 'audit-token' });
|
|
}
|
|
if (url.includes('/health')) return response(200, { status: 'ok' });
|
|
if (url.includes('/api/memory/search')) {
|
|
protectedProbeUrls.push(url);
|
|
if (!hasAuth(options)) return response(401, { error: 'Unauthorized', code: 'MISSING_TOKEN' });
|
|
const parsed = new URL(url);
|
|
const workspace = parsed.searchParams.get('workspace') ?? '';
|
|
const persona = PERSONAS.find((definition) => workspace.includes(definition.id));
|
|
return response(200, { results: workspace.includes('-isolation') || !persona ? [] : [{ content: persona.memoryAnchor }] });
|
|
}
|
|
if (url.includes('/api/personas')) {
|
|
protectedProbeUrls.push(url);
|
|
return hasAuth(options)
|
|
? response(200, { personas: PERSONAS.flatMap((persona) => persona.expectedPersonaIds).map((id) => ({ id })) })
|
|
: response(401, { error: 'Unauthorized', code: 'MISSING_TOKEN' });
|
|
}
|
|
if (url.includes('/api/')) {
|
|
protectedProbeUrls.push(url);
|
|
return hasAuth(options) ? response(200, {}) : response(401, { error: 'Unauthorized', code: 'MISSING_TOKEN' });
|
|
}
|
|
return response(200, {});
|
|
},
|
|
post: async (url: string, options?: { timeout?: number; headers?: Record<string, string>; data?: Record<string, unknown> }) => {
|
|
if (url.includes('/api/')) protectedProbeUrls.push(url);
|
|
if (!hasAuth(options)) return response(401, { error: 'Unauthorized', code: 'MISSING_TOKEN' });
|
|
if (url.includes('/api/workspaces')) return response(200, { id: options?.data?.name ?? 'workspace' });
|
|
if (url.includes('/api/memory/frames')) return response(200, { saved: true, frameId: 'frame-1' });
|
|
return response(200, {});
|
|
},
|
|
} as unknown as APIRequestContext;
|
|
const page = {
|
|
on: () => undefined,
|
|
goto: async () => response(200, {}),
|
|
waitForSelector: async () => undefined,
|
|
waitForTimeout: async () => undefined,
|
|
locator: () => ({
|
|
innerText: async () => 'Waggle personal AI workspace with meaningful app shell content for the audit.',
|
|
}),
|
|
request,
|
|
} as unknown as Page;
|
|
|
|
const context = await collectProbeContext(page);
|
|
|
|
expect(tokenCalls).toBe(1);
|
|
expect(context.api.personas.ok).toBe(true);
|
|
expect(context.api.workspaces.ok).toBe(true);
|
|
expect(context.memory[PERSONAS[0].id].saved).toBe(true);
|
|
expect(protectedProbeUrls.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
// eslint-disable-next-line no-empty-pattern -- Playwright requires a destructuring pattern for fixtures; this test mocks page/request entirely
|
|
test('reports cold-start readiness separately from warm product score', async ({}, testInfo) => {
|
|
const response = (status: number, body: unknown) => ({
|
|
ok: () => status >= 200 && status < 300,
|
|
status: () => status,
|
|
statusText: () => String(status),
|
|
json: async () => body,
|
|
text: async () => JSON.stringify(body),
|
|
});
|
|
let healthCalls = 0;
|
|
const request = {
|
|
get: async (url: string) => {
|
|
if (url.includes('/api/auth/session-token')) return response(200, { token: 'audit-token' });
|
|
if (url.includes('/health')) {
|
|
healthCalls += 1;
|
|
if (healthCalls === 1) await wait(900);
|
|
return response(200, { status: 'ok' });
|
|
}
|
|
if (url.includes('/api/personas')) {
|
|
return response(200, { personas: PERSONAS.flatMap((persona) => persona.expectedPersonaIds).map((id) => ({ id })) });
|
|
}
|
|
if (url.includes('/api/memory/search')) {
|
|
const parsed = new URL(url);
|
|
const workspace = parsed.searchParams.get('workspace') ?? '';
|
|
return response(200, {
|
|
results: workspace.includes('isolation') ? [] : [{ content: PERSONAS.map((definition) => definition.memoryAnchor).join(' ') }],
|
|
});
|
|
}
|
|
if (url.includes('/tasks')) return response(200, { tasks: [{ id: 'task-1', status: 'done' }] });
|
|
return response(200, {
|
|
results: [{
|
|
name: 'gmail google slack notion calendar excel microsoft github mcp webhook email marketing document presentation research writing brand markdown spreadsheet csv analysis skill automation agent',
|
|
}],
|
|
});
|
|
},
|
|
post: async (url: string, options?: { data?: Record<string, unknown> }) => {
|
|
if (url.includes('/tasks')) return response(201, { id: 'task-1', title: options?.data?.title, status: 'open' });
|
|
if (url.includes('/api/workspaces')) return response(200, { id: options?.data?.name ?? 'workspace' });
|
|
if (url.includes('/api/memory/frames')) return response(200, { saved: true, frameId: 'frame-1' });
|
|
return response(200, {});
|
|
},
|
|
patch: async () => response(200, { id: 'task-1', status: 'done' }),
|
|
} as unknown as APIRequestContext;
|
|
const page = {
|
|
on: () => undefined,
|
|
goto: async () => response(200, {}),
|
|
waitForSelector: async () => undefined,
|
|
waitForTimeout: async () => undefined,
|
|
locator: () => ({
|
|
innerText: async () => 'Waggle is the personal AI workspace that remembers your work, keeps context, and helps you move from capture to finished outcomes.',
|
|
}),
|
|
request,
|
|
} as unknown as Page;
|
|
|
|
const audit = await runAiOsPositioningAudit(page, testInfo);
|
|
|
|
expect(audit.overall.score).toBeGreaterThanOrEqual(90);
|
|
expect(audit.overall.coldStartScore).toBeLessThan(audit.overall.score);
|
|
expect(audit.coldStart.evidence.join('\n')).toContain('Cold health');
|
|
expect(healthCalls).toBeGreaterThanOrEqual(2);
|
|
});
|
|
|
|
test('requires completed workflow proof before awarding full workflow coverage', () => {
|
|
const persona = PERSONAS[0];
|
|
const context: ProbeContext = {
|
|
shellLoaded: true,
|
|
shellText: 'Waggle workspace AI for customer follow-ups.',
|
|
consoleErrors: [],
|
|
api: {
|
|
health: { status: 200, ok: true, ms: 100, body: {} },
|
|
personas: { status: 200, ok: true, ms: 100, body: {} },
|
|
workspaces: { status: 200, ok: true, ms: 100, body: {} },
|
|
marketplace: { status: 200, ok: true, ms: 100, body: {} },
|
|
hooks: { status: 200, ok: true, ms: 100, body: {} },
|
|
fleet: { status: 200, ok: true, ms: 100, body: {} },
|
|
events: { status: 200, ok: true, ms: 100, body: {} },
|
|
tier: { status: 200, ok: true, ms: 100, body: {} },
|
|
},
|
|
personaIds: persona.expectedPersonaIds,
|
|
personaText: '',
|
|
skillsText: persona.expectedSkillTerms.join(' '),
|
|
connectorsText: persona.expectedConnectorTerms.join(' '),
|
|
marketplaceText: '',
|
|
memory: {
|
|
[persona.id]: { saved: true, recalled: true, isolated: true, evidence: ['Memory proof exists.'] },
|
|
},
|
|
workflowCompletion: {
|
|
[persona.id]: { completed: false, evidence: ['No completed workflow proof.'] },
|
|
},
|
|
};
|
|
|
|
const score = scorePersona(persona, context);
|
|
const workflow = score.dimensions.find((dimension) => dimension.id === 'workflowCoverage');
|
|
|
|
expect(workflow?.score).toBeLessThan(workflow?.max ?? 0);
|
|
expect(workflow?.gaps).toContain('Actual workflow completion was not proven by this audit.');
|
|
});
|
|
|
|
test('reuses bounded probe workspaces so report mode does not exhaust free-tier limits', async () => {
|
|
const response = (status: number, body: unknown) => ({
|
|
ok: () => status >= 200 && status < 300,
|
|
status: () => status,
|
|
statusText: () => String(status),
|
|
json: async () => body,
|
|
text: async () => JSON.stringify(body),
|
|
});
|
|
let workspaceCreates = 0;
|
|
const taskRows: Record<string, unknown> = {};
|
|
const request = {
|
|
get: async (url: string) => {
|
|
if (url.includes('/api/auth/session-token')) return response(200, { token: 'audit-token' });
|
|
if (url.includes('/health')) return response(200, { status: 'ok' });
|
|
if (url.includes('/api/personas')) {
|
|
return response(200, { personas: PERSONAS.flatMap((persona) => persona.expectedPersonaIds).map((id) => ({ id })) });
|
|
}
|
|
if (url.includes('/api/memory/search')) {
|
|
const parsed = new URL(url);
|
|
const workspace = parsed.searchParams.get('workspace') ?? '';
|
|
return response(200, {
|
|
results: workspace.includes('isolation') ? [] : [{ content: PERSONAS.map((persona) => persona.memoryAnchor).join(' ') }],
|
|
});
|
|
}
|
|
if (url.includes('/tasks')) return response(200, { tasks: Object.values(taskRows) });
|
|
return response(200, {
|
|
results: [{
|
|
name: 'gmail google slack notion calendar excel microsoft github mcp webhook email marketing document presentation research writing brand markdown spreadsheet csv analysis skill automation agent',
|
|
}],
|
|
});
|
|
},
|
|
post: async (url: string, options?: { data?: Record<string, unknown> }) => {
|
|
if (url.endsWith('/api/workspaces')) {
|
|
workspaceCreates += 1;
|
|
if (workspaceCreates > 2) return response(403, { error: 'Workspace limit reached for FREE tier (2 max).' });
|
|
return response(201, { id: options?.data?.name ?? `workspace-${workspaceCreates}` });
|
|
}
|
|
if (url.includes('/tasks')) {
|
|
const id = `task-${Object.keys(taskRows).length + 1}`;
|
|
taskRows[id] = { id, title: options?.data?.title, status: 'done' };
|
|
return response(201, { id, status: 'open' });
|
|
}
|
|
if (url.includes('/api/memory/frames')) return response(200, { saved: true, frameId: 'frame-1' });
|
|
return response(200, {});
|
|
},
|
|
patch: async (url: string) => {
|
|
const taskId = url.split('/').pop() ?? '';
|
|
taskRows[taskId] = { ...(asRecord(taskRows[taskId])), id: taskId, status: 'done' };
|
|
return response(200, taskRows[taskId]);
|
|
},
|
|
} as unknown as APIRequestContext;
|
|
const page = {
|
|
on: () => undefined,
|
|
goto: async () => response(200, {}),
|
|
waitForSelector: async () => undefined,
|
|
waitForTimeout: async () => undefined,
|
|
locator: () => ({
|
|
innerText: async () => 'Waggle is the personal AI workspace that remembers your work, keeps context, and helps you move from capture to finished outcomes.',
|
|
}),
|
|
request,
|
|
} as unknown as Page;
|
|
|
|
const context = await collectProbeContext(page);
|
|
|
|
expect(workspaceCreates).toBeLessThanOrEqual(2);
|
|
expect(Object.values(context.memory).every((memory) => memory.saved && memory.recalled)).toBe(true);
|
|
expect(Object.values(context.workflowCompletion).every((workflow) => workflow.completed)).toBe(true);
|
|
});
|
|
|
|
test('generates a report-mode audit with five personas and score-aware improvement areas', async ({ page }, testInfo) => {
|
|
const audit = await runAiOsPositioningAudit(page, testInfo);
|
|
const expectedPersonaNames = ['Sofia', 'Mara', 'Imran', 'Daniel', 'Priya'];
|
|
|
|
expect(audit.personas).toHaveLength(5);
|
|
expect(audit.personas.map((persona) => persona.name)).toEqual(expect.arrayContaining(expectedPersonaNames));
|
|
expect(audit.overall.score).toBeGreaterThanOrEqual(0);
|
|
expect(audit.overall.score).toBeLessThanOrEqual(100);
|
|
expect(audit.overall.grade).toMatch(/workspace|chat|niche|promising|plausible/i);
|
|
expect(audit.addictionLevel).toMatch(/weak|emerging|strong|very strong/i);
|
|
if (hasPersonaDimensionBelowMax(audit.personas)) {
|
|
expect(audit.improvementAreas.length).toBeGreaterThan(0);
|
|
} else {
|
|
expect(audit.improvementAreas).toHaveLength(0);
|
|
}
|
|
for (const area of audit.improvementAreas) {
|
|
expect(area.impact).toBeGreaterThan(0);
|
|
expect(area.recommendation.trim().length).toBeGreaterThan(10);
|
|
}
|
|
expect(audit.artifacts.markdownPath).toMatch(/ai-os-positioning-audit\.md$/);
|
|
expect(audit.artifacts.jsonPath).toMatch(/ai-os-positioning-audit\.json$/);
|
|
|
|
const markdown = await readFile(audit.artifacts.markdownPath, 'utf8');
|
|
expect(markdown).toContain('# Waggle Personal AI Workspace Positioning Audit');
|
|
expect(markdown).toContain('Overall score:');
|
|
expect(markdown).toContain('Positioning verdict:');
|
|
expect(markdown).toContain('Addiction level:');
|
|
expect(markdown).toContain('## Persona Scores');
|
|
expect(markdown).toContain('## Improvement Areas');
|
|
for (const name of expectedPersonaNames) {
|
|
expect(markdown).toContain(name);
|
|
}
|
|
});
|
|
});
|