#!/usr/bin/env tsx
/**
 * Phase 5 §0.3 — Cost projection probe.
 *
 * Per PM ratification 2026-04-29 Phase 5 §0 preflight Ask #2: AUTHORIZED.
 * Per brief §0.3 #3 + §5.4 cost ceiling computation.
 *
 * Sends 5 prompts of varying complexity to each Phase 5 deployed variant via
 * the existing LiteLLM proxy (matches Faza 1 runner pattern):
 *   - claude::gen1-v1 model alias = claude-opus-4-7
 *   - qwen-thinking::gen1-v1 model alias = qwen3.6-35b-a3b-via-dashscope-direct
 *
 * Records actual input/output tokens × pricing snapshot 2026-04-29.
 * Emits per-request JSONL + computes p50/p95/max.
 *
 * Validates canary_cost_p95_ceiling = 740 × max(p95) × 1.20 ≤ $20.
 *
 * Pricing snapshot (2026-04-29 — public docs URL):
 *   - Claude Opus 4.7: $5/M input, $25/M output (NEW Opus 4.7 pricing per
 *     platform.claude.com/docs/en/docs/about-claude/pricing 2026-04-29)
 *   - DashScope Qwen 35B-A3B intl thinking: $0.25/M input, $2/M output
 *     (alibabacloud.com/help/en/model-studio/billing-for-model-studio 2026-04-29)
 *
 * NOTE on Faza 1 pricing reference: run-checkpoint-c.ts uses different
 * historical pricing ($15/$75 for Opus, $0.20/$0.80 for Qwen). Phase 5
 * uses the 2026-04-29 snapshot above (Opus 4.7 reduced rate; Qwen intl
 * thinking-mode rate). Probe pricing is the binding source for §0.3.
 *
 * Usage: npx tsx gepa-phase-5/scripts/cost-probe.ts
 */

import * as fs from 'node:fs';
import * as path from 'node:path';
import { fileURLToPath } from 'node:url';

import { claudeGen1V1Shape } from '../../packages/agent/src/prompt-shapes/gepa-evolved/claude-gen1-v1.js';
import { qwenThinkingGen1V1Shape } from '../../packages/agent/src/prompt-shapes/gepa-evolved/qwen-thinking-gen1-v1.js';

// Manual .env load (no dotenv dependency; keep probe self-contained)
const envPath = path.resolve(process.cwd(), '.env');
if (fs.existsSync(envPath)) {
  const raw = fs.readFileSync(envPath, 'utf-8');
  for (const line of raw.split(/\r?\n/)) {
    const m = /^([A-Z_][A-Z0-9_]*)=(.*)$/.exec(line.trim());
    if (m && !process.env[m[1]]) {
      let v = m[2];
      if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) {
        v = v.slice(1, -1);
      }
      process.env[m[1]] = v;
    }
  }
}

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const REPO_ROOT = path.resolve(__dirname, '../..');
const OUT_JSONL = path.join(REPO_ROOT, 'gepa-phase-5/cost-probe-2026-04-29.jsonl');
const OUT_SUMMARY = path.join(REPO_ROOT, 'gepa-phase-5/cost-probe-2026-04-29-summary.md');

const LITELLM_URL = process.env.LITELLM_URL ?? 'http://localhost:4000';
const LITELLM_MASTER_KEY = process.env.LITELLM_MASTER_KEY;

// 2026-04-29 pricing snapshot (Phase 5 binding)
const PRICING = {
  'claude::gen1-v1': { in_per_m: 5.0, out_per_m: 25.0, model_alias: 'claude-opus-4-7' },
  'qwen-thinking::gen1-v1': {
    in_per_m: 0.25,
    out_per_m: 2.0,
    model_alias: 'qwen3.6-35b-a3b-via-dashscope-direct',
  },
} as const;

interface ProbePrompt {
  complexity: 'trivial' | 'medium-1' | 'medium-2' | 'complex' | 'stretch';
  persona: string;
  materials: string;
  question: string;
  max_tokens: number;
}

const PROMPTS: ProbePrompt[] = [
  {
    complexity: 'trivial',
    persona: 'a concise factual analyst',
    materials: 'The number two added to the number two equals four. This is elementary arithmetic.',
    question: 'What is 2+2?',
    max_tokens: 200,
  },
  {
    complexity: 'medium-1',
    persona: 'a CFO advisor at a mid-stage SaaS startup',
    materials: [
      'Q3 metrics for Acme SaaS (Series B, $12M ARR):',
      '- ARR growth slowed from 18% QoQ to 9% QoQ.',
      '- Gross margin held at 78%.',
      '- Net dollar retention dropped from 121% to 108%.',
      '- Churn ticked up in SMB segment from 1.4% to 2.1% monthly.',
      '- Sales cycle elongated 22% in mid-market.',
      '- Two competitors raised Series C at higher valuations.',
    ].join('\n'),
    question: 'List the 3 highest-priority risks the CEO should brief the board on, in order.',
    max_tokens: 600,
  },
  {
    complexity: 'medium-2',
    persona: 'a software architect advising on legacy modernization',
    materials: [
      'Context: 8-year-old Rails monolith. ~400k LOC. Critical revenue path.',
      'Team: 12 backend engineers, 4 platform engineers. No prior modernization experience.',
      'Pressure: Sales reports the monolith blocks integration deals worth ~$8M ARR.',
      'Constraints: 9-month runway-extension clock; CEO wants visible progress every 60 days.',
    ].join('\n'),
    question: 'Compare strangler-fig vs big-bang refactor for this team. Recommend one with explicit risk acknowledgment.',
    max_tokens: 800,
  },
  {
    complexity: 'complex',
    persona: 'a strategic operations consultant',
    materials: [
      'NorthLane Retail (mid-market apparel chain):',
      '- Q3 SSS missed plan by 8% (-3.2% vs +5% planned).',
      '- Macro: discretionary spend down 4% sector-wide; 30y yield up 80bps in quarter.',
      '- Competitive: Aritzia opened 4 new doors in core trade areas; Old Navy ran 30% friends-and-family promo for 6 weeks.',
      '- Internal: New POS system rollout caused 6% transaction-error rate for 3 weeks; e-commerce site speed regressed (LCP 2.4s -> 4.1s).',
      '- Brand: Latest TikTok creator partnership underperformed engagement targets by 60%; Brand consideration score flat.',
      '- Inventory: $42M aged 90+ days; merch margin compressed 180bps to fund clearance.',
    ].join('\n'),
    question: 'Diagnose root causes across 4 dimensions (macro / competitive / internal execution / brand). For each, attribute approximate share of the 8% miss + propose one targeted Q4 intervention.',
    max_tokens: 1200,
  },
  {
    complexity: 'stretch',
    persona: 'an M&A analyst preparing an investment memo for the deal committee',
    materials: [
      'Target: Cobalt Cloud (Series D enterprise SaaS, governance + audit logging).',
      '- ARR: $50M trailing; +52% YoY; 110% NDR.',
      '- Customers: 320 logos, top-10 = 28% of ARR. Net new logos +18% YoY.',
      '- Gross margin: 76% (best-in-class for category 73-78%).',
      '- Sales efficiency (Magic Number): 1.1 (LTM); CAC payback 16 months.',
      '- Burn: $4M/quarter, 18 months runway.',
      '- Last round: $750M post on $35M ARR (21x), 2.5 years ago.',
      '- Comps: Drata (~25x ARR), Vanta (~22x ARR), Hyperproof (~14x ARR), AuditBoard (public, 8x).',
      '- Recent context: 2 strategic offers verbal at 12-15x ARR. Founder open to strategic at 15x+.',
    ].join('\n'),
    question: [
      'Walk through three valuation methods for Cobalt Cloud:',
      '1) DCF — state assumptions explicit (revenue growth fade, margin trajectory, discount rate, terminal multiple).',
      '2) Trading comps — apply weighted multiple from 4 comps to next-twelve-months revenue.',
      '3) LBO — assume Sponsor offer at $750M EV, 6.5x leverage, 5y hold, exit at 12x EBITDA. State minimum revenue CAGR for 25% IRR.',
      'Reconcile any gap > 25% between the three methods + recommend offer band.',
    ].join('\n'),
    max_tokens: 1500,
  },
];

interface ProbeResult {
  variant: 'claude::gen1-v1' | 'qwen-thinking::gen1-v1';
  complexity: ProbePrompt['complexity'];
  model_alias: string;
  input_tokens: number;
  output_tokens: number;
  cost_usd: number;
  latency_ms: number;
  ts: string;
  error?: string;
}

function appendJsonl(file: string, row: ProbeResult) {
  fs.appendFileSync(file, JSON.stringify(row) + '\n');
}

async function callViaLitellm(model: string, system: string, user: string, max_tokens: number) {
  if (!LITELLM_MASTER_KEY) throw new Error('LITELLM_MASTER_KEY not set');
  const payload = {
    model,
    messages: [
      { role: 'system', content: system },
      { role: 'user', content: user },
    ],
    max_tokens,
  };
  const t0 = Date.now();
  const resp = await fetch(`${LITELLM_URL}/chat/completions`, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      Authorization: `Bearer ${LITELLM_MASTER_KEY}`,
    },
    body: JSON.stringify(payload),
  });
  const latency_ms = Date.now() - t0;
  const data = (await resp.json()) as Record<string, unknown>;
  if (!resp.ok || 'error' in data) {
    const errMsg =
      (data as { error?: { message?: string } }).error?.message ??
      `HTTP ${resp.status}: ${JSON.stringify(data).slice(0, 200)}`;
    return { ok: false as const, error: String(errMsg), latency_ms };
  }
  const choices = data.choices as Array<{ message: { content: string } }> | undefined;
  const usage = data.usage as { prompt_tokens?: number; completion_tokens?: number } | undefined;
  return {
    ok: true as const,
    content: choices?.[0]?.message?.content ?? '',
    input_tokens: usage?.prompt_tokens ?? 0,
    output_tokens: usage?.completion_tokens ?? 0,
    latency_ms,
  };
}

async function probe(
  variant: 'claude::gen1-v1' | 'qwen-thinking::gen1-v1',
  prompt: ProbePrompt,
): Promise<ProbeResult> {
  const shape = variant === 'claude::gen1-v1' ? claudeGen1V1Shape : qwenThinkingGen1V1Shape;
  const config = PRICING[variant];

  const sysPrompt = shape.systemPrompt({
    persona: prompt.persona,
    question: prompt.question,
    isMultiStep: false,
  });
  const userPrompt = shape.soloUserPrompt({
    persona: prompt.persona,
    materials: prompt.materials,
    question: prompt.question,
  });

  const result = await callViaLitellm(config.model_alias, sysPrompt, userPrompt, prompt.max_tokens);

  if (!result.ok) {
    return {
      variant,
      complexity: prompt.complexity,
      model_alias: config.model_alias,
      input_tokens: 0,
      output_tokens: 0,
      cost_usd: 0,
      latency_ms: result.latency_ms,
      ts: new Date().toISOString(),
      error: result.error,
    };
  }

  const cost_usd =
    (result.input_tokens * config.in_per_m) / 1e6 +
    (result.output_tokens * config.out_per_m) / 1e6;
  return {
    variant,
    complexity: prompt.complexity,
    model_alias: config.model_alias,
    input_tokens: result.input_tokens,
    output_tokens: result.output_tokens,
    cost_usd,
    latency_ms: result.latency_ms,
    ts: new Date().toISOString(),
  };
}

function percentile(values: number[], p: number): number {
  if (values.length === 0) return 0;
  const sorted = [...values].sort((a, b) => a - b);
  const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1));
  return sorted[idx];
}

interface VariantSummary {
  variant: string;
  count: number;
  errors: number;
  p50: number;
  p95: number;
  max: number;
  mean: number;
  total: number;
}

function summarize(results: ProbeResult[], variant: string): VariantSummary {
  const ok = results.filter((r) => r.variant === variant && !r.error);
  const errors = results.filter((r) => r.variant === variant && r.error).length;
  if (ok.length === 0) {
    return {
      variant,
      count: 0,
      errors,
      p50: NaN,
      p95: NaN,
      max: NaN,
      mean: NaN,
      total: 0,
    };
  }
  const costs = ok.map((r) => r.cost_usd);
  const total = costs.reduce((a, b) => a + b, 0);
  return {
    variant,
    count: ok.length,
    errors,
    p50: percentile(costs, 50),
    p95: percentile(costs, 95),
    max: Math.max(...costs),
    mean: total / ok.length,
    total,
  };
}

async function main() {
  fs.writeFileSync(OUT_JSONL, '');

  console.log('Phase 5 §0.3 cost probe — start');
  console.log(`LiteLLM: ${LITELLM_URL}`);
  console.log(`Output JSONL: ${OUT_JSONL}`);

  const results: ProbeResult[] = [];

  for (const variant of ['claude::gen1-v1', 'qwen-thinking::gen1-v1'] as const) {
    for (const prompt of PROMPTS) {
      console.log(`  ${variant} <- ${prompt.complexity}`);
      const r = await probe(variant, prompt);
      results.push(r);
      appendJsonl(OUT_JSONL, r);
      if (r.error) console.log(`    ERROR: ${r.error}`);
      else
        console.log(
          `    in=${r.input_tokens} out=${r.output_tokens} cost=$${r.cost_usd.toFixed(4)} latency=${r.latency_ms}ms`,
        );
    }
  }

  const claudeSummary = summarize(results, 'claude::gen1-v1');
  const qwenSummary = summarize(results, 'qwen-thinking::gen1-v1');

  const validP95s = [claudeSummary.p95, qwenSummary.p95].filter(
    (v) => Number.isFinite(v) && v > 0,
  );
  const max_p95 = validP95s.length > 0 ? Math.max(...validP95s) : NaN;
  const VOLUME = 740;
  const BUFFER = 1.2;
  const canary_cost_p95_ceiling = Number.isFinite(max_p95) ? VOLUME * max_p95 * BUFFER : NaN;
  const HALT = 20;
  const HARD_CAP = 25;

  const lines: string[] = [];
  lines.push('# Phase 5 §0.3 Cost Probe Summary');
  lines.push('');
  lines.push(`**Date:** ${new Date().toISOString()}`);
  lines.push(`**Pricing snapshot:** 2026-04-29`);
  lines.push(`**Branch:** phase-5-deployment-v2`);
  lines.push(`**Endpoint:** ${LITELLM_URL} (LiteLLM proxy, matches Faza 1 runner pattern)`);
  lines.push('');
  lines.push('## Per-variant statistics');
  lines.push('');
  lines.push('| Variant | Model alias | OK | Errors | p50 | p95 | max | mean | total |');
  lines.push('|---|---|---|---|---|---|---|---|---|');
  for (const s of [claudeSummary, qwenSummary]) {
    const config = PRICING[s.variant as keyof typeof PRICING];
    if (s.count === 0) {
      lines.push(`| ${s.variant} | ${config.model_alias} | 0 | ${s.errors} | n/a | n/a | n/a | n/a | n/a |`);
    } else {
      lines.push(
        `| ${s.variant} | ${config.model_alias} | ${s.count} | ${s.errors} | $${s.p50.toFixed(4)} | $${s.p95.toFixed(4)} | $${s.max.toFixed(4)} | $${s.mean.toFixed(4)} | $${s.total.toFixed(4)} |`,
      );
    }
  }
  lines.push('');
  lines.push('## Canary cost ceiling (per brief §5.4)');
  lines.push('');
  lines.push(
    `Formula: \`canary_cost_p95_ceiling = ${VOLUME} requests × max(p95) × ${BUFFER} (buffer)\``,
  );
  lines.push('');
  if (Number.isFinite(max_p95)) {
    lines.push(`max(p95) = $${max_p95.toFixed(4)}`);
    lines.push(
      `canary_cost_p95_ceiling = ${VOLUME} × $${max_p95.toFixed(4)} × ${BUFFER} = **$${canary_cost_p95_ceiling.toFixed(2)}**`,
    );
  } else {
    lines.push(`max(p95) = NaN (no successful samples)`);
    lines.push(`canary_cost_p95_ceiling = INDETERMINATE`);
  }
  lines.push('');
  lines.push('## Verdict');
  lines.push('');
  if (Number.isFinite(canary_cost_p95_ceiling)) {
    if (canary_cost_p95_ceiling <= HALT) {
      lines.push(
        `**PASS** — ceiling $${canary_cost_p95_ceiling.toFixed(2)} <= halt trigger $${HALT}.`,
      );
    } else if (canary_cost_p95_ceiling <= HARD_CAP) {
      lines.push(
        `**HALT-AND-PM** — ceiling $${canary_cost_p95_ceiling.toFixed(2)} > halt trigger $${HALT}, but <= hard cap $${HARD_CAP}.`,
      );
    } else {
      lines.push(
        `**HARD-CAP-EXCEED** — ceiling $${canary_cost_p95_ceiling.toFixed(2)} > hard cap $${HARD_CAP}.`,
      );
    }
  } else {
    lines.push(`**INDETERMINATE** — no valid p95.`);
  }
  lines.push('');
  lines.push('## Probe spend (this script)');
  const totalSpend = (claudeSummary.total ?? 0) + (qwenSummary.total ?? 0);
  lines.push('');
  lines.push(`Total: **$${totalSpend.toFixed(4)}**`);
  lines.push(`Brief §5.4 probe budget: $0.30-$0.50.`);
  lines.push('');
  lines.push('## JSONL anchor');
  lines.push('');
  lines.push('Per-request rows: `gepa-phase-5/cost-probe-2026-04-29.jsonl`');

  const summaryText = lines.join('\n');
  fs.writeFileSync(OUT_SUMMARY, summaryText);

  console.log('');
  console.log('==========================================');
  console.log(summaryText);
  console.log('==========================================');
  console.log(`Summary: ${OUT_SUMMARY}`);
}

main().catch((err) => {
  console.error('FATAL:', err);
  process.exit(1);
});