This commit is contained in:
74
packages/agent/src/cookbook/catalog.ts
Normal file
74
packages/agent/src/cookbook/catalog.ts
Normal file
@@ -0,0 +1,74 @@
|
||||
/**
|
||||
* Curated Ollama-servable model catalog. Every `name` is a real `ollama pull` ref.
|
||||
* Clean-room replacement for Odysseus's 917-row HF `hf_models.json` — scoped to the
|
||||
* dense + MoE models a 2026 laptop/desktop user would actually run locally.
|
||||
* Maintained by hand (small on purpose); no runtime HF fetch.
|
||||
* (AGPL-3.0: data curated independently, no code/list copied.)
|
||||
*/
|
||||
|
||||
export interface CatalogModel {
|
||||
/** ollama pull ref, e.g. "llama3.1:8b" */
|
||||
readonly name: string;
|
||||
readonly provider: string;
|
||||
readonly parameterCount: string; // human label, e.g. "8B"
|
||||
readonly paramsB: number; // total params (billions) — VRAM footprint
|
||||
readonly activeParamsB?: number; // MoE active params/token — KV + speed
|
||||
readonly isMoe: boolean;
|
||||
readonly quant: string; // native/default GGUF quant tag
|
||||
readonly contextLength: number;
|
||||
readonly family: string; // 'llama' | 'qwen' | 'mistral' | ...
|
||||
readonly useCase: string; // 'general' | 'coding' | 'reasoning' | 'multimodal'
|
||||
readonly releaseDate: string; // ISO date for the recency tiebreak
|
||||
readonly gguf?: boolean; // Ollama models are GGUF; defaults true (serve-path gate)
|
||||
}
|
||||
|
||||
export const OLLAMA_CATALOG: ReadonlyArray<CatalogModel> = [
|
||||
// ── Llama ─────────────────────────────────────────────
|
||||
{ name: 'llama3.2:1b', provider: 'Meta', parameterCount: '1B', paramsB: 1.2, isMoe: false, quant: 'Q4_K_M', contextLength: 131072, family: 'llama', useCase: 'general', releaseDate: '2024-09-25' },
|
||||
{ name: 'llama3.2:3b', provider: 'Meta', parameterCount: '3B', paramsB: 3.2, isMoe: false, quant: 'Q4_K_M', contextLength: 131072, family: 'llama', useCase: 'general', releaseDate: '2024-09-25' },
|
||||
{ name: 'llama3.1:8b', provider: 'Meta', parameterCount: '8B', paramsB: 8, isMoe: false, quant: 'Q4_K_M', contextLength: 131072, family: 'llama', useCase: 'general', releaseDate: '2024-07-23' },
|
||||
{ name: 'llama3.1:70b', provider: 'Meta', parameterCount: '70B', paramsB: 70, isMoe: false, quant: 'Q4_K_M', contextLength: 131072, family: 'llama', useCase: 'general', releaseDate: '2024-07-23' },
|
||||
{ name: 'llama3.3:70b', provider: 'Meta', parameterCount: '70B', paramsB: 70, isMoe: false, quant: 'Q4_K_M', contextLength: 131072, family: 'llama', useCase: 'general', releaseDate: '2024-12-06' },
|
||||
// ── Qwen 2.5 ──────────────────────────────────────────
|
||||
{ name: 'qwen2.5:0.5b', provider: 'Alibaba', parameterCount: '0.5B', paramsB: 0.5, isMoe: false, quant: 'Q4_K_M', contextLength: 32768, family: 'qwen', useCase: 'general', releaseDate: '2024-09-19' },
|
||||
{ name: 'qwen2.5:3b', provider: 'Alibaba', parameterCount: '3B', paramsB: 3, isMoe: false, quant: 'Q4_K_M', contextLength: 32768, family: 'qwen', useCase: 'general', releaseDate: '2024-09-19' },
|
||||
{ name: 'qwen2.5:7b', provider: 'Alibaba', parameterCount: '7B', paramsB: 7, isMoe: false, quant: 'Q4_K_M', contextLength: 32768, family: 'qwen', useCase: 'general', releaseDate: '2024-09-19' },
|
||||
{ name: 'qwen2.5:14b', provider: 'Alibaba', parameterCount: '14B', paramsB: 14, isMoe: false, quant: 'Q4_K_M', contextLength: 32768, family: 'qwen', useCase: 'general', releaseDate: '2024-09-19' },
|
||||
{ name: 'qwen2.5:32b', provider: 'Alibaba', parameterCount: '32B', paramsB: 32, isMoe: false, quant: 'Q4_K_M', contextLength: 32768, family: 'qwen', useCase: 'general', releaseDate: '2024-09-19' },
|
||||
{ name: 'qwen2.5:72b', provider: 'Alibaba', parameterCount: '72B', paramsB: 72, isMoe: false, quant: 'Q4_K_M', contextLength: 32768, family: 'qwen', useCase: 'general', releaseDate: '2024-09-19' },
|
||||
// ── Qwen 2.5 Coder ────────────────────────────────────
|
||||
{ name: 'qwen2.5-coder:1.5b', provider: 'Alibaba', parameterCount: '1.5B', paramsB: 1.5, isMoe: false, quant: 'Q4_K_M', contextLength: 32768, family: 'qwen', useCase: 'coding', releaseDate: '2024-11-12' },
|
||||
{ name: 'qwen2.5-coder:7b', provider: 'Alibaba', parameterCount: '7B', paramsB: 7, isMoe: false, quant: 'Q4_K_M', contextLength: 32768, family: 'qwen', useCase: 'coding', releaseDate: '2024-11-12' },
|
||||
{ name: 'qwen2.5-coder:14b', provider: 'Alibaba', parameterCount: '14B', paramsB: 14, isMoe: false, quant: 'Q4_K_M', contextLength: 32768, family: 'qwen', useCase: 'coding', releaseDate: '2024-11-12' },
|
||||
{ name: 'qwen2.5-coder:32b', provider: 'Alibaba', parameterCount: '32B', paramsB: 32, isMoe: false, quant: 'Q4_K_M', contextLength: 32768, family: 'qwen', useCase: 'coding', releaseDate: '2024-11-12' },
|
||||
// ── Qwen 3 (incl. MoE) ────────────────────────────────
|
||||
{ name: 'qwen3:1.7b', provider: 'Alibaba', parameterCount: '1.7B', paramsB: 1.7, isMoe: false, quant: 'Q4_K_M', contextLength: 40960, family: 'qwen', useCase: 'general', releaseDate: '2025-04-28' },
|
||||
{ name: 'qwen3:4b', provider: 'Alibaba', parameterCount: '4B', paramsB: 4, isMoe: false, quant: 'Q4_K_M', contextLength: 40960, family: 'qwen', useCase: 'general', releaseDate: '2025-04-28' },
|
||||
{ name: 'qwen3:8b', provider: 'Alibaba', parameterCount: '8B', paramsB: 8, isMoe: false, quant: 'Q4_K_M', contextLength: 40960, family: 'qwen', useCase: 'general', releaseDate: '2025-04-28' },
|
||||
{ name: 'qwen3:14b', provider: 'Alibaba', parameterCount: '14B', paramsB: 14, isMoe: false, quant: 'Q4_K_M', contextLength: 40960, family: 'qwen', useCase: 'general', releaseDate: '2025-04-28' },
|
||||
{ name: 'qwen3:32b', provider: 'Alibaba', parameterCount: '32B', paramsB: 32, isMoe: false, quant: 'Q4_K_M', contextLength: 40960, family: 'qwen', useCase: 'reasoning', releaseDate: '2025-04-28' },
|
||||
{ name: 'qwen3:30b-a3b', provider: 'Alibaba', parameterCount: '30B', paramsB: 30.5, activeParamsB: 3.3, isMoe: true, quant: 'Q4_K_M', contextLength: 40960, family: 'qwen', useCase: 'general', releaseDate: '2025-04-28' },
|
||||
{ name: 'qwen3:235b-a22b', provider: 'Alibaba', parameterCount: '235B', paramsB: 235, activeParamsB: 22, isMoe: true, quant: 'Q4_K_M', contextLength: 40960, family: 'qwen', useCase: 'reasoning', releaseDate: '2025-04-28' },
|
||||
// ── Mistral / Mixtral ─────────────────────────────────
|
||||
{ name: 'mistral:7b', provider: 'Mistral', parameterCount: '7B', paramsB: 7, isMoe: false, quant: 'Q4_K_M', contextLength: 32768, family: 'mistral', useCase: 'general', releaseDate: '2023-09-27' },
|
||||
{ name: 'mistral-nemo:12b', provider: 'Mistral', parameterCount: '12B', paramsB: 12, isMoe: false, quant: 'Q4_K_M', contextLength: 131072, family: 'mistral', useCase: 'general', releaseDate: '2024-07-18' },
|
||||
{ name: 'mixtral:8x7b', provider: 'Mistral', parameterCount: '47B', paramsB: 46.7, activeParamsB: 12.9, isMoe: true, quant: 'Q4_K_M', contextLength: 32768, family: 'mistral', useCase: 'general', releaseDate: '2023-12-11' },
|
||||
// ── Gemma 2 / 3 ───────────────────────────────────────
|
||||
{ name: 'gemma2:2b', provider: 'Google', parameterCount: '2B', paramsB: 2.6, isMoe: false, quant: 'Q4_K_M', contextLength: 8192, family: 'gemma', useCase: 'general', releaseDate: '2024-07-31' },
|
||||
{ name: 'gemma2:9b', provider: 'Google', parameterCount: '9B', paramsB: 9, isMoe: false, quant: 'Q4_K_M', contextLength: 8192, family: 'gemma', useCase: 'general', releaseDate: '2024-06-27' },
|
||||
{ name: 'gemma2:27b', provider: 'Google', parameterCount: '27B', paramsB: 27, isMoe: false, quant: 'Q4_K_M', contextLength: 8192, family: 'gemma', useCase: 'general', releaseDate: '2024-06-27' },
|
||||
{ name: 'gemma3:4b', provider: 'Google', parameterCount: '4B', paramsB: 4.3, isMoe: false, quant: 'Q4_K_M', contextLength: 131072, family: 'gemma', useCase: 'multimodal', releaseDate: '2025-03-12' },
|
||||
{ name: 'gemma3:12b', provider: 'Google', parameterCount: '12B', paramsB: 12, isMoe: false, quant: 'Q4_K_M', contextLength: 131072, family: 'gemma', useCase: 'multimodal', releaseDate: '2025-03-12' },
|
||||
{ name: 'gemma3:27b', provider: 'Google', parameterCount: '27B', paramsB: 27, isMoe: false, quant: 'Q4_K_M', contextLength: 131072, family: 'gemma', useCase: 'multimodal', releaseDate: '2025-03-12' },
|
||||
// ── Phi ───────────────────────────────────────────────
|
||||
{ name: 'phi3:3.8b', provider: 'Microsoft', parameterCount: '3.8B', paramsB: 3.8, isMoe: false, quant: 'Q4_K_M', contextLength: 131072, family: 'phi', useCase: 'general', releaseDate: '2024-04-23' },
|
||||
{ name: 'phi4:14b', provider: 'Microsoft', parameterCount: '14B', paramsB: 14, isMoe: false, quant: 'Q4_K_M', contextLength: 16384, family: 'phi', useCase: 'reasoning', releaseDate: '2024-12-12' },
|
||||
// ── DeepSeek ──────────────────────────────────────────
|
||||
{ name: 'deepseek-coder-v2:16b', provider: 'DeepSeek', parameterCount: '16B', paramsB: 15.7, activeParamsB: 2.4, isMoe: true, quant: 'Q4_K_M', contextLength: 163840, family: 'deepseek', useCase: 'coding', releaseDate: '2024-06-17' },
|
||||
{ name: 'deepseek-r1:7b', provider: 'DeepSeek', parameterCount: '7B', paramsB: 7, isMoe: false, quant: 'Q4_K_M', contextLength: 131072, family: 'deepseek', useCase: 'reasoning', releaseDate: '2025-01-20' },
|
||||
{ name: 'deepseek-r1:14b', provider: 'DeepSeek', parameterCount: '14B', paramsB: 14, isMoe: false, quant: 'Q4_K_M', contextLength: 131072, family: 'deepseek', useCase: 'reasoning', releaseDate: '2025-01-20' },
|
||||
{ name: 'deepseek-r1:32b', provider: 'DeepSeek', parameterCount: '32B', paramsB: 32, isMoe: false, quant: 'Q4_K_M', contextLength: 131072, family: 'deepseek', useCase: 'reasoning', releaseDate: '2025-01-20' },
|
||||
// ── Vision / small ────────────────────────────────────
|
||||
{ name: 'llama3.2-vision:11b', provider: 'Meta', parameterCount: '11B', paramsB: 11, isMoe: false, quant: 'Q4_K_M', contextLength: 131072, family: 'llama', useCase: 'multimodal', releaseDate: '2024-11-06' },
|
||||
{ name: 'smollm2:1.7b', provider: 'HuggingFace', parameterCount: '1.7B', paramsB: 1.7, isMoe: false, quant: 'Q4_K_M', contextLength: 8192, family: 'smollm', useCase: 'general', releaseDate: '2024-11-01' },
|
||||
];
|
||||
55
packages/agent/src/cookbook/gpu-bandwidth.ts
Normal file
55
packages/agent/src/cookbook/gpu-bandwidth.ts
Normal file
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* Memory-bandwidth lookup (GB/s) for the tok/s model. Clean-room port of the
|
||||
* *concept* in Odysseus `fit.py` GPU_BANDWIDTH / APPLE_BANDWIDTH_FIXED — scope-cut
|
||||
* to consumer NVIDIA (RTX 20/30/40/50), a handful of consumer AMD Radeon, and
|
||||
* Apple Silicon. Datacenter (H100/A100/MI300…) and Apple core-count binning are cut:
|
||||
* Waggle's HardwareInfo carries no gpu_cores, so Apple resolves to the conservative
|
||||
* tier (Odysseus's own fallback when cores are unknown). Substring match, longest
|
||||
* key first, so "4070 ti super" wins over "4070".
|
||||
* (AGPL-3.0: tables/control-flow re-authored from the math, no code copied.)
|
||||
*/
|
||||
|
||||
const CONSUMER_GPU_BANDWIDTH: Readonly<Record<string, number>> = {
|
||||
// NVIDIA RTX 50
|
||||
'5090': 1792, '5080': 960, '5070 ti': 896, '5070': 672, '5060 ti': 448, '5060': 256,
|
||||
// NVIDIA RTX 40
|
||||
'4090': 1008, '4080 super': 736, '4080': 717, '4070 ti super': 672, '4070 ti': 504,
|
||||
'4070 super': 504, '4070': 504, '4060 ti': 288, '4060': 272,
|
||||
// NVIDIA RTX 30
|
||||
'3090 ti': 1008, '3090': 936, '3080 ti': 912, '3080': 760, '3070 ti': 608,
|
||||
'3070': 448, '3060 ti': 448, '3060': 360, '3050': 224,
|
||||
// NVIDIA RTX 20 / GTX 16 (older laptops)
|
||||
'2080 ti': 616, '2080': 448, '2070': 448, '2060': 336, '1660 ti': 288, '1650': 128,
|
||||
// AMD Radeon (consumer RDNA)
|
||||
'7900 xtx': 960, '7900 xt': 800, '7800 xt': 624, '7700 xt': 432, '7600': 288,
|
||||
'9070 xt': 624, '9070': 488, '6800 xt': 512, '6700 xt': 384, '6600': 224,
|
||||
};
|
||||
|
||||
// Apple Silicon unified-memory bandwidth (GB/s). Conservative tier per family
|
||||
// (no gpu_cores in HardwareInfo → cannot bin M*Max variants; pick the floor).
|
||||
const APPLE_BANDWIDTH: Readonly<Record<string, number>> = {
|
||||
'm1 ultra': 800, 'm1 max': 400, 'm1 pro': 200, 'm1': 68,
|
||||
'm2 ultra': 800, 'm2 max': 400, 'm2 pro': 200, 'm2': 100,
|
||||
'm3 ultra': 800, 'm3 max': 300, 'm3 pro': 150, 'm3': 100,
|
||||
'm4 max': 410, 'm4 pro': 273, 'm4': 120,
|
||||
};
|
||||
|
||||
const CONSUMER_KEYS = Object.keys(CONSUMER_GPU_BANDWIDTH).sort((a, b) => b.length - a.length);
|
||||
const APPLE_KEYS = Object.keys(APPLE_BANDWIDTH).sort((a, b) => b.length - a.length);
|
||||
|
||||
/** Conservative fallback bandwidth by backend class when the GPU isn't in the table. */
|
||||
export const FALLBACK_K: Readonly<Record<string, number>> = {
|
||||
cuda: 220, rocm: 180, metal: 150, cpu_x86: 70, cpu_arm: 90,
|
||||
};
|
||||
|
||||
/** Resolve VRAM/unified-memory bandwidth from a GPU name. null = not found. */
|
||||
export function lookupBandwidth(gpuName: string | null | undefined): number | null {
|
||||
if (!gpuName) return null;
|
||||
const gn = gpuName.toLowerCase();
|
||||
// Apple first (its names carry "apple", never collide with NVIDIA/AMD keys).
|
||||
if (gn.includes('apple')) {
|
||||
for (const key of APPLE_KEYS) if (gn.includes(key)) return APPLE_BANDWIDTH[key];
|
||||
}
|
||||
for (const key of CONSUMER_KEYS) if (gn.includes(key)) return CONSUMER_GPU_BANDWIDTH[key];
|
||||
return null;
|
||||
}
|
||||
11
packages/agent/src/cookbook/index.ts
Normal file
11
packages/agent/src/cookbook/index.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
export {
|
||||
rankModels, estimateMemoryGb, estimateTps, qualityScore, speedScore, fitScore,
|
||||
contextScore, archAgeBonus, versionKey, inferUseCase, isServable, activeParamsB,
|
||||
canonicalCpuBackend,
|
||||
type Hardware, type ModelRecommendation, type RankOptions, type RunMode, type FitLevel,
|
||||
} from './model-fit.js';
|
||||
export { OLLAMA_CATALOG, type CatalogModel } from './catalog.js';
|
||||
export { lookupBandwidth, FALLBACK_K } from './gpu-bandwidth.js';
|
||||
export {
|
||||
QUANT_HIERARCHY, QUANT_BYTES_PER_PARAM, QUANT_SPEED_MULT, QUANT_QUALITY_PENALTY,
|
||||
} from './quant-tables.js';
|
||||
401
packages/agent/src/cookbook/model-fit.ts
Normal file
401
packages/agent/src/cookbook/model-fit.ts
Normal file
@@ -0,0 +1,401 @@
|
||||
/**
|
||||
* Cookbook local-model fit engine — clean-room TypeScript port of the *algorithm*
|
||||
* behind Odysseus hwfit `fit.py` + `models.py` (memory-bandwidth tok/s, harmonic
|
||||
* CPU-offload blend, MoE active-param math, weighted quality/speed/fit/context
|
||||
* composite, arch-age + version tiebreak). No Odysseus code copied; no binary bundled.
|
||||
*
|
||||
* Pure & deterministic — all hardware/catalog data is injected. Unit-tested.
|
||||
*/
|
||||
import {
|
||||
QUANT_HIERARCHY, QUANT_BYTES_PER_PARAM, QUANT_SPEED_MULT, QUANT_QUALITY_PENALTY,
|
||||
DEFAULT_BPP, DEFAULT_SPEED_MULT,
|
||||
} from './quant-tables.js';
|
||||
import { lookupBandwidth, FALLBACK_K } from './gpu-bandwidth.js';
|
||||
import { type CatalogModel } from './catalog.js';
|
||||
|
||||
// ── Calibrated constants (from Odysseus fit.py, ported as named constants) ──
|
||||
const GPU_EFFICIENCY = 0.55; // realized fraction of peak bandwidth
|
||||
const CPU_OFFLOAD_BW = 55.0; // dual-channel DDR4/5 effective GB/s
|
||||
const MOE_SPEED_PENALTY = 0.8; // mixed-dtype/expert dispatch overhead
|
||||
const RUNTIME_BUFFER_GB = 0.5; // KV/compute base buffer
|
||||
const KV_PER_B_PER_TOKEN = 0.000008; // GB per active-billion-param per ctx token
|
||||
const MIN_CTX = 1024; // context-shrink floor
|
||||
|
||||
export type RunMode = 'gpu' | 'cpu_offload' | 'cpu_only' | 'no_fit';
|
||||
export type FitLevel = 'perfect' | 'good' | 'marginal' | 'too_tight';
|
||||
|
||||
/** Hardware input — a structural SUBSET of the route's HardwareInfo (assignable). */
|
||||
export interface Hardware {
|
||||
readonly totalRamGb: number;
|
||||
readonly availableRamGb: number;
|
||||
readonly hasGpu: boolean;
|
||||
readonly gpuName: string | null;
|
||||
readonly gpuVramGb: number | null;
|
||||
readonly gpuCount: number;
|
||||
readonly backend: string;
|
||||
readonly platform: string; // `${os.platform()} ${os.arch()}`
|
||||
}
|
||||
|
||||
/** Output — MUST match local-inference.ts ModelRecommendation exactly. */
|
||||
export interface ModelRecommendation {
|
||||
name: string;
|
||||
provider: string;
|
||||
parameterCount: string;
|
||||
paramsB: number;
|
||||
useCase: string;
|
||||
category: string;
|
||||
fitLevel: FitLevel;
|
||||
score: number;
|
||||
scoreComponents: { quality: number; speed: number; fit: number; context: number };
|
||||
estimatedTps: number;
|
||||
memoryRequiredGb: number;
|
||||
memoryAvailableGb: number;
|
||||
utilizationPct: number;
|
||||
bestQuant: string;
|
||||
runMode: RunMode;
|
||||
runtime: string;
|
||||
contextLength: number;
|
||||
isMoe: boolean;
|
||||
notes: string[];
|
||||
}
|
||||
|
||||
export interface RankOptions {
|
||||
useCase?: string; // scoring use-case (general/coding/reasoning/...)
|
||||
limit?: number; // top-N (default 20)
|
||||
quant?: string; // force a single quant (skip the best-fit ladder)
|
||||
fitOnly?: boolean; // drop too_tight rows
|
||||
search?: string; // name/provider substring filter
|
||||
}
|
||||
|
||||
// USE_CASE_WEIGHTS: (quality, speed, fit, context). Ported from fit.py.
|
||||
const USE_CASE_WEIGHTS: Readonly<Record<string, readonly [number, number, number, number]>> = {
|
||||
general: [0.45, 0.30, 0.15, 0.10],
|
||||
coding: [0.50, 0.20, 0.15, 0.15],
|
||||
reasoning: [0.55, 0.15, 0.15, 0.15],
|
||||
chat: [0.40, 0.35, 0.15, 0.10],
|
||||
multimodal: [0.50, 0.20, 0.15, 0.15],
|
||||
};
|
||||
const DEFAULT_WEIGHTS = USE_CASE_WEIGHTS.general;
|
||||
|
||||
const SPEED_TARGET: Readonly<Record<string, number>> = {
|
||||
general: 40, coding: 40, multimodal: 40, chat: 40, reasoning: 25,
|
||||
};
|
||||
const CONTEXT_TARGET: Readonly<Record<string, number>> = {
|
||||
general: 4096, chat: 4096, coding: 8192, reasoning: 8192, multimodal: 4096,
|
||||
};
|
||||
|
||||
const KNOWN_USE_CASES: ReadonlySet<string> = new Set(Object.keys(USE_CASE_WEIGHTS));
|
||||
|
||||
/** Sanitize a (possibly query-supplied) use-case to a known key before it indexes the
|
||||
* scoring Records — guards against inherited keys like '__proto__' (which are non-null,
|
||||
* so a `?? default` would not fire, and array-destructuring them throws). Unknown → 'general'. */
|
||||
export function normalizeUseCase(uc: string | undefined): string {
|
||||
return uc && KNOWN_USE_CASES.has(uc) ? uc : 'general';
|
||||
}
|
||||
|
||||
// ── Pure helpers ───────────────────────────────────────────────────────────
|
||||
|
||||
export function activeParamsB(model: CatalogModel): number {
|
||||
return model.isMoe && model.activeParamsB && model.activeParamsB > 0
|
||||
? model.activeParamsB
|
||||
: model.paramsB;
|
||||
}
|
||||
|
||||
/** VRAM/RAM (GB) to serve `model` at `quant` and `ctx`. All weights resident even
|
||||
* for MoE; KV cache scales with ACTIVE params. Port of estimate_memory_gb. */
|
||||
export function estimateMemoryGb(model: CatalogModel, quant: string, ctx: number): number {
|
||||
const bpp = QUANT_BYTES_PER_PARAM[quant] ?? DEFAULT_BPP;
|
||||
const kvParams = activeParamsB(model);
|
||||
return model.paramsB * bpp + KV_PER_B_PER_TOKEN * kvParams * ctx + RUNTIME_BUFFER_GB;
|
||||
}
|
||||
|
||||
/** Normalize backend → cpu_x86 | cpu_arm for the fallback speed path. */
|
||||
export function canonicalCpuBackend(hw: Hardware): 'cpu_x86' | 'cpu_arm' {
|
||||
const platform = hw.platform.toLowerCase();
|
||||
const backend = hw.backend.toLowerCase();
|
||||
if (platform.includes('arm64') || platform.includes('aarch64')) return 'cpu_arm';
|
||||
if (backend.includes('apple') || backend.includes('metal')) return 'cpu_arm';
|
||||
return 'cpu_x86';
|
||||
}
|
||||
|
||||
/** tok/s estimate. Memory-bandwidth model on GPU/offload; per-param fallback on CPU.
|
||||
* Port of _estimate_speed (harmonic CPU-offload blend, MoE ×0.8). */
|
||||
export function estimateTps(
|
||||
model: CatalogModel, quant: string, runMode: RunMode, hw: Hardware, offloadFrac = 0,
|
||||
): number {
|
||||
const activePb = activeParamsB(model);
|
||||
if (activePb <= 0) return 0;
|
||||
const bw = lookupBandwidth(hw.gpuName);
|
||||
|
||||
if (bw && (runMode === 'gpu' || runMode === 'cpu_offload')) {
|
||||
const bpp = QUANT_BYTES_PER_PARAM[quant] ?? DEFAULT_BPP;
|
||||
const modelGb = activePb * bpp; // bytes READ per token (active experts only)
|
||||
if (modelGb <= 0) return 0;
|
||||
if (runMode === 'cpu_offload') {
|
||||
let frac = Math.min(Math.max(offloadFrac, 0), 1);
|
||||
if (frac <= 0) frac = 0.5; // unknown spill → assume meaningful
|
||||
const effBw = 1 / (frac / CPU_OFFLOAD_BW + (1 - frac) / bw); // harmonic blend
|
||||
const raw = (effBw / modelGb) * GPU_EFFICIENCY;
|
||||
return model.isMoe ? raw * MOE_SPEED_PENALTY : raw;
|
||||
}
|
||||
const raw = (bw / modelGb) * GPU_EFFICIENCY;
|
||||
return model.isMoe ? raw * MOE_SPEED_PENALTY : raw;
|
||||
}
|
||||
|
||||
// CPU-only (or GPU not in the bandwidth table): per-active-param fallback.
|
||||
const backend = canonicalCpuBackend(hw);
|
||||
const k = FALLBACK_K[backend] ?? 70;
|
||||
const sm = QUANT_SPEED_MULT[quant] ?? DEFAULT_SPEED_MULT;
|
||||
return (k / activePb) * sm;
|
||||
}
|
||||
|
||||
/** Base quality by size + family/arch/quant/use-case adjustments. Port of _quality_score. */
|
||||
export function qualityScore(model: CatalogModel, quant: string, useCase: string): number {
|
||||
const pb = model.paramsB;
|
||||
let base: number;
|
||||
if (pb < 1) base = 30;
|
||||
else if (pb < 3) base = 45;
|
||||
else if (pb < 7) base = 60;
|
||||
else if (pb < 10) base = 75;
|
||||
else if (pb < 20) base = 82;
|
||||
else if (pb < 40) base = 89;
|
||||
else base = 95;
|
||||
|
||||
const n = model.name.toLowerCase();
|
||||
if (n.includes('qwen')) base += 2;
|
||||
if (n.includes('deepseek')) base += 3;
|
||||
if (n.includes('llama')) base += 2;
|
||||
if (n.includes('mistral') || n.includes('mixtral')) base += 1;
|
||||
if (n.includes('gemma')) base += 1;
|
||||
|
||||
base += archAgeBonus(model.name);
|
||||
base += QUANT_QUALITY_PENALTY[quant] ?? 0;
|
||||
|
||||
const modelUc = inferUseCase(model);
|
||||
if (modelUc === 'coding' && useCase === 'coding') base += 6;
|
||||
else if (modelUc === 'coding' && (useCase === 'general' || useCase === 'chat')) base -= 10;
|
||||
if (modelUc === 'reasoning' && useCase === 'reasoning' && pb >= 13) base += 5;
|
||||
else if (modelUc === 'reasoning' && useCase === 'chat') base -= 4;
|
||||
if (modelUc === 'multimodal' && useCase === 'multimodal') base += 6;
|
||||
|
||||
return Math.max(0, Math.min(100, base));
|
||||
}
|
||||
|
||||
export function speedScore(tps: number, useCase: string): number {
|
||||
const target = SPEED_TARGET[useCase] ?? 40;
|
||||
return Math.max(0, Math.min(100, (tps / target) * 100));
|
||||
}
|
||||
|
||||
/** Fit score — peaks at 0.5–0.8 VRAM utilization. Port of _fit_score. */
|
||||
export function fitScore(required: number, available: number): number {
|
||||
if (required > available) return 0;
|
||||
if (available <= 0) return 0;
|
||||
const ratio = required / available;
|
||||
if (ratio <= 0.5) return 60 + (ratio / 0.5) * 40;
|
||||
if (ratio <= 0.8) return 100;
|
||||
if (ratio <= 0.9) return 70;
|
||||
return 50;
|
||||
}
|
||||
|
||||
export function contextScore(ctx: number, useCase: string): number {
|
||||
const target = CONTEXT_TARGET[useCase] ?? 4096;
|
||||
if (ctx >= target) return 100;
|
||||
if (ctx >= target / 2) return 70;
|
||||
return 30;
|
||||
}
|
||||
|
||||
/** Small architecture-recency bonus (Qwen ladder). Port of _architecture_bonus. */
|
||||
export function archAgeBonus(name: string): number {
|
||||
const t = name.toLowerCase();
|
||||
if (t.includes('qwen3.6') || t.includes('qwen3_6')) return 9;
|
||||
if (t.includes('qwen3.5') || t.includes('qwen3_5')) return 8;
|
||||
if (t.includes('qwen3-next') || t.includes('qwen3_next')) return 6;
|
||||
if (t.includes('qwen3')) return 4;
|
||||
if (t.includes('qwen2.5') || t.includes('qwen2_5')) return 2;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/** Parse a version float from a display name for the score tiebreak. Port of _version_key.
|
||||
* 'MiniMax-M2.7'→2.7, 'Qwen3.6-35B'→3.6, 'Qwen3-235B'→3 (235 skipped), 'M2'→2. */
|
||||
export function versionKey(name: string): number {
|
||||
if (!name) return 0;
|
||||
const re = /[A-Za-z](\d+(?:\.\d+)?)(?![A-Za-z])/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(name)) !== null) {
|
||||
const raw = m[1];
|
||||
const f = Number.parseFloat(raw);
|
||||
if (Number.isNaN(f)) continue;
|
||||
if (!raw.includes('.') && f >= 100) continue; // bare ≥100 = param count, not version
|
||||
return f;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function inferUseCase(model: CatalogModel): string {
|
||||
if (model.useCase) return model.useCase;
|
||||
const c = `${model.name} ${model.family}`.toLowerCase();
|
||||
if (c.includes('embed') || c.includes('bge')) return 'embedding';
|
||||
if (c.includes('code')) return 'coding';
|
||||
if (c.includes('vision') || c.includes('-vl') || c.includes('multimodal')) return 'multimodal';
|
||||
if (c.includes('r1') || c.includes('reason')) return 'reasoning';
|
||||
return 'general';
|
||||
}
|
||||
|
||||
// ── Serve-path gating (scope-cut) ───────────────────────────────────────────
|
||||
/** Apple Silicon / Windows / consumer-AMD can only serve GGUF (Ollama/llama.cpp).
|
||||
* Every curated row IS GGUF, so this never drops a catalog row today — it guards
|
||||
* against a future non-GGUF entry. Port of the serve-path-truth concept. */
|
||||
export function isServable(model: CatalogModel, hw: Hardware): boolean {
|
||||
const isGguf = model.gguf !== false;
|
||||
if (isGguf) return true;
|
||||
const platform = hw.platform.toLowerCase();
|
||||
const backend = hw.backend.toLowerCase();
|
||||
const gpu = (hw.gpuName ?? '').toLowerCase();
|
||||
const appleSilicon = platform.includes('darwin') || backend.includes('metal') || backend.includes('apple');
|
||||
const isWindows = platform.includes('win32') || platform.includes('windows');
|
||||
const consumerAmd = /radeon|rx\s?\d{4}|\b9070\b|\b7900\b/.test(gpu) && !/instinct|mi\d{3}/.test(gpu);
|
||||
// Non-GGUF model: only CUDA/Linux can serve it (vLLM); gate out Apple/Win/RDNA.
|
||||
return !(appleSilicon || isWindows || consumerAmd);
|
||||
}
|
||||
|
||||
// ── Fit resolution ──────────────────────────────────────────────────────────
|
||||
interface FitResult {
|
||||
runMode: Exclude<RunMode, 'no_fit'>;
|
||||
quant: string;
|
||||
ctx: number;
|
||||
requiredGb: number;
|
||||
}
|
||||
|
||||
/** Pick best-fitting quant + run mode. GPU-resident (best quant first, then shrink
|
||||
* ctx) → offload → cpu_only. Returns null = doesn't fit anywhere (too_tight).
|
||||
* Adapts _try_quant_at + best_quant_for_budget. */
|
||||
function resolveFit(model: CatalogModel, hw: Hardware, opts: RankOptions): FitResult | null {
|
||||
const vram = hw.hasGpu && hw.gpuVramGb && hw.gpuVramGb > 0 ? hw.gpuVramGb : 0;
|
||||
const ram = hw.availableRamGb > 0 ? hw.availableRamGb : 0;
|
||||
const ladder = opts.quant ? [opts.quant] : [...QUANT_HIERARCHY];
|
||||
const fullCtx = model.contextLength > 0 ? model.contextLength : 4096;
|
||||
|
||||
// GPU-resident: prefer full ctx with the best quant that fits VRAM; shrink ctx if needed.
|
||||
if (vram > 0) {
|
||||
for (let ctx = fullCtx; ctx >= MIN_CTX; ) {
|
||||
for (const q of ladder) {
|
||||
const mem = estimateMemoryGb(model, q, ctx);
|
||||
if (mem <= vram) return { runMode: 'gpu', quant: q, ctx, requiredGb: mem };
|
||||
}
|
||||
if (ctx === MIN_CTX) break;
|
||||
ctx = Math.max(MIN_CTX, Math.floor(ctx / 2)); // clamp so the MIN_CTX floor is always tested
|
||||
}
|
||||
// Offload: doesn't fit VRAM but fits system RAM (spills experts/layers).
|
||||
for (const q of ladder) {
|
||||
const mem = estimateMemoryGb(model, q, fullCtx);
|
||||
if (mem <= ram) return { runMode: 'cpu_offload', quant: q, ctx: fullCtx, requiredGb: mem };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// No GPU: CPU-only. Best quant that fits RAM, shrinking ctx.
|
||||
for (let ctx = fullCtx; ctx >= MIN_CTX; ) {
|
||||
for (const q of ladder) {
|
||||
const mem = estimateMemoryGb(model, q, ctx);
|
||||
if (mem <= ram) return { runMode: 'cpu_only', quant: q, ctx, requiredGb: mem };
|
||||
}
|
||||
if (ctx === MIN_CTX) break;
|
||||
ctx = Math.max(MIN_CTX, Math.floor(ctx / 2));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function fitLevelFor(runMode: RunMode, requiredGb: number, budget: number, ram: number): FitLevel {
|
||||
if (runMode === 'gpu') {
|
||||
const ratio = budget > 0 ? requiredGb / budget : 1;
|
||||
if (ratio <= 0.7) return 'perfect';
|
||||
if (ratio <= 0.9) return 'good';
|
||||
return 'marginal';
|
||||
}
|
||||
if (runMode === 'cpu_offload') return ram >= requiredGb * 1.2 ? 'good' : 'marginal';
|
||||
return 'marginal'; // cpu_only
|
||||
}
|
||||
|
||||
function analyzeModel(model: CatalogModel, hw: Hardware, opts: RankOptions): ModelRecommendation {
|
||||
const scoreUseCase = normalizeUseCase(opts.useCase);
|
||||
const modelUseCase = inferUseCase(model);
|
||||
const category = modelUseCase.charAt(0).toUpperCase() + modelUseCase.slice(1);
|
||||
const vram = hw.hasGpu && hw.gpuVramGb && hw.gpuVramGb > 0 ? hw.gpuVramGb : 0;
|
||||
const ram = hw.availableRamGb;
|
||||
const fit = resolveFit(model, hw, opts);
|
||||
|
||||
if (!fit) {
|
||||
const q = opts.quant ?? QUANT_HIERARCHY[3]; // Q4_K_M reference
|
||||
const required = estimateMemoryGb(model, q, model.contextLength || 4096);
|
||||
return {
|
||||
name: model.name, provider: model.provider, parameterCount: model.parameterCount,
|
||||
paramsB: model.paramsB, useCase: modelUseCase, category,
|
||||
fitLevel: 'too_tight', score: 0,
|
||||
scoreComponents: { quality: 0, speed: 0, fit: 0, context: 0 },
|
||||
estimatedTps: 0, memoryRequiredGb: round1(required),
|
||||
memoryAvailableGb: vram > 0 ? vram : ram, utilizationPct: 0,
|
||||
bestQuant: q, runMode: 'no_fit', runtime: 'Ollama',
|
||||
contextLength: model.contextLength, isMoe: model.isMoe,
|
||||
notes: ['Exceeds available memory at the smallest quant.'],
|
||||
};
|
||||
}
|
||||
|
||||
const budget = fit.runMode === 'gpu' ? vram : ram;
|
||||
let offloadFrac = 0;
|
||||
if (fit.runMode === 'cpu_offload' && fit.requiredGb > 0 && vram > 0) {
|
||||
offloadFrac = Math.max(0, (fit.requiredGb - vram) / fit.requiredGb);
|
||||
}
|
||||
const tps = estimateTps(model, fit.quant, fit.runMode, hw, offloadFrac);
|
||||
const quality = qualityScore(model, fit.quant, scoreUseCase);
|
||||
const speed = speedScore(tps, scoreUseCase);
|
||||
const fitS = fitScore(fit.requiredGb, budget);
|
||||
const ctxS = contextScore(fit.ctx, scoreUseCase);
|
||||
const [wq, ws, wf, wc] = USE_CASE_WEIGHTS[scoreUseCase] ?? DEFAULT_WEIGHTS;
|
||||
const composite = quality * wq + speed * ws + fitS * wf + ctxS * wc;
|
||||
|
||||
const notes: string[] = [];
|
||||
if (fit.runMode === 'cpu_offload') notes.push('Partially offloaded to system RAM (slower).');
|
||||
if (fit.runMode === 'cpu_only') notes.push('Runs on CPU — no compatible GPU detected.');
|
||||
if (model.isMoe) notes.push(`Mixture-of-Experts: ~${activeParamsB(model)}B active per token.`);
|
||||
if (fit.ctx < (model.contextLength || 0)) notes.push(`Context reduced to ${fit.ctx} to fit memory.`);
|
||||
|
||||
return {
|
||||
name: model.name, provider: model.provider, parameterCount: model.parameterCount,
|
||||
paramsB: model.paramsB, useCase: modelUseCase, category,
|
||||
fitLevel: fitLevelFor(fit.runMode, fit.requiredGb, budget, ram),
|
||||
score: round1(composite),
|
||||
scoreComponents: { quality: round1(quality), speed: round1(speed), fit: round1(fitS), context: round1(ctxS) },
|
||||
estimatedTps: round1(tps), memoryRequiredGb: round1(fit.requiredGb),
|
||||
memoryAvailableGb: round1(budget),
|
||||
utilizationPct: budget > 0 ? Math.round((fit.requiredGb / budget) * 100) : 0,
|
||||
bestQuant: fit.quant, runMode: fit.runMode, runtime: 'Ollama',
|
||||
contextLength: model.contextLength, isMoe: model.isMoe, notes,
|
||||
};
|
||||
}
|
||||
|
||||
function round1(n: number): number { return Math.round(n * 10) / 10; }
|
||||
|
||||
/** Rank a catalog against detected hardware. Sorted by composite score desc, then
|
||||
* newer version (tiebreak). Port of rank_models (serve-path gated, use-case filtered). */
|
||||
export function rankModels(
|
||||
catalog: ReadonlyArray<CatalogModel>, hw: Hardware, opts: RankOptions = {},
|
||||
): ModelRecommendation[] {
|
||||
const limit = opts.limit ?? 20;
|
||||
const search = opts.search?.toLowerCase();
|
||||
const wantUseCase = normalizeUseCase(opts.useCase); // unknown/inherited keys → 'general'
|
||||
const out: Array<{ rec: ModelRecommendation; version: number }> = [];
|
||||
|
||||
for (const model of catalog) {
|
||||
if (!isServable(model, hw)) continue;
|
||||
if (search && !model.name.toLowerCase().includes(search) && !model.provider.toLowerCase().includes(search)) continue;
|
||||
// Use-case filter: when a concrete (non-general) use-case is requested, keep
|
||||
// only models of that use-case. 'general' shows everything.
|
||||
if (wantUseCase !== 'general' && inferUseCase(model) !== wantUseCase) continue;
|
||||
const rec = analyzeModel(model, hw, opts);
|
||||
if (opts.fitOnly && rec.fitLevel === 'too_tight') continue;
|
||||
out.push({ rec, version: versionKey(model.name) });
|
||||
}
|
||||
|
||||
out.sort((a, b) => (b.rec.score - a.rec.score) || (b.version - a.version));
|
||||
return out.slice(0, limit).map((x) => x.rec);
|
||||
}
|
||||
35
packages/agent/src/cookbook/quant-tables.ts
Normal file
35
packages/agent/src/cookbook/quant-tables.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* Quant realism tables — clean-room port of the *concept* behind Odysseus
|
||||
* hwfit `models.py` (QUANT_BYTES_PER_PARAM / QUANT_SPEED_MULT / QUANT_QUALITY_PENALTY).
|
||||
* Scope-cut to GGUF k-quant tiers + the float formats the memory/speed math needs.
|
||||
* The AWQ/GPTQ/MLX/FP4-MoE-mixed prequant long tail is intentionally omitted — a
|
||||
* curated Ollama catalog never surfaces those serving paths. (AGPL-3.0: math/tables
|
||||
* authored fresh, no code copied, no binary bundled.)
|
||||
*/
|
||||
|
||||
/** GGUF quant tiers, highest quality → smallest. Walked to pick best-fitting quant. */
|
||||
export const QUANT_HIERARCHY = ['Q8_0', 'Q6_K', 'Q5_K_M', 'Q4_K_M', 'Q3_K_M', 'Q2_K'] as const;
|
||||
|
||||
/** Bytes per parameter — drives VRAM/RAM weight footprint. */
|
||||
export const QUANT_BYTES_PER_PARAM: Readonly<Record<string, number>> = {
|
||||
F16: 2.0, BF16: 2.0, FP8: 1.0,
|
||||
Q8_0: 1.0, Q6_K: 0.75, Q5_K_M: 0.625,
|
||||
Q4_K_M: 0.5, Q4_0: 0.5, Q3_K_M: 0.375, Q2_K: 0.25,
|
||||
};
|
||||
|
||||
/** Speed multiplier for the CPU/fallback tok/s path — smaller quants stream faster. */
|
||||
export const QUANT_SPEED_MULT: Readonly<Record<string, number>> = {
|
||||
F16: 0.6, BF16: 0.6, FP8: 0.85,
|
||||
Q8_0: 0.8, Q6_K: 0.95, Q5_K_M: 1.0,
|
||||
Q4_K_M: 1.15, Q4_0: 1.15, Q3_K_M: 1.25, Q2_K: 1.35,
|
||||
};
|
||||
|
||||
/** Quality delta (points) added to the base quality score for the chosen quant. */
|
||||
export const QUANT_QUALITY_PENALTY: Readonly<Record<string, number>> = {
|
||||
F16: 0.0, BF16: 0.0, FP8: 0.0,
|
||||
Q8_0: 0.0, Q6_K: -1.0, Q5_K_M: -2.0,
|
||||
Q4_K_M: -5.0, Q4_0: -5.0, Q3_K_M: -8.0, Q2_K: -12.0,
|
||||
};
|
||||
|
||||
export const DEFAULT_BPP = 0.5; // unknown quant ≈ a 4-bit GGUF
|
||||
export const DEFAULT_SPEED_MULT = 1.0;
|
||||
Reference in New Issue
Block a user