Files
waggle-os/scripts/build-sidecar.mjs
shumeiko.it bf74f02420 fix: ship capability packs and starter skills, report the usable model (UF-066, UF-067)
The SDK reads pack manifests and starter skills next to its own module,
which in the bundle is service.js, and no build step copied them there:
installed builds showed no packs and no starter skills. build-sidecar now
copies both folders next to service.js and Tauri bundles them (UF-066).

GET /api/agent/status returned the configured model even when it had no
key, so the status bar named Claude while chats ran on the local model.
It now resolves the model the way the chat route does (UF-067).
2026-11-01 11:15:37 +01:00

319 lines
12 KiB
JavaScript

#!/usr/bin/env node
/**
* Build the Node.js sidecar for Tauri production.
*
* Bundles packages/server/src/local/service.ts into a single JS file
* at app/src-tauri/resources/service.js using esbuild JS API.
*
* Usage:
* node scripts/build-sidecar.mjs
*/
import fs from 'node:fs';
import { createHash } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import os from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import ts from 'typescript';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const root = path.resolve(__dirname, '..');
const resourcesDir = path.join(root, 'app', 'src-tauri', 'resources');
const outFile = path.join(resourcesDir, 'service.js');
const sourceMapFile = `${outFile}.map`;
const entryPoint = path.join(root, 'packages', 'server', 'src', 'local', 'service.ts');
const marketplaceDbRelative = 'packages/marketplace/marketplace.db';
const marketplaceDb = path.join(root, 'packages', 'marketplace', 'marketplace.db');
const marketplaceResource = path.join(resourcesDir, 'marketplace.db');
const provenancePrefix = '// Waggle-Sidecar-Provenance: ';
const buildScriptRelative = 'scripts/build-sidecar.mjs';
const entryPointRelative = 'packages/server/src/local/service.ts';
function sha256(value) {
return createHash('sha256').update(value).digest('hex');
}
function readGitBlob(revision, relative) {
return execFileSync(
'git',
['-C', root, 'cat-file', 'blob', `${revision}:${relative}`],
{ maxBuffer: 64 * 1024 * 1024, windowsHide: true },
);
}
/** The checkout's exact revision, or null when the tree is not a git checkout (a source archive). */
function gitRevision() {
if (!fs.existsSync(path.join(root, '.git'))) return null;
const revision = execFileSync(
'git',
['-C', root, 'rev-parse', 'HEAD'],
{ encoding: 'utf8', windowsHide: true },
).trim().toLowerCase();
if (!/^[0-9a-f]{40}$/.test(revision)) {
throw new Error(`Could not resolve an exact source revision: ${revision}`);
}
return revision;
}
function repositoryRelative(absolutePath) {
const relative = path.relative(root, absolutePath);
if (!relative || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
throw new Error(`Sidecar input is outside the repository: ${absolutePath}`);
}
return relative.split(path.sep).join('/');
}
function addImplicitBuildInputs(localInputs, inputRelative) {
let current = path.dirname(path.join(root, ...inputRelative.split('/')));
while (true) {
for (const name of ['package.json', 'tsconfig.json']) {
const candidate = path.join(current, name);
if (fs.existsSync(candidate)) localInputs.add(repositoryRelative(candidate));
}
if (current === root) break;
const parent = path.dirname(current);
if (parent === current) break;
current = parent;
}
}
function addExtendedTsconfigs(localInputs) {
const visited = new Set();
const pending = [...localInputs].filter((relative) => path.basename(relative) === 'tsconfig.json');
while (pending.length > 0) {
const relative = pending.pop();
if (visited.has(relative)) continue;
visited.add(relative);
const absolute = path.join(root, ...relative.split('/'));
const parsed = ts.parseConfigFileTextToJson(absolute, fs.readFileSync(absolute, 'utf8'));
if (parsed.error) throw new Error(`Could not parse sidecar tsconfig: ${relative}`);
const rawExtends = parsed.config?.extends;
const extendedConfigs = Array.isArray(rawExtends)
? rawExtends
: typeof rawExtends === 'string'
? [rawExtends]
: [];
for (const extendedConfig of extendedConfigs) {
if (typeof extendedConfig !== 'string' || !extendedConfig.startsWith('.')) continue;
let extended = path.resolve(path.dirname(absolute), extendedConfig);
if (!fs.existsSync(extended) && fs.existsSync(`${extended}.json`)) extended += '.json';
const extendedRelative = repositoryRelative(extended);
if (!fs.existsSync(extended)) {
throw new Error(`Extended sidecar tsconfig is missing: ${extendedRelative}`);
}
localInputs.add(extendedRelative);
pending.push(extendedRelative);
}
}
}
// Metafile goes to a temp path (NOT resources/) so it's neither bundled into
// the app nor left as an untracked repo artifact. stage-sidecar-deps.mjs reads
// it back from the same well-known path. Keep the two in sync.
const metaFile = path.join(os.tmpdir(), 'waggle-sidecar-meta.json');
// Ensure resources directory exists
fs.mkdirSync(resourcesDir, { recursive: true });
// Production resources are shipped verbatim by Tauri. Remove maps left by an
// older build before bundling so full TypeScript sources cannot ride along in
// an installer even when the resources directory is reused.
fs.rmSync(sourceMapFile, { force: true });
console.log('[build-sidecar] Bundling server into', outFile);
// Native modules that can't be bundled — must be installed alongside the sidecar
const EXTERNAL = [
'better-sqlite3',
'@vscode/sqlite3',
'pg',
'mammoth',
'pdf-parse',
'exceljs',
'archiver',
'@fastify/static',
'@huggingface/transformers',
'onnxruntime-node',
'onnxruntime-common',
'onnxruntime-web',
'sharp',
'playwright-core',
'playwright-core/*',
'@playwright/*',
'chromium-bidi',
'chromium-bidi/*',
// WhatsApp channel (ADR-0005). whatsapp-web.js is CommonJS that relies on
// __dirname and fails inside the ESM bundle; bundling it would also inline
// its LGPL-3.0 dependency node-webpmux and the TypeScript compiler (an
// optional peer of Puppeteer's config loader). Staged as separate packages.
'whatsapp-web.js',
'@puppeteer/browsers',
// Memory (ADR-0007). mem0 imports a client for every provider it supports
// (vector stores, LLMs, embedders) on demand; those are optional peers that
// Waggle does not install. Kept external and staged with its own
// dependencies, so the providers Waggle does not use are never resolved.
'mem0ai',
];
try {
if (
!fs.existsSync(marketplaceDb)
|| !fs.lstatSync(marketplaceDb).isFile()
|| fs.lstatSync(marketplaceDb).isSymbolicLink()
) {
throw new Error(`Required marketplace database is missing or unsafe: ${marketplaceDb}`);
}
// A git checkout binds the bundle to its exact revision. A source archive
// has no history to check against, so its provenance records no revision.
const sourceRevision = gitRevision();
const marketplaceBytes = fs.readFileSync(marketplaceDb);
if (sourceRevision) {
const marketplaceGitBlob = readGitBlob(sourceRevision, marketplaceDbRelative);
if (!marketplaceBytes.equals(marketplaceGitBlob)) {
throw new Error('canonical marketplace database does not match exact source revision');
}
} else {
console.log('[build-sidecar] Not a git checkout: provenance records no source revision');
}
// Dynamic import esbuild (available via vite dependency)
const esbuild = await import('esbuild');
const result = await esbuild.build({
entryPoints: [entryPoint],
absWorkingDir: root,
bundle: true,
platform: 'node',
target: 'node20',
format: 'esm',
outfile: outFile,
external: EXTERNAL,
// P4/D12: workspace packages publish dist/ entries, which are gitignored
// and can be stale. Every first-party package also exports its source
// under the `source` condition, so the bundle always compiles from src/
// and a clean checkout builds without build:packages.
conditions: ['source'],
sourcemap: false,
minify: true,
// metafile lets stage-sidecar-deps.mjs enumerate exactly which of the
// EXTERNAL packages the bundle actually `require()`s at runtime, so it
// stages precisely that set (and its transitive prod deps) into
// resources/node_modules/ — no more, no less. Written next to the bundle.
metafile: true,
banner: {
js: '// Waggle Sidecar — bundled server for Tauri desktop\n'
+ '// Generated by scripts/build-sidecar.mjs\n'
+ 'import { createRequire } from "node:module";\n'
+ 'const require = createRequire(import.meta.url);\n',
},
logLevel: 'warning',
});
if (result.errors.length > 0) {
console.error('[build-sidecar] Build errors:', result.errors);
process.exit(1);
}
if (result.warnings.length > 0) {
console.warn(`[build-sidecar] ${result.warnings.length} warnings (non-blocking)`);
}
fs.writeFileSync(metaFile, JSON.stringify(result.metafile));
console.log('[build-sidecar] Wrote esbuild metafile', metaFile);
const trackedFiles = sourceRevision
? new Set(
execFileSync(
'git',
['-C', root, 'ls-files', '-z'],
{ encoding: 'utf8', windowsHide: true },
).split('\0').filter(Boolean),
)
: null;
const localInputs = new Set([buildScriptRelative, 'package.json', 'package-lock.json']);
for (const input of Object.keys(result.metafile.inputs)) {
const relative = repositoryRelative(path.resolve(root, input));
if (relative === 'node_modules' || relative.startsWith('node_modules/')) continue;
localInputs.add(relative);
}
for (const relative of [...localInputs]) {
addImplicitBuildInputs(localInputs, relative);
}
addExtendedTsconfigs(localInputs);
const sourceInputs = [...localInputs]
.sort((left, right) => left < right ? -1 : left > right ? 1 : 0)
.map((relative) => {
if (trackedFiles && !trackedFiles.has(relative)) {
throw new Error(`Sidecar repository input is not tracked: ${relative}`);
}
const absolute = path.join(root, ...relative.split('/'));
const stat = fs.lstatSync(absolute);
if (!stat.isFile() || stat.isSymbolicLink()) {
throw new Error(`Sidecar repository input is not a regular file: ${relative}`);
}
return {
path: relative,
sha256: sha256(fs.readFileSync(absolute)),
};
});
const bundlePayload = fs.readFileSync(outFile);
const provenance = {
schemaVersion: 1,
sourceRevision,
entryPoint: entryPointRelative,
sourceInputs,
bundlePayload: {
sizeBytes: bundlePayload.byteLength,
sha256: sha256(bundlePayload),
},
};
const provenanceLine = Buffer.from(
`${provenancePrefix}${Buffer.from(JSON.stringify(provenance)).toString('base64')}\n`,
'utf8',
);
const provenanceTemp = `${outFile}.provenance-${process.pid}.tmp`;
try {
fs.writeFileSync(provenanceTemp, Buffer.concat([provenanceLine, bundlePayload]));
fs.rmSync(outFile, { force: true });
fs.renameSync(provenanceTemp, outFile);
} finally {
fs.rmSync(provenanceTemp, { force: true });
}
console.log(
`[build-sidecar] Embedded ${sourceInputs.length} source inputs at ${sourceRevision ? sourceRevision.slice(0, 12) : 'no revision'}`,
);
const stat = fs.statSync(outFile);
const sizeMB = (stat.size / 1024 / 1024).toFixed(1);
console.log(`[build-sidecar] Done. Output: ${sizeMB} MB`);
// Production startup seeds the user's writable DB from this immutable
// packaged resource. Publish it through a temporary file so a failed write
// can never leave a partial database in the bundle.
const marketplaceTemp = `${marketplaceResource}.stage-${process.pid}.tmp`;
try {
fs.writeFileSync(marketplaceTemp, marketplaceBytes, { flag: 'wx' });
fs.renameSync(marketplaceTemp, marketplaceResource);
} finally {
fs.rmSync(marketplaceTemp, { force: true });
}
console.log('[build-sidecar] Copied canonical marketplace.db');
// @waggle/sdk reads its capability packs and starter skills from disk, next
// to its own module. In the bundle that module is service.js, so the files
// go next to it (getCapabilityPacksDir, getStarterSkillsDir).
for (const [name, extension] of [['capability-packs', '.json'], ['starter-skills', '.md']]) {
const source = path.join(root, 'packages', 'sdk', 'src', name);
const target = path.join(resourcesDir, name);
fs.rmSync(target, { recursive: true, force: true });
if (!fs.existsSync(source)) continue;
fs.mkdirSync(target, { recursive: true });
const files = fs.readdirSync(source).filter((file) => file.endsWith(extension));
for (const file of files) fs.copyFileSync(path.join(source, file), path.join(target, file));
console.log(`[build-sidecar] Copied ${files.length} ${name} files`);
}
} catch (err) {
console.error('[build-sidecar] Build failed:', err.message);
process.exit(1);
}