build-sidecar.mjs copies LICENSE and THIRD_PARTY_NOTICES.md from the repository root next to service.js in app/src-tauri/resources, and fails when either is missing. The resources/* glob of tauri.conf.json bundles them into every desktop installer, and build-server-archive.mjs refuses a payload without them, so the server archive carries them too. The copies are ignored by git. Tests cover the copy, the archive and the ignore rule.
330 lines
13 KiB
JavaScript
330 lines
13 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Build the Node.js sidecar for Tauri production.
|
|
*
|
|
* Bundles packages/server/src/local/service.ts into a single JS file
|
|
* at app/src-tauri/resources/service.js using esbuild JS API.
|
|
*
|
|
* Usage:
|
|
* node scripts/build-sidecar.mjs
|
|
*/
|
|
|
|
import fs from 'node:fs';
|
|
import { createHash } from 'node:crypto';
|
|
import { execFileSync } from 'node:child_process';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import ts from 'typescript';
|
|
|
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
const root = path.resolve(__dirname, '..');
|
|
const resourcesDir = path.join(root, 'app', 'src-tauri', 'resources');
|
|
const outFile = path.join(resourcesDir, 'service.js');
|
|
const sourceMapFile = `${outFile}.map`;
|
|
const entryPoint = path.join(root, 'packages', 'server', 'src', 'local', 'service.ts');
|
|
const marketplaceDbRelative = 'packages/marketplace/marketplace.db';
|
|
const marketplaceDb = path.join(root, 'packages', 'marketplace', 'marketplace.db');
|
|
const marketplaceResource = path.join(resourcesDir, 'marketplace.db');
|
|
const provenancePrefix = '// Waggle-Sidecar-Provenance: ';
|
|
const LEGAL_FILES = ['LICENSE', 'THIRD_PARTY_NOTICES.md'];
|
|
const buildScriptRelative = 'scripts/build-sidecar.mjs';
|
|
const entryPointRelative = 'packages/server/src/local/service.ts';
|
|
|
|
function sha256(value) {
|
|
return createHash('sha256').update(value).digest('hex');
|
|
}
|
|
|
|
function readGitBlob(revision, relative) {
|
|
return execFileSync(
|
|
'git',
|
|
['-C', root, 'cat-file', 'blob', `${revision}:${relative}`],
|
|
{ maxBuffer: 64 * 1024 * 1024, windowsHide: true },
|
|
);
|
|
}
|
|
|
|
/** The checkout's exact revision, or null when the tree is not a git checkout (a source archive). */
|
|
function gitRevision() {
|
|
if (!fs.existsSync(path.join(root, '.git'))) return null;
|
|
const revision = execFileSync(
|
|
'git',
|
|
['-C', root, 'rev-parse', 'HEAD'],
|
|
{ encoding: 'utf8', windowsHide: true },
|
|
).trim().toLowerCase();
|
|
if (!/^[0-9a-f]{40}$/.test(revision)) {
|
|
throw new Error(`Could not resolve an exact source revision: ${revision}`);
|
|
}
|
|
return revision;
|
|
}
|
|
|
|
function repositoryRelative(absolutePath) {
|
|
const relative = path.relative(root, absolutePath);
|
|
if (!relative || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
|
throw new Error(`Sidecar input is outside the repository: ${absolutePath}`);
|
|
}
|
|
return relative.split(path.sep).join('/');
|
|
}
|
|
|
|
function addImplicitBuildInputs(localInputs, inputRelative) {
|
|
let current = path.dirname(path.join(root, ...inputRelative.split('/')));
|
|
while (true) {
|
|
for (const name of ['package.json', 'tsconfig.json']) {
|
|
const candidate = path.join(current, name);
|
|
if (fs.existsSync(candidate)) localInputs.add(repositoryRelative(candidate));
|
|
}
|
|
if (current === root) break;
|
|
const parent = path.dirname(current);
|
|
if (parent === current) break;
|
|
current = parent;
|
|
}
|
|
}
|
|
|
|
function addExtendedTsconfigs(localInputs) {
|
|
const visited = new Set();
|
|
const pending = [...localInputs].filter((relative) => path.basename(relative) === 'tsconfig.json');
|
|
while (pending.length > 0) {
|
|
const relative = pending.pop();
|
|
if (visited.has(relative)) continue;
|
|
visited.add(relative);
|
|
const absolute = path.join(root, ...relative.split('/'));
|
|
const parsed = ts.parseConfigFileTextToJson(absolute, fs.readFileSync(absolute, 'utf8'));
|
|
if (parsed.error) throw new Error(`Could not parse sidecar tsconfig: ${relative}`);
|
|
const rawExtends = parsed.config?.extends;
|
|
const extendedConfigs = Array.isArray(rawExtends)
|
|
? rawExtends
|
|
: typeof rawExtends === 'string'
|
|
? [rawExtends]
|
|
: [];
|
|
for (const extendedConfig of extendedConfigs) {
|
|
if (typeof extendedConfig !== 'string' || !extendedConfig.startsWith('.')) continue;
|
|
let extended = path.resolve(path.dirname(absolute), extendedConfig);
|
|
if (!fs.existsSync(extended) && fs.existsSync(`${extended}.json`)) extended += '.json';
|
|
const extendedRelative = repositoryRelative(extended);
|
|
if (!fs.existsSync(extended)) {
|
|
throw new Error(`Extended sidecar tsconfig is missing: ${extendedRelative}`);
|
|
}
|
|
localInputs.add(extendedRelative);
|
|
pending.push(extendedRelative);
|
|
}
|
|
}
|
|
}
|
|
// Metafile goes to a temp path (NOT resources/) so it's neither bundled into
|
|
// the app nor left as an untracked repo artifact. stage-sidecar-deps.mjs reads
|
|
// it back from the same well-known path. Keep the two in sync.
|
|
const metaFile = path.join(os.tmpdir(), 'waggle-sidecar-meta.json');
|
|
|
|
// Ensure resources directory exists
|
|
fs.mkdirSync(resourcesDir, { recursive: true });
|
|
// Production resources are shipped verbatim by Tauri. Remove maps left by an
|
|
// older build before bundling so full TypeScript sources cannot ride along in
|
|
// an installer even when the resources directory is reused.
|
|
fs.rmSync(sourceMapFile, { force: true });
|
|
|
|
console.log('[build-sidecar] Bundling server into', outFile);
|
|
|
|
// Native modules that can't be bundled — must be installed alongside the sidecar
|
|
const EXTERNAL = [
|
|
'better-sqlite3',
|
|
'@vscode/sqlite3',
|
|
'pg',
|
|
'mammoth',
|
|
'pdf-parse',
|
|
'exceljs',
|
|
'archiver',
|
|
'@fastify/static',
|
|
'@huggingface/transformers',
|
|
'onnxruntime-node',
|
|
'onnxruntime-common',
|
|
'onnxruntime-web',
|
|
'sharp',
|
|
'playwright-core',
|
|
'playwright-core/*',
|
|
'@playwright/*',
|
|
'chromium-bidi',
|
|
'chromium-bidi/*',
|
|
// WhatsApp channel (ADR-0005). whatsapp-web.js is CommonJS that relies on
|
|
// __dirname and fails inside the ESM bundle; bundling it would also inline
|
|
// its LGPL-3.0 dependency node-webpmux and the TypeScript compiler (an
|
|
// optional peer of Puppeteer's config loader). Staged as separate packages.
|
|
'whatsapp-web.js',
|
|
'@puppeteer/browsers',
|
|
// Memory (ADR-0007). mem0 imports a client for every provider it supports
|
|
// (vector stores, LLMs, embedders) on demand; those are optional peers that
|
|
// Waggle does not install. Kept external and staged with its own
|
|
// dependencies, so the providers Waggle does not use are never resolved.
|
|
'mem0ai',
|
|
];
|
|
|
|
try {
|
|
if (
|
|
!fs.existsSync(marketplaceDb)
|
|
|| !fs.lstatSync(marketplaceDb).isFile()
|
|
|| fs.lstatSync(marketplaceDb).isSymbolicLink()
|
|
) {
|
|
throw new Error(`Required marketplace database is missing or unsafe: ${marketplaceDb}`);
|
|
}
|
|
// A git checkout binds the bundle to its exact revision. A source archive
|
|
// has no history to check against, so its provenance records no revision.
|
|
const sourceRevision = gitRevision();
|
|
const marketplaceBytes = fs.readFileSync(marketplaceDb);
|
|
if (sourceRevision) {
|
|
const marketplaceGitBlob = readGitBlob(sourceRevision, marketplaceDbRelative);
|
|
if (!marketplaceBytes.equals(marketplaceGitBlob)) {
|
|
throw new Error('canonical marketplace database does not match exact source revision');
|
|
}
|
|
} else {
|
|
console.log('[build-sidecar] Not a git checkout: provenance records no source revision');
|
|
}
|
|
|
|
// Dynamic import esbuild (available via vite dependency)
|
|
const esbuild = await import('esbuild');
|
|
|
|
const result = await esbuild.build({
|
|
entryPoints: [entryPoint],
|
|
absWorkingDir: root,
|
|
bundle: true,
|
|
platform: 'node',
|
|
target: 'node20',
|
|
format: 'esm',
|
|
outfile: outFile,
|
|
external: EXTERNAL,
|
|
// P4/D12: workspace packages publish dist/ entries, which are gitignored
|
|
// and can be stale. Every first-party package also exports its source
|
|
// under the `source` condition, so the bundle always compiles from src/
|
|
// and a clean checkout builds without build:packages.
|
|
conditions: ['source'],
|
|
sourcemap: false,
|
|
minify: true,
|
|
// metafile lets stage-sidecar-deps.mjs enumerate exactly which of the
|
|
// EXTERNAL packages the bundle actually `require()`s at runtime, so it
|
|
// stages precisely that set (and its transitive prod deps) into
|
|
// resources/node_modules/ — no more, no less. Written next to the bundle.
|
|
metafile: true,
|
|
banner: {
|
|
js: '// Waggle Sidecar — bundled server for Tauri desktop\n'
|
|
+ '// Generated by scripts/build-sidecar.mjs\n'
|
|
+ 'import { createRequire } from "node:module";\n'
|
|
+ 'const require = createRequire(import.meta.url);\n',
|
|
},
|
|
logLevel: 'warning',
|
|
});
|
|
|
|
if (result.errors.length > 0) {
|
|
console.error('[build-sidecar] Build errors:', result.errors);
|
|
process.exit(1);
|
|
}
|
|
|
|
if (result.warnings.length > 0) {
|
|
console.warn(`[build-sidecar] ${result.warnings.length} warnings (non-blocking)`);
|
|
}
|
|
|
|
fs.writeFileSync(metaFile, JSON.stringify(result.metafile));
|
|
console.log('[build-sidecar] Wrote esbuild metafile', metaFile);
|
|
|
|
const trackedFiles = sourceRevision
|
|
? new Set(
|
|
execFileSync(
|
|
'git',
|
|
['-C', root, 'ls-files', '-z'],
|
|
{ encoding: 'utf8', windowsHide: true },
|
|
).split('\0').filter(Boolean),
|
|
)
|
|
: null;
|
|
const localInputs = new Set([buildScriptRelative, 'package.json', 'package-lock.json']);
|
|
for (const input of Object.keys(result.metafile.inputs)) {
|
|
const relative = repositoryRelative(path.resolve(root, input));
|
|
if (relative === 'node_modules' || relative.startsWith('node_modules/')) continue;
|
|
localInputs.add(relative);
|
|
}
|
|
for (const relative of [...localInputs]) {
|
|
addImplicitBuildInputs(localInputs, relative);
|
|
}
|
|
addExtendedTsconfigs(localInputs);
|
|
|
|
const sourceInputs = [...localInputs]
|
|
.sort((left, right) => left < right ? -1 : left > right ? 1 : 0)
|
|
.map((relative) => {
|
|
if (trackedFiles && !trackedFiles.has(relative)) {
|
|
throw new Error(`Sidecar repository input is not tracked: ${relative}`);
|
|
}
|
|
const absolute = path.join(root, ...relative.split('/'));
|
|
const stat = fs.lstatSync(absolute);
|
|
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
throw new Error(`Sidecar repository input is not a regular file: ${relative}`);
|
|
}
|
|
return {
|
|
path: relative,
|
|
sha256: sha256(fs.readFileSync(absolute)),
|
|
};
|
|
});
|
|
const bundlePayload = fs.readFileSync(outFile);
|
|
const provenance = {
|
|
schemaVersion: 1,
|
|
sourceRevision,
|
|
entryPoint: entryPointRelative,
|
|
sourceInputs,
|
|
bundlePayload: {
|
|
sizeBytes: bundlePayload.byteLength,
|
|
sha256: sha256(bundlePayload),
|
|
},
|
|
};
|
|
const provenanceLine = Buffer.from(
|
|
`${provenancePrefix}${Buffer.from(JSON.stringify(provenance)).toString('base64')}\n`,
|
|
'utf8',
|
|
);
|
|
const provenanceTemp = `${outFile}.provenance-${process.pid}.tmp`;
|
|
try {
|
|
fs.writeFileSync(provenanceTemp, Buffer.concat([provenanceLine, bundlePayload]));
|
|
fs.rmSync(outFile, { force: true });
|
|
fs.renameSync(provenanceTemp, outFile);
|
|
} finally {
|
|
fs.rmSync(provenanceTemp, { force: true });
|
|
}
|
|
console.log(
|
|
`[build-sidecar] Embedded ${sourceInputs.length} source inputs at ${sourceRevision ? sourceRevision.slice(0, 12) : 'no revision'}`,
|
|
);
|
|
|
|
const stat = fs.statSync(outFile);
|
|
const sizeMB = (stat.size / 1024 / 1024).toFixed(1);
|
|
console.log(`[build-sidecar] Done. Output: ${sizeMB} MB`);
|
|
|
|
// Production startup seeds the user's writable DB from this immutable
|
|
// packaged resource. Publish it through a temporary file so a failed write
|
|
// can never leave a partial database in the bundle.
|
|
const marketplaceTemp = `${marketplaceResource}.stage-${process.pid}.tmp`;
|
|
try {
|
|
fs.writeFileSync(marketplaceTemp, marketplaceBytes, { flag: 'wx' });
|
|
fs.renameSync(marketplaceTemp, marketplaceResource);
|
|
} finally {
|
|
fs.rmSync(marketplaceTemp, { force: true });
|
|
}
|
|
console.log('[build-sidecar] Copied canonical marketplace.db');
|
|
|
|
// @waggle/sdk reads its capability packs and starter skills from disk, next
|
|
// to its own module. In the bundle that module is service.js, so the files
|
|
// go next to it (getCapabilityPacksDir, getStarterSkillsDir).
|
|
for (const [name, extension] of [['capability-packs', '.json'], ['starter-skills', '.md']]) {
|
|
const source = path.join(root, 'packages', 'sdk', 'src', name);
|
|
const target = path.join(resourcesDir, name);
|
|
fs.rmSync(target, { recursive: true, force: true });
|
|
if (!fs.existsSync(source)) continue;
|
|
fs.mkdirSync(target, { recursive: true });
|
|
const files = fs.readdirSync(source).filter((file) => file.endsWith(extension));
|
|
for (const file of files) fs.copyFileSync(path.join(source, file), path.join(target, file));
|
|
console.log(`[build-sidecar] Copied ${files.length} ${name} files`);
|
|
}
|
|
|
|
// The license and the third-party notices travel with every installer and
|
|
// with the server archive, which packs this folder (build-server-archive.mjs).
|
|
// The sidecar serves them to the UI from here (routes/legal.ts).
|
|
for (const name of LEGAL_FILES) {
|
|
const source = path.join(root, name);
|
|
if (!fs.existsSync(source)) throw new Error(`Required ${name} is missing at the repository root`);
|
|
fs.copyFileSync(source, path.join(resourcesDir, name));
|
|
}
|
|
console.log(`[build-sidecar] Copied ${LEGAL_FILES.join(' and ')}`);
|
|
} catch (err) {
|
|
console.error('[build-sidecar] Build failed:', err.message);
|
|
process.exit(1);
|
|
}
|