- tsconfig.build.json references the nine in-scope packages; each package is composite and references the workspace packages it depends on. build:packages is now `tsc -b tsconfig.build.json` instead of a chain of cd commands. - Every in-scope package exports its TypeScript entry under a `source` condition next to the dist/ entries. Vitest, tsx (dev:server, the Tauri dev launch, the e2e server), the sidecar and launcher bundles and the web app resolve it, so none of them needs build:packages first or can pick up a stale dist/. The vitest src aliases remain only for the excluded packages that have no source export. - Published packages with a files list (agent, cli, marketplace) pack src/ too, so the source condition never points at a missing file. - tsconfig.tests.json and `npm run typecheck:tests` type-check the test suites of the in-scope packages. They report 426 existing errors, so the CI step is advisory for now. - The root test run leaves out the excluded and postponed packages (hive-mind-*, memory-mcp, wiki-compiler, team-server, waggle-dance, worker, weaver); they are not built either. - CI runs the test job on Ubuntu, macOS and Windows with Node 22.23.2 (ADR-0008), and no longer builds the hook runtime and the memory MCP server.
289 lines
11 KiB
JavaScript
289 lines
11 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Build the Node.js sidecar for Tauri production.
|
|
*
|
|
* Bundles packages/server/src/local/service.ts into a single JS file
|
|
* at app/src-tauri/resources/service.js using esbuild JS API.
|
|
*
|
|
* Usage:
|
|
* node scripts/build-sidecar.mjs
|
|
*/
|
|
|
|
import fs from 'node:fs';
|
|
import { createHash } from 'node:crypto';
|
|
import { execFileSync } from 'node:child_process';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import ts from 'typescript';
|
|
|
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
const root = path.resolve(__dirname, '..');
|
|
const resourcesDir = path.join(root, 'app', 'src-tauri', 'resources');
|
|
const outFile = path.join(resourcesDir, 'service.js');
|
|
const sourceMapFile = `${outFile}.map`;
|
|
const entryPoint = path.join(root, 'packages', 'server', 'src', 'local', 'service.ts');
|
|
const marketplaceDbRelative = 'packages/marketplace/marketplace.db';
|
|
const marketplaceDb = path.join(root, 'packages', 'marketplace', 'marketplace.db');
|
|
const marketplaceResource = path.join(resourcesDir, 'marketplace.db');
|
|
const provenancePrefix = '// Waggle-Sidecar-Provenance: ';
|
|
const buildScriptRelative = 'scripts/build-sidecar.mjs';
|
|
const entryPointRelative = 'packages/server/src/local/service.ts';
|
|
|
|
function sha256(value) {
|
|
return createHash('sha256').update(value).digest('hex');
|
|
}
|
|
|
|
function readGitBlob(revision, relative) {
|
|
return execFileSync(
|
|
'git',
|
|
['-C', root, 'cat-file', 'blob', `${revision}:${relative}`],
|
|
{ maxBuffer: 64 * 1024 * 1024, windowsHide: true },
|
|
);
|
|
}
|
|
|
|
function repositoryRelative(absolutePath) {
|
|
const relative = path.relative(root, absolutePath);
|
|
if (!relative || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
|
throw new Error(`Sidecar input is outside the repository: ${absolutePath}`);
|
|
}
|
|
return relative.split(path.sep).join('/');
|
|
}
|
|
|
|
function addImplicitBuildInputs(localInputs, inputRelative) {
|
|
let current = path.dirname(path.join(root, ...inputRelative.split('/')));
|
|
while (true) {
|
|
for (const name of ['package.json', 'tsconfig.json']) {
|
|
const candidate = path.join(current, name);
|
|
if (fs.existsSync(candidate)) localInputs.add(repositoryRelative(candidate));
|
|
}
|
|
if (current === root) break;
|
|
const parent = path.dirname(current);
|
|
if (parent === current) break;
|
|
current = parent;
|
|
}
|
|
}
|
|
|
|
function addExtendedTsconfigs(localInputs) {
|
|
const visited = new Set();
|
|
const pending = [...localInputs].filter((relative) => path.basename(relative) === 'tsconfig.json');
|
|
while (pending.length > 0) {
|
|
const relative = pending.pop();
|
|
if (visited.has(relative)) continue;
|
|
visited.add(relative);
|
|
const absolute = path.join(root, ...relative.split('/'));
|
|
const parsed = ts.parseConfigFileTextToJson(absolute, fs.readFileSync(absolute, 'utf8'));
|
|
if (parsed.error) throw new Error(`Could not parse sidecar tsconfig: ${relative}`);
|
|
const rawExtends = parsed.config?.extends;
|
|
const extendedConfigs = Array.isArray(rawExtends)
|
|
? rawExtends
|
|
: typeof rawExtends === 'string'
|
|
? [rawExtends]
|
|
: [];
|
|
for (const extendedConfig of extendedConfigs) {
|
|
if (typeof extendedConfig !== 'string' || !extendedConfig.startsWith('.')) continue;
|
|
let extended = path.resolve(path.dirname(absolute), extendedConfig);
|
|
if (!fs.existsSync(extended) && fs.existsSync(`${extended}.json`)) extended += '.json';
|
|
const extendedRelative = repositoryRelative(extended);
|
|
if (!fs.existsSync(extended)) {
|
|
throw new Error(`Extended sidecar tsconfig is missing: ${extendedRelative}`);
|
|
}
|
|
localInputs.add(extendedRelative);
|
|
pending.push(extendedRelative);
|
|
}
|
|
}
|
|
}
|
|
// Metafile goes to a temp path (NOT resources/) so it's neither bundled into
|
|
// the app nor left as an untracked repo artifact. stage-sidecar-deps.mjs reads
|
|
// it back from the same well-known path. Keep the two in sync.
|
|
const metaFile = path.join(os.tmpdir(), 'waggle-sidecar-meta.json');
|
|
|
|
// Ensure resources directory exists
|
|
fs.mkdirSync(resourcesDir, { recursive: true });
|
|
// Production resources are shipped verbatim by Tauri. Remove maps left by an
|
|
// older build before bundling so full TypeScript sources cannot ride along in
|
|
// an installer even when the resources directory is reused.
|
|
fs.rmSync(sourceMapFile, { force: true });
|
|
|
|
console.log('[build-sidecar] Bundling server into', outFile);
|
|
|
|
// Native modules that can't be bundled — must be installed alongside the sidecar
|
|
const EXTERNAL = [
|
|
'better-sqlite3',
|
|
'@vscode/sqlite3',
|
|
'pg',
|
|
'mammoth',
|
|
'pdf-parse',
|
|
'exceljs',
|
|
'archiver',
|
|
'@fastify/static',
|
|
'@huggingface/transformers',
|
|
'onnxruntime-node',
|
|
'onnxruntime-common',
|
|
'onnxruntime-web',
|
|
'sharp',
|
|
'playwright-core',
|
|
'playwright-core/*',
|
|
'@playwright/*',
|
|
'chromium-bidi',
|
|
'chromium-bidi/*',
|
|
// WhatsApp channel (ADR-0005). whatsapp-web.js is CommonJS that relies on
|
|
// __dirname and fails inside the ESM bundle; bundling it would also inline
|
|
// its LGPL-3.0 dependency node-webpmux and the TypeScript compiler (an
|
|
// optional peer of Puppeteer's config loader). Staged as separate packages.
|
|
'whatsapp-web.js',
|
|
'@puppeteer/browsers',
|
|
// Memory (ADR-0007). mem0 imports a client for every provider it supports
|
|
// (vector stores, LLMs, embedders) on demand; those are optional peers that
|
|
// Waggle does not install. Kept external and staged with its own
|
|
// dependencies, so the providers Waggle does not use are never resolved.
|
|
'mem0ai',
|
|
];
|
|
|
|
try {
|
|
if (
|
|
!fs.existsSync(marketplaceDb)
|
|
|| !fs.lstatSync(marketplaceDb).isFile()
|
|
|| fs.lstatSync(marketplaceDb).isSymbolicLink()
|
|
) {
|
|
throw new Error(`Required marketplace database is missing or unsafe: ${marketplaceDb}`);
|
|
}
|
|
const sourceRevision = execFileSync(
|
|
'git',
|
|
['-C', root, 'rev-parse', 'HEAD'],
|
|
{ encoding: 'utf8', windowsHide: true },
|
|
).trim().toLowerCase();
|
|
if (!/^[0-9a-f]{40}$/.test(sourceRevision)) {
|
|
throw new Error(`Could not resolve an exact source revision: ${sourceRevision}`);
|
|
}
|
|
const marketplaceGitBlob = readGitBlob(sourceRevision, marketplaceDbRelative);
|
|
if (!fs.readFileSync(marketplaceDb).equals(marketplaceGitBlob)) {
|
|
throw new Error('canonical marketplace database does not match exact source revision');
|
|
}
|
|
|
|
// Dynamic import esbuild (available via vite dependency)
|
|
const esbuild = await import('esbuild');
|
|
|
|
const result = await esbuild.build({
|
|
entryPoints: [entryPoint],
|
|
absWorkingDir: root,
|
|
bundle: true,
|
|
platform: 'node',
|
|
target: 'node20',
|
|
format: 'esm',
|
|
outfile: outFile,
|
|
external: EXTERNAL,
|
|
// P4/D12: workspace packages publish dist/ entries, which are gitignored
|
|
// and can be stale. Every first-party package also exports its source
|
|
// under the `source` condition, so the bundle always compiles from src/
|
|
// and a clean checkout builds without build:packages.
|
|
conditions: ['source'],
|
|
sourcemap: false,
|
|
minify: true,
|
|
// metafile lets stage-sidecar-deps.mjs enumerate exactly which of the
|
|
// EXTERNAL packages the bundle actually `require()`s at runtime, so it
|
|
// stages precisely that set (and its transitive prod deps) into
|
|
// resources/node_modules/ — no more, no less. Written next to the bundle.
|
|
metafile: true,
|
|
banner: {
|
|
js: '// Waggle Sidecar — bundled server for Tauri desktop\n'
|
|
+ '// Generated by scripts/build-sidecar.mjs\n'
|
|
+ 'import { createRequire } from "node:module";\n'
|
|
+ 'const require = createRequire(import.meta.url);\n',
|
|
},
|
|
logLevel: 'warning',
|
|
});
|
|
|
|
if (result.errors.length > 0) {
|
|
console.error('[build-sidecar] Build errors:', result.errors);
|
|
process.exit(1);
|
|
}
|
|
|
|
if (result.warnings.length > 0) {
|
|
console.warn(`[build-sidecar] ${result.warnings.length} warnings (non-blocking)`);
|
|
}
|
|
|
|
fs.writeFileSync(metaFile, JSON.stringify(result.metafile));
|
|
console.log('[build-sidecar] Wrote esbuild metafile', metaFile);
|
|
|
|
const trackedFiles = new Set(
|
|
execFileSync(
|
|
'git',
|
|
['-C', root, 'ls-files', '-z'],
|
|
{ encoding: 'utf8', windowsHide: true },
|
|
).split('\0').filter(Boolean),
|
|
);
|
|
const localInputs = new Set([buildScriptRelative, 'package.json', 'package-lock.json']);
|
|
for (const input of Object.keys(result.metafile.inputs)) {
|
|
const relative = repositoryRelative(path.resolve(root, input));
|
|
if (relative === 'node_modules' || relative.startsWith('node_modules/')) continue;
|
|
localInputs.add(relative);
|
|
}
|
|
for (const relative of [...localInputs]) {
|
|
addImplicitBuildInputs(localInputs, relative);
|
|
}
|
|
addExtendedTsconfigs(localInputs);
|
|
|
|
const sourceInputs = [...localInputs]
|
|
.sort((left, right) => left < right ? -1 : left > right ? 1 : 0)
|
|
.map((relative) => {
|
|
if (!trackedFiles.has(relative)) {
|
|
throw new Error(`Sidecar repository input is not tracked: ${relative}`);
|
|
}
|
|
const absolute = path.join(root, ...relative.split('/'));
|
|
const stat = fs.lstatSync(absolute);
|
|
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
throw new Error(`Sidecar repository input is not a regular file: ${relative}`);
|
|
}
|
|
return {
|
|
path: relative,
|
|
sha256: sha256(fs.readFileSync(absolute)),
|
|
};
|
|
});
|
|
const bundlePayload = fs.readFileSync(outFile);
|
|
const provenance = {
|
|
schemaVersion: 1,
|
|
sourceRevision,
|
|
entryPoint: entryPointRelative,
|
|
sourceInputs,
|
|
bundlePayload: {
|
|
sizeBytes: bundlePayload.byteLength,
|
|
sha256: sha256(bundlePayload),
|
|
},
|
|
};
|
|
const provenanceLine = Buffer.from(
|
|
`${provenancePrefix}${Buffer.from(JSON.stringify(provenance)).toString('base64')}\n`,
|
|
'utf8',
|
|
);
|
|
const provenanceTemp = `${outFile}.provenance-${process.pid}.tmp`;
|
|
try {
|
|
fs.writeFileSync(provenanceTemp, Buffer.concat([provenanceLine, bundlePayload]));
|
|
fs.rmSync(outFile, { force: true });
|
|
fs.renameSync(provenanceTemp, outFile);
|
|
} finally {
|
|
fs.rmSync(provenanceTemp, { force: true });
|
|
}
|
|
console.log(
|
|
`[build-sidecar] Embedded ${sourceInputs.length} source inputs at ${sourceRevision.slice(0, 12)}`,
|
|
);
|
|
|
|
const stat = fs.statSync(outFile);
|
|
const sizeMB = (stat.size / 1024 / 1024).toFixed(1);
|
|
console.log(`[build-sidecar] Done. Output: ${sizeMB} MB`);
|
|
|
|
// Production startup seeds the user's writable DB from this immutable
|
|
// packaged resource. Publish the exact Git blob through a temporary file so
|
|
// a failed write can never leave a partial database in the bundle.
|
|
const marketplaceTemp = `${marketplaceResource}.stage-${process.pid}.tmp`;
|
|
try {
|
|
fs.writeFileSync(marketplaceTemp, marketplaceGitBlob, { flag: 'wx' });
|
|
fs.renameSync(marketplaceTemp, marketplaceResource);
|
|
} finally {
|
|
fs.rmSync(marketplaceTemp, { force: true });
|
|
}
|
|
console.log('[build-sidecar] Copied canonical marketplace.db');
|
|
} catch (err) {
|
|
console.error('[build-sidecar] Build failed:', err.message);
|
|
process.exit(1);
|
|
}
|