build: package the sidecar without the legacy memory runtime (P1.5, P1.3b)

- Stop building and staging the hive-mind CLI, the coding-agent hook
  packages and the memory MCP server: the hooks, Waggle Room and the
  memory MCP server are hidden in v1 (ADR-0002) and all of them write to
  the legacy .mind files.
- Drop sqlite-vec: nothing loads a SQLite extension since the legacy
  memory engine left. bundle-native-deps no longer copies vec0, the
  resource check no longer probes it, the Tauri shell no longer sets
  WAGGLE_SQLITE_VEC_PATH and install.sh checks plain SQLite.
- Stage bundle roots that npm keeps inside a workspace. mem0ai is not
  hoisted (it stays in packages/memory-mem0), so staging skipped it
  silently and the packaged sidecar would fail on the first memory
  access. Roots that are not installed are now listed in the output.
- js-yaml is a runtime dependency of Puppeteer's config loader since the
  WhatsApp channel moved to whatsapp-web.js (ADR-0005); it leaves the
  resource-check denylist. chromium-bidi stays unstaged: Puppeteer loads
  it only for WebDriver BiDi sessions.
This commit is contained in:
shumeiko.it
2026-10-24 16:37:48 +02:00
parent e435e7da94
commit bbfb3c7f82
6 changed files with 60 additions and 119 deletions

View File

@@ -681,17 +681,6 @@ fn build_service_command(
);
cmd.env("NODE_PATH", node_path);
let vec_ext = if cfg!(windows) {
native_dir.join("vec0.dll")
} else if cfg!(target_os = "macos") {
native_dir.join("vec0.dylib")
} else {
native_dir.join("vec0.so")
};
if vec_ext.exists() {
cmd.env("WAGGLE_SQLITE_VEC_PATH", vec_ext.to_string_lossy().as_ref());
}
let ort_dir = native_dir.join("onnxruntime");
if ort_dir.exists() {
cmd.env(

View File

@@ -241,22 +241,19 @@ run_wizard() {
}
# ── Post-install runtime verification (D10) ───────────────────────────────────
# Confirms the native SQLite stack actually loads in the installed tree: opens
# an in-memory better-sqlite3 db and loads sqlite-vec exactly as MindDB does
# (honoring WAGGLE_SQLITE_VEC_PATH). Turns the "does sqlite-vec resolve on this
# platform?" unknown into a visible, actionable check.
# Confirms the native SQLite module actually loads in the installed tree: opens
# an in-memory better-sqlite3 db, as waggle.db and the memory store do. Turns
# the "does the native module load on this platform?" unknown into a visible,
# actionable check.
verify_runtime() {
say "Verifying native SQLite + sqlite-vec"
say "Verifying native SQLite"
local script
script='
const Database = require("better-sqlite3");
const db = new Database(":memory:");
const vecPath = process.env.WAGGLE_SQLITE_VEC_PATH;
if (vecPath) { db.loadExtension(vecPath); }
else { require("sqlite-vec").load(db); }
const row = db.prepare("select vec_version() as v").get();
const row = db.prepare("select sqlite_version() as v").get();
db.close();
console.log("sqlite-vec " + row.v);
console.log("SQLite " + row.v);
'
local errfile
errfile="$(mktemp "${TMPDIR:-/tmp}/waggle-verify.XXXXXX")" \
@@ -268,10 +265,9 @@ verify_runtime() {
fi
local err; err="$(cat "$errfile" 2>/dev/null || true)"
rm -f "$errfile" 2>/dev/null || true
warn "Native SQLite/sqlite-vec check failed:
warn "Native SQLite check failed:
${err}
Remedies:
- Point WAGGLE_SQLITE_VEC_PATH at a sqlite-vec loadable extension for your platform, then re-run.
- Ensure a C/C++ toolchain is installed so better-sqlite3 can build:
$( [ "$OS" = macos ] && echo 'xcode-select --install' || echo 'sudo apt install build-essential python3' )
- Re-run: cd ${INSTALL_DIR} && npm install"

View File

@@ -22,7 +22,10 @@ const WINDOWS_1252_EXTRA_CODEPOINTS = new Set([
0x0160, 0x2039, 0x0152, 0x017d, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022,
0x2013, 0x2014, 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0x017e, 0x0178,
]);
const STAGED_DEPENDENCY_ALLOWLIST = new Set(['onnxruntime-web']);
// Declared dependencies the staged runtime never loads: onnxruntime-web (the
// browser backend of Transformers.js) and chromium-bidi, which Puppeteer loads
// only for WebDriver BiDi sessions; whatsapp-web.js drives Chrome over CDP.
const STAGED_DEPENDENCY_ALLOWLIST = new Set(['onnxruntime-web', 'chromium-bidi']);
const FIRST_PARTY_RUNTIME_ENTRIES = new Set([
'dist',
'LICENSE',
@@ -960,10 +963,10 @@ describe('Tauri Production Configuration', () => {
expect(vulnerableSharp.stderr).toContain('sharp@0.34.5');
writeManifest('sharp', 'sharp', '0.35.3');
// Puppeteer (WhatsApp, ADR-0005) parses its config with js-yaml at runtime.
writeManifest('vendor/node_modules/js-yaml', 'js-yaml', '4.3.0');
const stagedDevDependency = run();
expect(stagedDevDependency.status).toBe(1);
expect(stagedDevDependency.stderr).toContain('development-only js-yaml');
const stagedYamlParser = run();
expect(stagedYamlParser.stderr).not.toContain('js-yaml');
fs.rmSync(path.join(fixture, 'vendor'), { recursive: true, force: true });
fs.rmSync(
@@ -988,8 +991,9 @@ describe('Tauri Production Configuration', () => {
expect(script).toContain('execFileSync(nodePath');
expect(script).toContain('const database = new Database');
expect(script).toContain('SELECT 1 AS ok');
expect(script).toContain('SELECT vec_version() AS version');
expect(script).toContain('const onnx = require(process.argv[4])');
// No SQLite extension is loaded since the legacy memory engine left (P1.5).
expect(script).not.toContain('vec_version');
expect(script).toContain('const onnx = require(process.argv[3])');
expect(script).toContain('const { RawImage } = require(process.argv[1])');
expect(script).toContain('image.toSharp().resize(1, 1).png().toBuffer()');
expect(script).toContain('Buffer.from([137,80,78,71,13,10,26,10])');
@@ -1036,7 +1040,7 @@ describe('Tauri Production Configuration', () => {
);
expect(script).toContain("execFileSync('/usr/bin/lipo'");
expect(script).toContain('required native dependency');
expect(script).toContain('sqlite-vec-${vecOs}-${arch}');
expect(script).not.toContain('sqlite-vec');
expect(script).not.toContain("arch === 'arm64' ? 'aarch64'");
});
@@ -1051,21 +1055,9 @@ describe('Tauri Production Configuration', () => {
const fixtureChecker = path.join(fixtureScripts, 'check-sidecar-resources.mjs');
const requiredNativeFiles = [
'better_sqlite3.node',
'vec0.dll',
'onnxruntime/onnxruntime_binding.node',
];
const stagedRuntimeFiles = [
'@waggle/hive-mind-core/dist/index.js',
'@waggle/hive-mind-cli/dist/index.js',
'@waggle/hive-mind-hooks-claude-code/dist/bin/claude-code-hooks-cli.js',
'@waggle/hive-mind-hooks-claude-desktop/dist/bin/claude-desktop-hooks.js',
'@waggle/hive-mind-hooks-codex/dist/bin/codex-hooks.js',
'@waggle/hive-mind-hooks-codex-desktop/dist/bin/codex-desktop-hooks.js',
'@waggle/hive-mind-hooks-cursor/dist/bin/cursor-hooks.js',
'@waggle/hive-mind-hooks-hermes/dist/bin/hermes-hooks.js',
'@waggle/hive-mind-hooks-openclaw/dist/bin/openclaw-hooks.js',
'@waggle/hive-mind-hooks-openclaw/dist/handler.bundle.cjs',
'waggle-memory-mcp/dist/index.js',
'waggle-test-runtime/dist/index.js',
];
const writeFixtureFile = (base: string, relative: string, content = '') => {
@@ -1245,7 +1237,6 @@ describe('Tauri Production Configuration', () => {
);
}
const installedBinding = path.join(installedBetterSqlite, 'build', 'Release', 'better_sqlite3.node');
const installedVec = path.join(ROOT, 'node_modules', 'sqlite-vec-windows-x64', 'vec0.dll');
const installedOnnx = path.join(ROOT, 'node_modules', 'onnxruntime-node');
const fixtureOnnx = path.join(fixtureResources, 'node_modules', 'onnxruntime-node');
fs.mkdirSync(fixtureOnnx, { recursive: true });
@@ -1327,7 +1318,6 @@ describe('Tauri Production Configuration', () => {
for (const entry of requiredNativeFiles) {
const target = writeFixtureFile(path.join(fixtureResources, 'native'), entry);
if (entry === 'better_sqlite3.node') fs.copyFileSync(installedBinding, target);
if (entry === 'vec0.dll') fs.copyFileSync(installedVec, target);
if (entry === 'onnxruntime/onnxruntime_binding.node') fs.copyFileSync(installedOnnxBinding, target);
}
for (const entry of stagedRuntimeFiles) {
@@ -1616,8 +1606,7 @@ describe('Tauri Production Configuration', () => {
'better_sqlite3.node',
);
const stagedOnnxBinding = path.join(fixtureOnnxBin, 'onnxruntime_binding.node');
const fixtureVec = path.join(fixtureResources, 'native', 'vec0.dll');
for (const target of [stagedBinding, stagedOnnxBinding, fixtureVec]) {
for (const target of [stagedBinding, stagedOnnxBinding]) {
const original = fs.readFileSync(target);
fs.writeFileSync(target, 'not a native payload');
const invalidRuntimeResult = await runChecker({ runtimeProbe: 'native' });

View File

@@ -19,14 +19,14 @@ const nativeDir = path.join(root, 'app', 'src-tauri', 'resources', 'native');
const platform = process.platform;
const arch = process.env.TARGET_ARCH || process.arch;
// macOS "universal" is not a valid staging target: sqlite-vec and onnxruntime
// macOS "universal" is not a valid staging target: better-sqlite3 and onnxruntime
// ship per-arch binaries (no darwin/universal path exists), so a universal run
// silently falls through to the x64 variant and mis-stages the arm64 half.
// Build each arch separately and lipo the app bundle instead.
if (arch === 'universal') {
console.error(
'[bundle-native-deps] FATAL — TARGET_ARCH=universal is not supported.\n'
+ ' Native modules (sqlite-vec, onnxruntime-node) are per-arch. Build each\n'
+ ' Native modules (better-sqlite3, onnxruntime-node) are per-arch. Build each\n'
+ ' arch separately: TARGET_ARCH=arm64 (aarch64-apple-darwin) and\n'
+ ' TARGET_ARCH=x64 (x86_64-apple-darwin) — see release.yml\'s macOS matrix\n'
+ ' and the app tauri:build:mac:arm64 / :x64 scripts.',
@@ -114,16 +114,7 @@ function copyDir(srcDir, destSubDir) {
console.log('[bundle-native-deps] better-sqlite3:');
copyFile('node_modules/better-sqlite3/build/Release/better_sqlite3.node', 'better_sqlite3.node');
// 2. sqlite-vec
console.log('[bundle-native-deps] sqlite-vec:');
const vecOs = platform === 'win32' ? 'windows' : platform === 'darwin' ? 'darwin' : 'linux';
const vecExt = platform === 'win32' ? 'dll' : platform === 'darwin' ? 'dylib' : 'so';
copyFile(
`node_modules/sqlite-vec-${vecOs}-${arch}/vec0.${vecExt}`,
`vec0.${vecExt}`,
);
// 3. onnxruntime-node (multiple files — binding + shared libraries)
// 2. onnxruntime-node (multiple files — binding + shared libraries)
console.log('[bundle-native-deps] onnxruntime-node:');
const ortOs = platform === 'win32' ? 'win32' : platform === 'darwin' ? 'darwin' : 'linux';
const ortDir = `node_modules/onnxruntime-node/bin/napi-v3/${ortOs}/${arch}`;

View File

@@ -33,9 +33,8 @@ const targetArch = process.env.TARGET_ARCH || process.arch;
const SIDECAR_PROVENANCE_PREFIX = '// Waggle-Sidecar-Provenance: ';
const SOURCE_ARTIFACT_PATTERN = /(?:\.map|\.(?:[cm]?ts|tsx)|\.tsbuildinfo)$/i;
const FIRST_PARTY_RUNTIME_ENTRY_PATTERN = /^(?:dist|package\.json|licen[cs]e(?:\.(?:md|txt))?|notice(?:\.(?:md|txt))?)$/i;
const MANUAL_FIRST_PARTY_RUNTIME_TARGETS = new Map([
['@waggle/hive-mind-hooks-openclaw', ['dist/handler.bundle.cjs']],
]);
// Runtime files of a staged first-party package that its manifest does not name.
const MANUAL_FIRST_PARTY_RUNTIME_TARGETS = new Map();
const REQUIRED_SHARP_VERSION = '0.35.3';
const REQUIRED_BETTER_SQLITE_RANGE = '>=12.6.2 <13';
const STAGED_DEPENDENCY_VERSION_ALLOWLISTS = new Map([
@@ -43,7 +42,10 @@ const STAGED_DEPENDENCY_VERSION_ALLOWLISTS = new Map([
['fast-uri', new Set(['3.1.5'])],
['ip-address', new Set(['10.4.0'])],
]);
const STAGED_DEPENDENCY_DENYLIST = new Set(['js-yaml']);
// Packages that must never reach the installer. js-yaml left this list with the
// WhatsApp channel (ADR-0005): Puppeteer reads its config through cosmiconfig,
// which parses YAML with it at runtime.
const STAGED_DEPENDENCY_DENYLIST = new Set();
const missing = [];
const unsafe = [];
@@ -717,7 +719,6 @@ const nativeEntries = fs.existsSync(nativeDir)
: [];
const requiredWindowsNativeFiles = [
'better_sqlite3.node',
'vec0.dll',
'onnxruntime/onnxruntime_binding.node',
];
if (process.platform === 'win32') {
@@ -731,7 +732,6 @@ if (process.platform === 'win32') {
} else if (process.platform === 'darwin') {
const requiredMacNativeFiles = [
'better_sqlite3.node',
'vec0.dylib',
'onnxruntime/onnxruntime_binding.node',
];
for (const entry of requiredMacNativeFiles) {
@@ -772,10 +772,6 @@ const stagedSharpWindowsBinding = path.join(
'lib',
`sharp-win32-x64-${REQUIRED_SHARP_VERSION}.node`,
);
const vecExtension = path.join(
nativeDir,
`vec0.${process.platform === 'win32' ? 'dll' : process.platform === 'darwin' ? 'dylib' : 'so'}`,
);
if (!fs.existsSync(path.join(stagedBetterSqlite, 'package.json'))) {
missing.push('resources/node_modules/* (run: node scripts/stage-sidecar-deps.mjs)');
}
@@ -902,20 +898,16 @@ if (
fs.existsSync(nodePath)
&& fs.existsSync(path.join(stagedBetterSqlite, 'package.json'))
&& fs.existsSync(path.join(stagedOnnxRuntime, 'package.json'))
&& fs.existsSync(vecExtension)
) {
try {
const probe = [
'const Database = require(process.argv[1]);',
'if (process.arch !== process.argv[2]) throw new Error(`architecture ${process.arch}`);',
'const database = new Database(\':memory:\');',
'database.loadExtension(process.argv[3]);',
'const row = database.prepare(\'SELECT 1 AS ok\').get();',
'const vec = database.prepare(\'SELECT vec_version() AS version\').get();',
'database.close();',
'if (row.ok !== 1) throw new Error(\'SQLite query failed\');',
'if (typeof vec.version !== \'string\' || vec.version.length === 0) throw new Error(\'sqlite-vec query failed\');',
'const onnx = require(process.argv[4]);',
'const onnx = require(process.argv[3]);',
'if (typeof onnx.InferenceSession !== \'function\') throw new Error(\'ONNX binding failed\');',
].join('');
execFileSync(nodePath, [
@@ -923,7 +915,6 @@ if (
probe,
stagedBetterSqlite,
targetArch,
vecExtension,
stagedOnnxRuntime,
], {
cwd: resourcesDir,
@@ -932,32 +923,12 @@ if (
});
} catch {
missing.push(
`resources native runtime probe failed for better-sqlite3, sqlite-vec, or onnxruntime-node `
`resources native runtime probe failed for better-sqlite3 or onnxruntime-node `
+ `using bundled ${nodeBinary} for ${targetArch}`,
);
}
}
// External agents and hook management run directly from this staged payload;
// none of these packages are available from npm in a packaged installation.
const hookRuntimeEntries = [
'@waggle/hive-mind-cli/dist/index.js',
'@waggle/hive-mind-hooks-claude-code/dist/bin/claude-code-hooks-cli.js',
'@waggle/hive-mind-hooks-claude-desktop/dist/bin/claude-desktop-hooks.js',
'@waggle/hive-mind-hooks-codex/dist/bin/codex-hooks.js',
'@waggle/hive-mind-hooks-codex-desktop/dist/bin/codex-desktop-hooks.js',
'@waggle/hive-mind-hooks-cursor/dist/bin/cursor-hooks.js',
'@waggle/hive-mind-hooks-hermes/dist/bin/hermes-hooks.js',
'@waggle/hive-mind-hooks-openclaw/dist/bin/openclaw-hooks.js',
'@waggle/hive-mind-hooks-openclaw/dist/handler.bundle.cjs',
'waggle-memory-mcp/dist/index.js',
];
for (const entry of hookRuntimeEntries) {
if (!fs.existsSync(path.join(stagedDepsDir, ...entry.split('/')))) {
missing.push(`resources/node_modules/${entry} (run: node scripts/stage-sidecar-deps.mjs)`);
}
}
if (missing.length > 0 || unsafe.length > 0) {
console.error('[check-sidecar-resources] FATAL — sidecar resources are not release-safe:');
for (const m of missing) console.error(` - ${m}`);

View File

@@ -29,7 +29,6 @@
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { execFileSync } from 'node:child_process';
import { builtinModules } from 'node:module';
import { fileURLToPath } from 'node:url';
@@ -40,18 +39,6 @@ const resourcesDir = path.join(root, 'app', 'src-tauri', 'resources');
const metaFile = path.join(os.tmpdir(), 'waggle-sidecar-meta.json');
const stageDir = path.join(resourcesDir, 'node_modules');
const bundledNpmRuntimeDir = path.join(stageDir, 'waggle-node-runtime');
const hookRuntimeBuild = path.join(root, 'scripts', 'build-hook-runtime.mjs');
const HOOK_RUNTIME_ROOTS = new Set([
'@waggle/hive-mind-cli',
'@waggle/hive-mind-hooks-claude-code',
'@waggle/hive-mind-hooks-claude-desktop',
'@waggle/hive-mind-hooks-codex',
'@waggle/hive-mind-hooks-codex-desktop',
'@waggle/hive-mind-hooks-cursor',
'@waggle/hive-mind-hooks-hermes',
'@waggle/hive-mind-hooks-openclaw',
'waggle-memory-mcp',
]);
// These are loaded through computed require() calls, so esbuild's metafile
// cannot discover them even though installed desktop features require them.
const DYNAMIC_RUNTIME_ROOTS = new Set(['adm-zip']);
@@ -109,9 +96,8 @@ const RUNTIME_PRUNED_DIR_NAMES = new Set([
]);
const SOURCE_ARTIFACT_PATTERN = /(?:\.map|\.(?:[cm]?ts|tsx)|\.tsbuildinfo)$/i;
const WORKSPACE_RUNTIME_ENTRY_PATTERN = /^(?:dist|package\.json|licen[cs]e(?:\.(?:md|txt))?|notice(?:\.(?:md|txt))?)$/i;
const MANUAL_WORKSPACE_RUNTIME_TARGETS = new Map([
['@waggle/hive-mind-hooks-openclaw', ['dist/handler.bundle.cjs']],
]);
// Runtime files of a staged workspace package that its manifest does not name.
const MANUAL_WORKSPACE_RUNTIME_TARGETS = new Map();
const WINDOWS_1252_EXTRA_CODEPOINTS = new Set([
0x20ac, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030,
0x0160, 0x2039, 0x0152, 0x017d, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022,
@@ -168,6 +154,26 @@ function resolvePkgDir(name, fromDir) {
}
}
const WORKSPACE_DIRS = ['packages', 'apps'].flatMap((base) => {
const dir = path.join(root, base);
return fs.existsSync(dir) ? fs.readdirSync(dir).map((name) => path.join(dir, name)) : [];
});
/**
* Resolve a bundle root. npm keeps a package inside the workspace that depends
* on it when it cannot hoist it (mem0ai stays in packages/memory-mem0), so look
* in the workspaces when the repo root does not have it.
*/
function resolveRootPkgDir(name) {
const fromRoot = resolvePkgDir(name, root);
if (fromRoot) return fromRoot;
for (const workspaceDir of WORKSPACE_DIRS) {
const candidate = path.join(workspaceDir, 'node_modules', name);
if (fs.existsSync(path.join(candidate, 'package.json'))) return candidate;
}
return null;
}
let copiedPackages = 0;
let prunedRuntimeDirs = 0;
let prunedRuntimeFiles = 0;
@@ -243,16 +249,18 @@ function readManifest(pkgDir) {
*/
function stageClosure(rootNames) {
const processedDirs = new Set();
const queue = [...rootNames].map((name) => ({ name, fromDir: root }));
const queue = [...rootNames].map((name) => ({ name, fromDir: null }));
const absentRoots = [];
while (queue.length > 0) {
const { name, fromDir } = queue.shift();
if (SKIP.has(name)) continue;
const pkgDir = resolvePkgDir(name, fromDir);
const pkgDir = fromDir === null ? resolveRootPkgDir(name) : resolvePkgDir(name, fromDir);
if (!pkgDir) {
// Optional/absent (e.g. bufferutil, utf-8-validate, pg): the bundle
// guards these or never reaches them — nothing to stage.
if (fromDir === null) absentRoots.push(name);
continue;
}
const pkgKey = fs.realpathSync.native(pkgDir);
@@ -267,6 +275,9 @@ function stageClosure(rootNames) {
queue.push({ name: dep, fromDir: pkgDir });
}
}
if (absentRoots.length) {
console.log(`[stage-sidecar-deps] not installed, not staged: ${absentRoots.sort().join(', ')}`);
}
return processedDirs;
}
@@ -622,11 +633,6 @@ function preserveBundledNpmRuntime() {
// ── main ───────────────────────────────────────────────────────────
console.log(`[stage-sidecar-deps] Platform: ${platform}-${arch}`);
// These workspace packages are loaded by hook installers/external agents, not
// by the sidecar bundle itself, so the esbuild metafile cannot discover them.
// Build them explicitly before copying their production dependency closure.
execFileSync(process.execPath, [hookRuntimeBuild], { cwd: root, stdio: 'inherit' });
// Fresh stage dir each run so a removed dep never lingers in a stale bundle.
const preservedNpmRuntime = preserveBundledNpmRuntime();
try {
@@ -643,7 +649,6 @@ try {
const externals = readExternalPackages();
const runtimeRoots = new Set([
...externals,
...HOOK_RUNTIME_ROOTS,
...DYNAMIC_RUNTIME_ROOTS,
]);
const staged = [...runtimeRoots].filter((n) => !SKIP.has(n)).sort();