refactor: remove S3 file storage, part of team mode (P1.2)

Nothing created an S3FileStore outside team workspaces; team storage is
postponed with team mode (ADR-0003), so such workspaces use virtual storage.
@aws-sdk/client-s3 leaves core's dependencies.
This commit is contained in:
shumeiko.it
2026-10-24 12:09:05 +02:00
parent cb51ed0d28
commit 0cbde468a9
7 changed files with 4 additions and 564 deletions

1
package-lock.json generated
View File

@@ -24804,7 +24804,6 @@
"glob": "^13.0.6"
},
"devDependencies": {
"@aws-sdk/client-s3": "^3.1019.0",
"@types/better-sqlite3": "^7.6.13"
}
},

View File

@@ -25,7 +25,6 @@
"glob": "^13.0.6"
},
"devDependencies": {
"@aws-sdk/client-s3": "^3.1019.0",
"@types/better-sqlite3": "^7.6.13"
},
"license": "MIT"

View File

@@ -6,7 +6,6 @@
*
* - LocalFileStore: manages files in ~/.waggle/workspaces/{id}/files/ (virtual storage)
* - LinkedDirStore: reads/writes to workspace.directory (linked to external folder)
* - S3FileStore: MinIO/S3-backed storage for team/cloud deployments (lazy SDK import)
*
* All operations enforce path traversal protection — files cannot escape
* the workspace boundary.
@@ -15,9 +14,6 @@
import fs from 'node:fs';
import path from 'node:path';
import { glob } from 'glob';
// Type-only import: erased at compile time, so it does not force @aws-sdk/client-s3
// (a devDependency) into the runtime bundle for local-only users.
import type { S3Client } from '@aws-sdk/client-s3';
// ── Interface ───────────────────────────────────────────────────────
@@ -191,33 +187,6 @@ function containGlobMatches(root: string, matches: string[]): string[] {
});
}
/**
* Convert a `*`/`?` glob to a RegExp that is SAFE against catastrophic
* backtracking (ReDoS). Every regex metacharacter is escaped first, so the only
* specials left are the two wildcards we re-introduce (`.*`, `.`) — neither can
* nest a quantifier, so a hostile pattern like `(a+)+x` becomes a harmless
* literal. Used by S3FileStore.searchFiles where the pattern is caller-controlled.
* Unanchored (substring match) to preserve the prior behavior.
*/
function globToRegExp(pattern: string): RegExp {
const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const body = escaped.replace(/\\\*/g, '.*').replace(/\\\?/g, '.');
return new RegExp(body, 'i');
}
/**
* Reject an S3 key relative-path that would escape its workspace prefix. S3 keys
* are literal, but MinIO (path-style, filesystem-backed) can normalize `..`, so a
* `..` segment could cross into a sibling workspace's prefix. Makes the file
* header's "all operations enforce path traversal protection" true for S3 too.
*/
function assertSafeRelativeKey(relativePath: string): void {
const segments = relativePath.replace(/\\/g, '/').split('/');
if (segments.some(s => s === '..')) {
throw new Error(`Path traversal denied: ${relativePath}`);
}
}
function ensureDir(dirPath: string): void {
if (!fs.existsSync(dirPath)) {
fs.mkdirSync(dirPath, { recursive: true });
@@ -461,167 +430,10 @@ export class LinkedDirStore implements FileStore {
}
}
// ── S3FileStore (MinIO/S3 for team workspace storage) ───────────────
export interface S3Config {
endpoint: string;
bucket: string;
accessKey: string;
secretKey: string;
prefix: string; // e.g., "workspaces/{id}/"
region?: string;
}
export class S3FileStore implements FileStore {
private config: S3Config;
// Lazily constructed S3 client; the SDK is imported on first use to avoid
// bundling it for local-only users.
private client: S3Client | undefined;
constructor(config: S3Config) {
this.config = config;
}
private async getClient(): Promise<S3Client> {
if (this.client) return this.client;
const { S3Client } = await import('@aws-sdk/client-s3');
this.client = new S3Client({
endpoint: `http://${this.config.endpoint}`,
region: this.config.region ?? 'us-east-1',
credentials: {
accessKeyId: this.config.accessKey,
secretAccessKey: this.config.secretKey,
},
forcePathStyle: true, // Required for MinIO
});
return this.client;
}
getRootPath(): string { return `s3://${this.config.bucket}/${this.config.prefix}`; }
getStorageType(): 'virtual' | 'linked' { return 'virtual'; }
async readFile(relativePath: string): Promise<Buffer> {
assertSafeRelativeKey(relativePath);
const client = await this.getClient();
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
const key = this.config.prefix + relativePath;
const response = await client.send(new GetObjectCommand({
Bucket: this.config.bucket,
Key: key,
}));
const body = response.Body;
if (!body) throw new Error(`S3 object has no body: ${key}`);
const chunks: Uint8Array[] = [];
// In Node.js the S3 streaming body is an async-iterable readable stream.
for await (const chunk of body as AsyncIterable<Uint8Array>) chunks.push(chunk);
return Buffer.concat(chunks);
}
async writeFile(relativePath: string, content: Buffer | string): Promise<void> {
assertSafeRelativeKey(relativePath);
const client = await this.getClient();
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
const key = this.config.prefix + relativePath;
await client.send(new PutObjectCommand({
Bucket: this.config.bucket,
Key: key,
Body: typeof content === 'string' ? Buffer.from(content) : content,
}));
}
async deleteFile(relativePath: string): Promise<void> {
assertSafeRelativeKey(relativePath);
const client = await this.getClient();
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3');
const key = this.config.prefix + relativePath;
await client.send(new DeleteObjectCommand({
Bucket: this.config.bucket,
Key: key,
}));
}
async listFiles(directory?: string): Promise<FileEntry[]> {
if (directory) assertSafeRelativeKey(directory);
const client = await this.getClient();
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
const pfx = this.config.prefix + (directory ? directory + '/' : '');
const response = await client.send(new ListObjectsV2Command({
Bucket: this.config.bucket,
Prefix: pfx,
Delimiter: '/',
}));
const entries: FileEntry[] = [];
for (const obj of response.Contents ?? []) {
if (!obj.Key) continue;
const relPath = obj.Key.slice(this.config.prefix.length);
entries.push({
name: relPath.split('/').pop() ?? relPath,
path: relPath,
size: obj.Size ?? 0,
modified: obj.LastModified?.toISOString() ?? '',
isDirectory: false,
});
}
for (const cpfx of response.CommonPrefixes ?? []) {
if (!cpfx.Prefix) continue;
const relPath = cpfx.Prefix.slice(this.config.prefix.length).replace(/\/$/, '');
entries.push({
name: relPath.split('/').pop() ?? relPath,
path: relPath,
size: 0,
modified: '',
isDirectory: true,
});
}
return entries;
}
async searchFiles(pattern: string): Promise<FileEntry[]> {
const all = await this.listFiles();
const regex = globToRegExp(pattern); // ReDoS-safe glob→regex (pattern is caller-controlled)
return all.filter(f => regex.test(f.path) || regex.test(f.name));
}
async moveFile(from: string, to: string): Promise<void> {
assertSafeRelativeKey(from);
assertSafeRelativeKey(to);
const client = await this.getClient();
const { CopyObjectCommand, DeleteObjectCommand } = await import('@aws-sdk/client-s3');
const fromKey = this.config.prefix + from;
const toKey = this.config.prefix + to;
await client.send(new CopyObjectCommand({
Bucket: this.config.bucket,
CopySource: `${this.config.bucket}/${fromKey}`,
Key: toKey,
}));
await client.send(new DeleteObjectCommand({
Bucket: this.config.bucket,
Key: fromKey,
}));
}
async getStorageInfo(): Promise<StorageInfo> {
const client = await this.getClient();
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
const response = await client.send(new ListObjectsV2Command({
Bucket: this.config.bucket,
Prefix: this.config.prefix,
}));
let usedBytes = 0;
let fileCount = 0;
for (const obj of response.Contents ?? []) {
usedBytes += obj.Size ?? 0;
fileCount++;
}
return { usedBytes, fileCount, storageType: 'virtual' };
}
}
// ── Factory ─────────────────────────────────────────────────────────
/**
* Create the appropriate FileStore for a workspace.
* If s3Config is provided, use S3FileStore (team/cloud deployment).
* If the workspace has a linked directory, use LinkedDirStore.
* Otherwise, use LocalFileStore (virtual managed storage).
*/
@@ -629,9 +441,7 @@ export function createFileStore(
dataDir: string,
workspaceId: string,
linkedDirectory?: string,
s3Config?: S3Config,
): FileStore {
if (s3Config) return new S3FileStore(s3Config);
if (linkedDirectory && fs.existsSync(linkedDirectory)) {
return new LinkedDirStore(linkedDirectory);
}

View File

@@ -162,5 +162,5 @@ export {
SkillHashStore, computeSkillHash, SKILL_HASHES_TABLE_SQL,
type SkillHash,
} from './skill-hashes.js';
export { createFileStore, LocalFileStore, LinkedDirStore, S3FileStore, isSensitiveFilePath, type FileStore, type FileEntry, type StorageInfo, type S3Config } from './file-store.js';
export { createFileStore, LocalFileStore, LinkedDirStore, isSensitiveFilePath, type FileStore, type FileEntry, type StorageInfo } from './file-store.js';
export { FileIndexer, MAX_CONTENT_BYTES, type FileIndexRow, type FileIndexResult } from './file-indexer.js';

View File

@@ -1,266 +0,0 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { S3FileStore, type S3Config } from '../src/file-store.js';
// ── Mock @aws-sdk/client-s3 ────────────────────────────────────────
const mockSend = vi.fn();
type CommandInput = Record<string, unknown>;
class MockS3Client {
send = mockSend;
constructor(_config: unknown) {}
}
vi.mock('@aws-sdk/client-s3', () => {
return {
S3Client: MockS3Client,
GetObjectCommand: class { [k: string]: unknown; _type = 'GetObject'; constructor(input: CommandInput) { Object.assign(this, input); } },
PutObjectCommand: class { [k: string]: unknown; _type = 'PutObject'; constructor(input: CommandInput) { Object.assign(this, input); } },
DeleteObjectCommand: class { [k: string]: unknown; _type = 'DeleteObject'; constructor(input: CommandInput) { Object.assign(this, input); } },
ListObjectsV2Command: class { [k: string]: unknown; _type = 'ListObjects'; constructor(input: CommandInput) { Object.assign(this, input); } },
CopyObjectCommand: class { [k: string]: unknown; _type = 'CopyObject'; constructor(input: CommandInput) { Object.assign(this, input); } },
};
});
// ── Test setup ──────────────────────────────────────────────────────
const testConfig: S3Config = {
endpoint: 'minio:9000',
bucket: 'waggle-files',
accessKey: 'waggle',
secretKey: 'waggle_s3_prod',
prefix: 'workspaces/ws-123/',
};
describe('S3FileStore', () => {
let store: S3FileStore;
beforeEach(() => {
vi.clearAllMocks();
store = new S3FileStore(testConfig);
});
// ── Meta ────────────────────────────────────────────────────────
it('getStorageType returns virtual', () => {
expect(store.getStorageType()).toBe('virtual');
});
it('getRootPath returns s3:// URL', () => {
expect(store.getRootPath()).toBe('s3://waggle-files/workspaces/ws-123/');
});
// ── writeFile ───────────────────────────────────────────────────
it('writeFile sends PutObjectCommand with correct bucket and key', async () => {
mockSend.mockResolvedValueOnce({});
await store.writeFile('docs/notes.md', 'hello world');
expect(mockSend).toHaveBeenCalledTimes(1);
const cmd = mockSend.mock.calls[0][0];
expect(cmd._type).toBe('PutObject');
expect(cmd.Bucket).toBe('waggle-files');
expect(cmd.Key).toBe('workspaces/ws-123/docs/notes.md');
expect(cmd.Body).toEqual(Buffer.from('hello world'));
});
it('writeFile accepts Buffer content', async () => {
mockSend.mockResolvedValueOnce({});
const buf = Buffer.from([0x89, 0x50, 0x4e, 0x47]);
await store.writeFile('image.png', buf);
const cmd = mockSend.mock.calls[0][0];
expect(cmd.Body).toBe(buf);
});
// ── readFile ────────────────────────────────────────────────────
it('readFile sends GetObjectCommand and returns Buffer', async () => {
const chunks = [Buffer.from('chunk1'), Buffer.from('chunk2')];
const asyncIterable = {
async *[Symbol.asyncIterator]() {
for (const c of chunks) yield c;
},
};
mockSend.mockResolvedValueOnce({ Body: asyncIterable });
const result = await store.readFile('docs/notes.md');
expect(mockSend).toHaveBeenCalledTimes(1);
const cmd = mockSend.mock.calls[0][0];
expect(cmd._type).toBe('GetObject');
expect(cmd.Bucket).toBe('waggle-files');
expect(cmd.Key).toBe('workspaces/ws-123/docs/notes.md');
expect(Buffer.isBuffer(result)).toBe(true);
expect(result.toString()).toBe('chunk1chunk2');
});
// ── deleteFile ──────────────────────────────────────────────────
it('deleteFile sends DeleteObjectCommand', async () => {
mockSend.mockResolvedValueOnce({});
await store.deleteFile('old-file.txt');
expect(mockSend).toHaveBeenCalledTimes(1);
const cmd = mockSend.mock.calls[0][0];
expect(cmd._type).toBe('DeleteObject');
expect(cmd.Bucket).toBe('waggle-files');
expect(cmd.Key).toBe('workspaces/ws-123/old-file.txt');
});
// ── listFiles ───────────────────────────────────────────────────
it('listFiles sends ListObjectsV2Command and returns FileEntry[]', async () => {
mockSend.mockResolvedValueOnce({
Contents: [
{ Key: 'workspaces/ws-123/readme.md', Size: 1024, LastModified: new Date('2025-06-01T00:00:00Z') },
{ Key: 'workspaces/ws-123/src/index.ts', Size: 512, LastModified: new Date('2025-06-02T00:00:00Z') },
],
CommonPrefixes: [
{ Prefix: 'workspaces/ws-123/docs/' },
],
});
const entries = await store.listFiles();
expect(mockSend).toHaveBeenCalledTimes(1);
const cmd = mockSend.mock.calls[0][0];
expect(cmd._type).toBe('ListObjects');
expect(cmd.Bucket).toBe('waggle-files');
expect(cmd.Prefix).toBe('workspaces/ws-123/');
expect(cmd.Delimiter).toBe('/');
expect(entries).toHaveLength(3);
// File entries
expect(entries[0]).toEqual({
name: 'readme.md',
path: 'readme.md',
size: 1024,
modified: '2025-06-01T00:00:00.000Z',
isDirectory: false,
});
expect(entries[1]).toEqual({
name: 'index.ts',
path: 'src/index.ts',
size: 512,
modified: '2025-06-02T00:00:00.000Z',
isDirectory: false,
});
// Directory entry
expect(entries[2]).toEqual({
name: 'docs',
path: 'docs',
size: 0,
modified: '',
isDirectory: true,
});
});
it('listFiles with directory argument appends to prefix', async () => {
mockSend.mockResolvedValueOnce({ Contents: [], CommonPrefixes: [] });
await store.listFiles('src');
const cmd = mockSend.mock.calls[0][0];
expect(cmd.Prefix).toBe('workspaces/ws-123/src/');
});
it('listFiles handles empty response', async () => {
mockSend.mockResolvedValueOnce({});
const entries = await store.listFiles();
expect(entries).toEqual([]);
});
// ── moveFile ────────────────────────────────────────────────────
it('moveFile copies then deletes', async () => {
mockSend.mockResolvedValueOnce({}); // CopyObject
mockSend.mockResolvedValueOnce({}); // DeleteObject
await store.moveFile('old.txt', 'new.txt');
expect(mockSend).toHaveBeenCalledTimes(2);
const copyCmd = mockSend.mock.calls[0][0];
expect(copyCmd._type).toBe('CopyObject');
expect(copyCmd.CopySource).toBe('waggle-files/workspaces/ws-123/old.txt');
expect(copyCmd.Key).toBe('workspaces/ws-123/new.txt');
const deleteCmd = mockSend.mock.calls[1][0];
expect(deleteCmd._type).toBe('DeleteObject');
expect(deleteCmd.Key).toBe('workspaces/ws-123/old.txt');
});
// ── searchFiles ─────────────────────────────────────────────────
it('searchFiles filters listFiles results by pattern', async () => {
mockSend.mockResolvedValueOnce({
Contents: [
{ Key: 'workspaces/ws-123/readme.md', Size: 100, LastModified: new Date() },
{ Key: 'workspaces/ws-123/notes.txt', Size: 200, LastModified: new Date() },
{ Key: 'workspaces/ws-123/data.md', Size: 50, LastModified: new Date() },
],
CommonPrefixes: [],
});
const results = await store.searchFiles('*.md');
expect(results).toHaveLength(2);
expect(results[0].name).toBe('readme.md');
expect(results[1].name).toBe('data.md');
});
it('searchFiles is ReDoS-safe with a hostile regex pattern', async () => {
mockSend.mockResolvedValueOnce({
Contents: [{ Key: 'workspaces/ws-123/' + 'a'.repeat(40) + 'X', Size: 1, LastModified: new Date() }],
CommonPrefixes: [],
});
const start = Date.now();
// Under the old `new RegExp(pattern…)` this built /(a+)+$/i and catastrophically
// backtracked on the key. Now the metachars are escaped → a literal match → instant.
const results = await store.searchFiles('(a+)+$');
expect(Date.now() - start).toBeLessThan(1000);
expect(results).toHaveLength(0); // matched literally; no key contains "(a+)+$"
});
// ── key traversal protection ────────────────────────────────────
it('rejects ../ traversal in every op BEFORE any S3 call', async () => {
await expect(store.readFile('../ws-456/secret')).rejects.toThrow(/traversal denied/i);
await expect(store.writeFile('../ws-456/x', 'y')).rejects.toThrow(/traversal denied/i);
await expect(store.deleteFile('../../etc/passwd')).rejects.toThrow(/traversal denied/i);
await expect(store.moveFile('ok', '../ws-456/b')).rejects.toThrow(/traversal denied/i);
await expect(store.listFiles('../ws-456')).rejects.toThrow(/traversal denied/i);
expect(mockSend).not.toHaveBeenCalled();
});
it('allows a normal nested key', async () => {
mockSend.mockResolvedValueOnce({});
await store.writeFile('docs/sub/file.md', 'x');
expect(mockSend.mock.calls[0][0].Key).toBe('workspaces/ws-123/docs/sub/file.md');
});
// ── getStorageInfo ──────────────────────────────────────────────
it('getStorageInfo sums sizes from S3 listing', async () => {
mockSend.mockResolvedValueOnce({
Contents: [
{ Key: 'workspaces/ws-123/a.txt', Size: 100 },
{ Key: 'workspaces/ws-123/b.txt', Size: 250 },
{ Key: 'workspaces/ws-123/c.txt', Size: 50 },
],
});
const info = await store.getStorageInfo();
expect(info).toEqual({
usedBytes: 400,
fileCount: 3,
storageType: 'virtual',
});
});
});

View File

@@ -1,13 +1,10 @@
import path from 'node:path';
import { FsStorageProvider } from './fs-provider.js';
import { S3StorageProvider } from './s3-provider.js';
import type { StorageProvider } from './types.js';
import type { S3Config } from '@waggle/core';
export type { StorageProvider, FileEntry } from './types.js';
export { STANDARD_DIRS, MAX_UPLOAD_SIZE } from './types.js';
export { FsStorageProvider } from './fs-provider.js';
export { S3StorageProvider } from './s3-provider.js';
export { safePath, toRelativePath } from './security.js';
interface WorkspaceLike {
@@ -22,7 +19,8 @@ interface WorkspaceLike {
*
* - virtual: files in ~/.waggle/workspaces/{id}/files/ (app-managed)
* - local: user-specified directory on their machine (like Claude Code)
* - team: MinIO/S3 bucket shared across team members
* - team: shared team storage is part of team mode, which the community
* edition does not ship (ADR-0003); such workspaces use virtual storage
*/
export function getStorageProvider(workspace: WorkspaceLike, dataDir: string): StorageProvider {
switch (workspace.storageType) {
@@ -33,37 +31,7 @@ export function getStorageProvider(workspace: WorkspaceLike, dataDir: string): S
return new FsStorageProvider(workspace.storagePath, { denySensitive: true });
}
case 'team': {
const cfg = workspace.storageConfig;
// If S3 config is provided (team server sets this), use S3
if (cfg?.endpoint && cfg?.bucket && cfg?.accessKey && cfg?.secretKey) {
const s3Config: S3Config = {
endpoint: cfg.endpoint as string,
bucket: cfg.bucket as string,
accessKey: cfg.accessKey as string,
secretKey: cfg.secretKey as string,
prefix: cfg.prefix as string ?? `teams/${workspace.id}/`,
region: cfg.region as string | undefined,
};
return new S3StorageProvider(s3Config);
}
// Fallback: env-based MinIO config (dev docker-compose)
const env = process.env;
if (env.MINIO_ENDPOINT && env.MINIO_BUCKET) {
const s3Config: S3Config = {
endpoint: env.MINIO_ENDPOINT,
bucket: env.MINIO_BUCKET,
accessKey: env.MINIO_ACCESS_KEY ?? 'waggle',
secretKey: env.MINIO_SECRET_KEY ?? 'waggle_s3_dev',
prefix: `teams/${workspace.id}/`,
};
return new S3StorageProvider(s3Config);
}
// No S3 config — fall back to virtual
const virtualRoot = path.join(dataDir, 'workspaces', workspace.id, 'files');
return new FsStorageProvider(virtualRoot);
}
case 'team':
case 'virtual':
default: {
const virtualRoot = path.join(dataDir, 'workspaces', workspace.id, 'files');

View File

@@ -1,70 +0,0 @@
/**
* S3StorageProvider — adapts the core S3FileStore to the StorageProvider interface.
* Used for team workspace files backed by MinIO/S3.
*/
import { S3FileStore, type S3Config, type FileEntry as CoreFileEntry } from '@waggle/core';
import type { StorageProvider, FileEntry } from './types.js';
function coreToProvider(f: CoreFileEntry): FileEntry {
return {
name: f.name,
path: f.path,
type: f.isDirectory ? 'directory' : 'file',
size: f.size,
modifiedAt: f.modified,
};
}
export class S3StorageProvider implements StorageProvider {
private store: S3FileStore;
constructor(config: S3Config) {
this.store = new S3FileStore(config);
}
async list(dirPath: string): Promise<FileEntry[]> {
const files = await this.store.listFiles(dirPath);
return files.map(coreToProvider);
}
async read(filePath: string): Promise<Buffer> {
return this.store.readFile(filePath);
}
async write(filePath: string, data: Buffer): Promise<FileEntry> {
await this.store.writeFile(filePath, data);
return { name: filePath.split('/').pop() ?? filePath, path: filePath, type: 'file', size: data.length };
}
async delete(targetPath: string): Promise<void> {
await this.store.deleteFile(targetPath);
}
async move(from: string, to: string): Promise<FileEntry> {
await this.store.moveFile(from, to);
return { name: to.split('/').pop() ?? to, path: to, type: 'file' };
}
async copy(from: string, to: string): Promise<FileEntry> {
// S3 copy: read + write (S3FileStore doesn't have native copy)
const data = await this.store.readFile(from);
await this.store.writeFile(to, data);
return { name: to.split('/').pop() ?? to, path: to, type: 'file', size: data.length };
}
async mkdir(dirPath: string): Promise<FileEntry> {
// S3 doesn't have real directories — write a zero-byte marker
await this.store.writeFile(dirPath + '/.keep', Buffer.alloc(0));
return { name: dirPath.split('/').pop() ?? dirPath, path: dirPath, type: 'directory' };
}
async exists(targetPath: string): Promise<boolean> {
try {
await this.store.readFile(targetPath);
return true;
} catch {
return false;
}
}
}