refactor: remove S3 file storage, part of team mode (P1.2)
Nothing created an S3FileStore outside team workspaces; team storage is postponed with team mode (ADR-0003), so such workspaces use virtual storage. @aws-sdk/client-s3 leaves core's dependencies.
This commit is contained in:
@@ -6,7 +6,6 @@
|
||||
*
|
||||
* - LocalFileStore: manages files in ~/.waggle/workspaces/{id}/files/ (virtual storage)
|
||||
* - LinkedDirStore: reads/writes to workspace.directory (linked to external folder)
|
||||
* - S3FileStore: MinIO/S3-backed storage for team/cloud deployments (lazy SDK import)
|
||||
*
|
||||
* All operations enforce path traversal protection — files cannot escape
|
||||
* the workspace boundary.
|
||||
@@ -15,9 +14,6 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { glob } from 'glob';
|
||||
// Type-only import: erased at compile time, so it does not force @aws-sdk/client-s3
|
||||
// (a devDependency) into the runtime bundle for local-only users.
|
||||
import type { S3Client } from '@aws-sdk/client-s3';
|
||||
|
||||
// ── Interface ───────────────────────────────────────────────────────
|
||||
|
||||
@@ -191,33 +187,6 @@ function containGlobMatches(root: string, matches: string[]): string[] {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a `*`/`?` glob to a RegExp that is SAFE against catastrophic
|
||||
* backtracking (ReDoS). Every regex metacharacter is escaped first, so the only
|
||||
* specials left are the two wildcards we re-introduce (`.*`, `.`) — neither can
|
||||
* nest a quantifier, so a hostile pattern like `(a+)+x` becomes a harmless
|
||||
* literal. Used by S3FileStore.searchFiles where the pattern is caller-controlled.
|
||||
* Unanchored (substring match) to preserve the prior behavior.
|
||||
*/
|
||||
function globToRegExp(pattern: string): RegExp {
|
||||
const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const body = escaped.replace(/\\\*/g, '.*').replace(/\\\?/g, '.');
|
||||
return new RegExp(body, 'i');
|
||||
}
|
||||
|
||||
/**
|
||||
* Reject an S3 key relative-path that would escape its workspace prefix. S3 keys
|
||||
* are literal, but MinIO (path-style, filesystem-backed) can normalize `..`, so a
|
||||
* `..` segment could cross into a sibling workspace's prefix. Makes the file
|
||||
* header's "all operations enforce path traversal protection" true for S3 too.
|
||||
*/
|
||||
function assertSafeRelativeKey(relativePath: string): void {
|
||||
const segments = relativePath.replace(/\\/g, '/').split('/');
|
||||
if (segments.some(s => s === '..')) {
|
||||
throw new Error(`Path traversal denied: ${relativePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
function ensureDir(dirPath: string): void {
|
||||
if (!fs.existsSync(dirPath)) {
|
||||
fs.mkdirSync(dirPath, { recursive: true });
|
||||
@@ -461,167 +430,10 @@ export class LinkedDirStore implements FileStore {
|
||||
}
|
||||
}
|
||||
|
||||
// ── S3FileStore (MinIO/S3 for team workspace storage) ───────────────
|
||||
|
||||
export interface S3Config {
|
||||
endpoint: string;
|
||||
bucket: string;
|
||||
accessKey: string;
|
||||
secretKey: string;
|
||||
prefix: string; // e.g., "workspaces/{id}/"
|
||||
region?: string;
|
||||
}
|
||||
|
||||
export class S3FileStore implements FileStore {
|
||||
private config: S3Config;
|
||||
// Lazily constructed S3 client; the SDK is imported on first use to avoid
|
||||
// bundling it for local-only users.
|
||||
private client: S3Client | undefined;
|
||||
|
||||
constructor(config: S3Config) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
private async getClient(): Promise<S3Client> {
|
||||
if (this.client) return this.client;
|
||||
const { S3Client } = await import('@aws-sdk/client-s3');
|
||||
this.client = new S3Client({
|
||||
endpoint: `http://${this.config.endpoint}`,
|
||||
region: this.config.region ?? 'us-east-1',
|
||||
credentials: {
|
||||
accessKeyId: this.config.accessKey,
|
||||
secretAccessKey: this.config.secretKey,
|
||||
},
|
||||
forcePathStyle: true, // Required for MinIO
|
||||
});
|
||||
return this.client;
|
||||
}
|
||||
|
||||
getRootPath(): string { return `s3://${this.config.bucket}/${this.config.prefix}`; }
|
||||
getStorageType(): 'virtual' | 'linked' { return 'virtual'; }
|
||||
|
||||
async readFile(relativePath: string): Promise<Buffer> {
|
||||
assertSafeRelativeKey(relativePath);
|
||||
const client = await this.getClient();
|
||||
const { GetObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
const key = this.config.prefix + relativePath;
|
||||
const response = await client.send(new GetObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
Key: key,
|
||||
}));
|
||||
const body = response.Body;
|
||||
if (!body) throw new Error(`S3 object has no body: ${key}`);
|
||||
const chunks: Uint8Array[] = [];
|
||||
// In Node.js the S3 streaming body is an async-iterable readable stream.
|
||||
for await (const chunk of body as AsyncIterable<Uint8Array>) chunks.push(chunk);
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
async writeFile(relativePath: string, content: Buffer | string): Promise<void> {
|
||||
assertSafeRelativeKey(relativePath);
|
||||
const client = await this.getClient();
|
||||
const { PutObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
const key = this.config.prefix + relativePath;
|
||||
await client.send(new PutObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
Key: key,
|
||||
Body: typeof content === 'string' ? Buffer.from(content) : content,
|
||||
}));
|
||||
}
|
||||
|
||||
async deleteFile(relativePath: string): Promise<void> {
|
||||
assertSafeRelativeKey(relativePath);
|
||||
const client = await this.getClient();
|
||||
const { DeleteObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
const key = this.config.prefix + relativePath;
|
||||
await client.send(new DeleteObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
Key: key,
|
||||
}));
|
||||
}
|
||||
|
||||
async listFiles(directory?: string): Promise<FileEntry[]> {
|
||||
if (directory) assertSafeRelativeKey(directory);
|
||||
const client = await this.getClient();
|
||||
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
|
||||
const pfx = this.config.prefix + (directory ? directory + '/' : '');
|
||||
const response = await client.send(new ListObjectsV2Command({
|
||||
Bucket: this.config.bucket,
|
||||
Prefix: pfx,
|
||||
Delimiter: '/',
|
||||
}));
|
||||
const entries: FileEntry[] = [];
|
||||
for (const obj of response.Contents ?? []) {
|
||||
if (!obj.Key) continue;
|
||||
const relPath = obj.Key.slice(this.config.prefix.length);
|
||||
entries.push({
|
||||
name: relPath.split('/').pop() ?? relPath,
|
||||
path: relPath,
|
||||
size: obj.Size ?? 0,
|
||||
modified: obj.LastModified?.toISOString() ?? '',
|
||||
isDirectory: false,
|
||||
});
|
||||
}
|
||||
for (const cpfx of response.CommonPrefixes ?? []) {
|
||||
if (!cpfx.Prefix) continue;
|
||||
const relPath = cpfx.Prefix.slice(this.config.prefix.length).replace(/\/$/, '');
|
||||
entries.push({
|
||||
name: relPath.split('/').pop() ?? relPath,
|
||||
path: relPath,
|
||||
size: 0,
|
||||
modified: '',
|
||||
isDirectory: true,
|
||||
});
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
async searchFiles(pattern: string): Promise<FileEntry[]> {
|
||||
const all = await this.listFiles();
|
||||
const regex = globToRegExp(pattern); // ReDoS-safe glob→regex (pattern is caller-controlled)
|
||||
return all.filter(f => regex.test(f.path) || regex.test(f.name));
|
||||
}
|
||||
|
||||
async moveFile(from: string, to: string): Promise<void> {
|
||||
assertSafeRelativeKey(from);
|
||||
assertSafeRelativeKey(to);
|
||||
const client = await this.getClient();
|
||||
const { CopyObjectCommand, DeleteObjectCommand } = await import('@aws-sdk/client-s3');
|
||||
const fromKey = this.config.prefix + from;
|
||||
const toKey = this.config.prefix + to;
|
||||
await client.send(new CopyObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
CopySource: `${this.config.bucket}/${fromKey}`,
|
||||
Key: toKey,
|
||||
}));
|
||||
await client.send(new DeleteObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
Key: fromKey,
|
||||
}));
|
||||
}
|
||||
|
||||
async getStorageInfo(): Promise<StorageInfo> {
|
||||
const client = await this.getClient();
|
||||
const { ListObjectsV2Command } = await import('@aws-sdk/client-s3');
|
||||
const response = await client.send(new ListObjectsV2Command({
|
||||
Bucket: this.config.bucket,
|
||||
Prefix: this.config.prefix,
|
||||
}));
|
||||
let usedBytes = 0;
|
||||
let fileCount = 0;
|
||||
for (const obj of response.Contents ?? []) {
|
||||
usedBytes += obj.Size ?? 0;
|
||||
fileCount++;
|
||||
}
|
||||
return { usedBytes, fileCount, storageType: 'virtual' };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Factory ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Create the appropriate FileStore for a workspace.
|
||||
* If s3Config is provided, use S3FileStore (team/cloud deployment).
|
||||
* If the workspace has a linked directory, use LinkedDirStore.
|
||||
* Otherwise, use LocalFileStore (virtual managed storage).
|
||||
*/
|
||||
@@ -629,9 +441,7 @@ export function createFileStore(
|
||||
dataDir: string,
|
||||
workspaceId: string,
|
||||
linkedDirectory?: string,
|
||||
s3Config?: S3Config,
|
||||
): FileStore {
|
||||
if (s3Config) return new S3FileStore(s3Config);
|
||||
if (linkedDirectory && fs.existsSync(linkedDirectory)) {
|
||||
return new LinkedDirStore(linkedDirectory);
|
||||
}
|
||||
|
||||
@@ -162,5 +162,5 @@ export {
|
||||
SkillHashStore, computeSkillHash, SKILL_HASHES_TABLE_SQL,
|
||||
type SkillHash,
|
||||
} from './skill-hashes.js';
|
||||
export { createFileStore, LocalFileStore, LinkedDirStore, S3FileStore, isSensitiveFilePath, type FileStore, type FileEntry, type StorageInfo, type S3Config } from './file-store.js';
|
||||
export { createFileStore, LocalFileStore, LinkedDirStore, isSensitiveFilePath, type FileStore, type FileEntry, type StorageInfo } from './file-store.js';
|
||||
export { FileIndexer, MAX_CONTENT_BYTES, type FileIndexRow, type FileIndexResult } from './file-indexer.js';
|
||||
|
||||
Reference in New Issue
Block a user