114 lines
4.3 KiB
TypeScript
114 lines
4.3 KiB
TypeScript
import { describe, it, expect } from 'vitest';
|
|
import { chunkText } from '../../src/mind/chunker.js';
|
|
|
|
/**
|
|
* D1 (oss-drift triage, 2026-06-11) — chunker unit tests for the
|
|
* reverse-ported OSS hive-mind semantic chunker (paragraph-first,
|
|
* sentence-fallback, max 2000 chars, 200 overlap).
|
|
*/
|
|
|
|
/** A single paragraph of `sentences` short sentences (~55 chars each). */
|
|
function para(topic: string, sentences: number): string {
|
|
return Array.from(
|
|
{ length: sentences },
|
|
(_, i) => `The ${topic} system processes record number ${i} every day.`
|
|
).join(' ');
|
|
}
|
|
|
|
describe('chunkText (D1 chunk-level retrieval)', () => {
|
|
it('returns [] for empty input', () => {
|
|
expect(chunkText('')).toEqual([]);
|
|
});
|
|
|
|
it('returns short content as a single chunk with full-span offsets', () => {
|
|
const text = 'A short memory frame about the deploy checklist.';
|
|
const chunks = chunkText(text);
|
|
expect(chunks).toHaveLength(1);
|
|
expect(chunks[0].text).toBe(text);
|
|
expect(chunks[0].charStart).toBe(0);
|
|
expect(chunks[0].charEnd).toBe(text.length);
|
|
});
|
|
|
|
it('treats content at exactly minChunkChars as a single chunk', () => {
|
|
const text = 'x'.repeat(1500);
|
|
const chunks = chunkText(text);
|
|
expect(chunks).toHaveLength(1);
|
|
});
|
|
|
|
it('splits multi-paragraph content on blank lines (paragraph-first)', () => {
|
|
const p1 = para('alpha', 30); // ~1650 chars
|
|
const p2 = para('beta', 30); // ~1650 chars
|
|
const text = `${p1}\n\n${p2}`;
|
|
|
|
const chunks = chunkText(text);
|
|
// p1 + p2 can't pack into one 2000-char chunk → 2 chunks.
|
|
expect(chunks).toHaveLength(2);
|
|
expect(chunks[0].text).toBe(p1);
|
|
expect(chunks[0].charStart).toBe(0);
|
|
expect(chunks[0].charEnd).toBe(p1.length);
|
|
// Second chunk's PRIMARY span is p2 (overlap never alters offsets).
|
|
expect(chunks[1].charStart).toBe(p1.length + 2);
|
|
expect(chunks[1].charEnd).toBe(text.length);
|
|
expect(chunks[1].text).toContain('beta');
|
|
});
|
|
|
|
it('falls back to sentence splitting for a single oversize paragraph', () => {
|
|
// One paragraph, no blank lines, > maxChars → must sub-split on sentences.
|
|
const text = para('gamma', 60); // ~3300 chars, single paragraph
|
|
const chunks = chunkText(text);
|
|
expect(chunks.length).toBeGreaterThan(1);
|
|
for (const c of chunks) {
|
|
// 2000 maxChars + up to 200 prepended overlap + 1 joining newline.
|
|
expect(c.text.length).toBeLessThanOrEqual(2000 + 200 + 1);
|
|
}
|
|
// Sentence boundaries respected: each chunk's primary span starts at a
|
|
// sentence start within the source text.
|
|
expect(text.slice(chunks[1].charStart)).toMatch(/^The gamma system/);
|
|
});
|
|
|
|
it('hard-cuts a single sentence longer than maxChars', () => {
|
|
const text = 'y'.repeat(4500); // no sentence boundaries at all
|
|
const chunks = chunkText(text, { overlapChars: 0 });
|
|
expect(chunks.length).toBeGreaterThanOrEqual(3);
|
|
expect(chunks[0].text).toBe('y'.repeat(2000));
|
|
expect(chunks[0].charStart).toBe(0);
|
|
expect(chunks[0].charEnd).toBe(2000);
|
|
expect(chunks[1].charStart).toBe(2000);
|
|
});
|
|
|
|
it('prepends the previous chunk tail as overlap (offsets untouched)', () => {
|
|
const p1 = para('delta', 30);
|
|
const p2 = para('epsilon', 30);
|
|
const text = `${p1}\n\n${p2}`;
|
|
|
|
const chunks = chunkText(text, { overlapChars: 200 });
|
|
expect(chunks).toHaveLength(2);
|
|
const prevTail = chunks[0].text.slice(-200);
|
|
expect(chunks[1].text.startsWith(prevTail)).toBe(true);
|
|
// Offsets still describe the primary span only.
|
|
expect(chunks[1].charStart).toBe(p1.length + 2);
|
|
});
|
|
|
|
it('overlapChars: 0 makes chunk text exactly equal its source span', () => {
|
|
const p1 = para('zeta', 30);
|
|
const p2 = para('eta', 30);
|
|
const text = `${p1}\n\n${p2}`;
|
|
|
|
const chunks = chunkText(text, { overlapChars: 0 });
|
|
expect(chunks).toHaveLength(2);
|
|
for (const c of chunks) {
|
|
expect(c.text).toBe(text.slice(c.charStart, c.charEnd));
|
|
}
|
|
});
|
|
|
|
it('returns at least one chunk for whitespace-padded long content', () => {
|
|
const text = `${para('theta', 30)}\n\n \n\n${para('iota', 30)}`;
|
|
const chunks = chunkText(text);
|
|
expect(chunks.length).toBeGreaterThanOrEqual(2);
|
|
// Blank/whitespace-only paragraphs never become chunks.
|
|
for (const c of chunks) {
|
|
expect(c.text.trim().length).toBeGreaterThan(0);
|
|
}
|
|
});
|
|
});
|