feat(#90): integrate HierarchyBuilder into ingestion pipeline

ingestText() now calls buildHierarchy() after page creation to produce
the full Books→Volumes→Shelves hierarchy on every ingest call.
IngestResult extended with books[], volumes[], shelves[] fields.

Closes #90

Co-authored-by: devlux76 <[email protected]>

Author: Copilot

lib/hippocampus/Ingest.ts

@@ -1,12 +1,12 @@
-import type { Book, MetadataStore, VectorStore } from "../core/types";
+import type { Book, MetadataStore, Volume, Shelf, VectorStore } from "../core/types";
 import type { ModelProfile } from "../core/ModelProfile";
 import { hashText } from "../core/crypto/hash";
 import type { KeyPair } from "../core/crypto/sign";
 import { EmbeddingRunner } from "../embeddings/EmbeddingRunner";
 import { chunkText } from "./Chunker";
 import { buildPage } from "./PageBuilder";
-import { runPromotionSweep } from "../core/SalienceEngine";
 import { insertSemanticNeighbors } from "./FastNeighborInsert";
+import { buildHierarchy } from "./HierarchyBuilder";
 
 export interface IngestOptions {
   modelProfile: ModelProfile;
@@ -19,46 +19,15 @@ export interface IngestOptions {
 
 export interface IngestResult {
   pages: Array<Awaited<ReturnType<typeof buildPage>>>;
-  /** The single Book representing everything ingested by this call.
-   *  One ingest call = one Book, always. All pages are members.
-   *  A collection of Books becomes a Volume; a collection of Volumes
-   *  becomes a Shelf — those tiers are assembled by the Daydreamer. */
+  /** All Books produced by this ingest call.  The hierarchy builder chunks
+   *  pages into books of up to PAGES_PER_BOOK and computes a medoid for each. */
+  books: Book[];
+  /** Convenience alias for `books[0]` — undefined when no pages were ingested. */
   book?: Book;
-}
-
-function cosineDistance(a: Float32Array, b: Float32Array): number {
-  let dot = 0;
-  let normA = 0;
-  let normB = 0;
-  for (let i = 0; i < a.length; i++) {
-    dot += a[i] * b[i];
-    normA += a[i] * a[i];
-    normB += b[i] * b[i];
-  }
-  const denom = Math.sqrt(normA) * Math.sqrt(normB);
-  if (denom === 0) return 0;
-  return 1 - dot / denom;
-}
-
-/**
- * Selects the index of the medoid: the element that minimises total cosine
- * distance to every other element in the set.
- */
-function selectMedoidIndex(vectors: Float32Array[]): number {
-  if (vectors.length === 1) return 0;
-  let bestIdx = 0;
-  let bestTotal = Infinity;
-  for (let i = 0; i < vectors.length; i++) {
-    let total = 0;
-    for (let j = 0; j < vectors.length; j++) {
-      if (i !== j) total += cosineDistance(vectors[i], vectors[j]);
-    }
-    if (total < bestTotal) {
-      bestTotal = total;
-      bestIdx = i;
-    }
-  }
-  return bestIdx;
+  /** Volumes produced by grouping books during hierarchy construction. */
+  volumes: Volume[];
+  /** Shelves produced by grouping volumes during hierarchy construction. */
+  shelves: Shelf[];
 }
 
 export async function ingestText(
@@ -76,7 +45,7 @@ export async function ingestText(
 
   const chunks = chunkText(text, modelProfile);
   if (chunks.length === 0) {
-    return { pages: [], book: undefined };
+    return { pages: [], books: [], book: undefined, volumes: [], shelves: [] };
   }
 
   const createdAt = new Date(now).toISOString();
@@ -124,23 +93,7 @@ export async function ingestText(
     });
   }
 
-  // Build ONE Book for the entire ingest.
-  // A Book = the document we just ingested; its identity is the sorted set of
-  // its pages. Its representative is the page whose embedding is the medoid
-  // (minimum total cosine distance to all other pages in the document).
-  const medoidIdx = selectMedoidIndex(embeddings);
-  const sortedPageIds = [...pageIds].sort();
-  const bookId = await hashText(sortedPageIds.join("|"));
-  const book: Book = {
-    bookId,
-    pageIds,
-    medoidPageId: pageIds[medoidIdx],
-    meta: {},
-  };
-  await metadataStore.putBook(book);
-
   // Insert semantic neighbor edges for the new pages against all stored pages.
-  // Volumes and Shelves are assembled by the Daydreamer from accumulated Books.
   const allPages = await metadataStore.getAllPages();
   const allPageIds = allPages.map((p) => p.pageId);
   await insertSemanticNeighbors(pageIds, allPageIds, {
@@ -149,8 +102,20 @@ export async function ingestText(
     metadataStore,
   });
 
-  // Run hotpath promotion for the newly ingested pages and book.
-  await runPromotionSweep([...pageIds, bookId], metadataStore);
+  // Build the full hierarchy: Pages → Books → Volumes → Shelves.
+  // buildHierarchy handles medoid selection, adjacency edges, prototype
+  // computation, Williams fanout enforcement, and promotion sweeps.
+  const hierarchy = await buildHierarchy(pageIds, {
+    modelProfile,
+    vectorStore,
+    metadataStore,
+  });
 
-  return { pages, book };
+  return {
+    pages,
+    books: hierarchy.books,
+    book: hierarchy.books[0],
+    volumes: hierarchy.volumes,
+    shelves: hierarchy.shelves,
+  };
 }

tests/hippocampus/Ingest.test.ts

@@ -67,10 +67,21 @@ describe("hippocampus ingest", () => {
       lastQueryAt: result.pages[0].createdAt,
     });
 
-    // Book should contain the pages
+    // Book should contain some of the pages (hierarchy builder chunks by PAGES_PER_BOOK)
+    expect(result.book).toBeDefined();
     const storedBook = await metadataStore.getBook(result.book!.bookId);
     expect(storedBook).toEqual(result.book);
 
+    // All pages should be covered by the books
+    const allBookPageIds = result.books.flatMap((b) => b.pageIds);
+    for (const page of result.pages) {
+      expect(allBookPageIds).toContain(page.pageId);
+    }
+
+    // Volumes and shelves should be produced
+    expect(result.volumes.length).toBeGreaterThanOrEqual(1);
+    expect(result.shelves.length).toBeGreaterThanOrEqual(1);
+
     // Vector store should have data stored for each page
     expect(vectorStore.byteLength).toBeGreaterThan(0);
   });

tests/integration/IngestQuery.test.ts

@@ -234,10 +234,12 @@ describe("integration: ingest and query", () => {
       expect(stored!.embeddingDim).toBe(EMBEDDING_DIM);
     }
 
-    // Book should reference all page IDs
-    const book = await metadataStore.getBook(result.book!.bookId);
-    expect(book).toBeDefined();
-    expect(book!.pageIds).toEqual(result.pages.map((p) => p.pageId));
+    // Books should collectively reference all page IDs
+    expect(result.books.length).toBeGreaterThanOrEqual(1);
+    const allBookPageIds = result.books.flatMap((b) => b.pageIds);
+    for (const page of result.pages) {
+      expect(allBookPageIds).toContain(page.pageId);
+    }
 
     // Activity records should be initialized for each page
     for (const page of result.pages) {
@@ -278,7 +280,7 @@ describe("integration: ingest and query", () => {
     });
 
     const ingestedPageIds = result.pages.map((p) => p.pageId);
-    const bookId = result.book!.bookId;
+    const bookIds = result.books.map((b) => b.bookId);
 
     // ---- Session 2: Reopen the same database and verify persistence ----
 
@@ -291,10 +293,11 @@ describe("integration: ingest and query", () => {
       expect(page!.pageId).toBe(pageId);
     }
 
-    // Book should still be there
-    const book = await store2.getBook(bookId);
-    expect(book).toBeDefined();
-    expect(book!.pageIds).toEqual(ingestedPageIds);
+    // Books should still be there
+    for (const bookId of bookIds) {
+      const book = await store2.getBook(bookId);
+      expect(book).toBeDefined();
+    }
 
     // Activity records should survive
     for (const pageId of ingestedPageIds) {
@@ -398,7 +401,7 @@ describe("integration (v0.5): hierarchical and dialectical ingest/query", () =>
     (globalThis as Record<string, unknown>)["IDBKeyRange"] = FakeIDBKeyRange;
   });
 
-  it("ingest produces a single Book containing all ingested pages", async () => {
+  it("ingest produces Books, Volumes, and Shelves via HierarchyBuilder", async () => {
     const dbName = freshDbName();
     const metadataStore = await IndexedDbMetadataStore.open(dbName);
     const vectorStore = new MemoryVectorStore();
@@ -417,21 +420,39 @@ describe("integration (v0.5): hierarchical and dialectical ingest/query", () =>
     // Pages were created
     expect(result.pages.length).toBeGreaterThanOrEqual(1);
 
-    // Exactly one Book was created and it contains ALL ingested pages
+    // At least one Book was created
+    expect(result.books.length).toBeGreaterThanOrEqual(1);
     expect(result.book).toBeDefined();
-    const storedBook = await metadataStore.getBook(result.book!.bookId);
-    expect(storedBook).toBeDefined();
-    expect(storedBook!.medoidPageId).toBeDefined();
-    expect(storedBook!.pageIds).toContain(storedBook!.medoidPageId);
-    // Every page from the ingest must be a member of the book
+
+    // Every page must belong to exactly one book
+    const allBookPageIds = result.books.flatMap((b) => b.pageIds);
     for (const page of result.pages) {
-      expect(storedBook!.pageIds).toContain(page.pageId);
+      expect(allBookPageIds).toContain(page.pageId);
+    }
+    // Every book's medoid must be one of its own pages
+    for (const book of result.books) {
+      const storedBook = await metadataStore.getBook(book.bookId);
+      expect(storedBook).toBeDefined();
+      expect(storedBook!.medoidPageId).toBeDefined();
+      expect(storedBook!.pageIds).toContain(storedBook!.medoidPageId);
     }
-    // The book covers all pages — not just a subset
-    expect(storedBook!.pageIds.length).toBe(result.pages.length);
 
-    // Volumes and Shelves are assembled by the Daydreamer; not created at ingest time
-    expect(result.book).toBeDefined(); // only book is returned
+    // Volumes and Shelves are now produced during ingest via HierarchyBuilder
+    expect(result.volumes.length).toBeGreaterThanOrEqual(1);
+    expect(result.shelves.length).toBeGreaterThanOrEqual(1);
+
+    // Each volume should be persisted
+    for (const vol of result.volumes) {
+      const stored = await metadataStore.getVolume(vol.volumeId);
+      expect(stored).toBeDefined();
+      expect(stored!.bookIds.length).toBeGreaterThan(0);
+    }
+    // Each shelf should be persisted
+    for (const shelf of result.shelves) {
+      const stored = await metadataStore.getShelf(shelf.shelfId);
+      expect(stored).toBeDefined();
+      expect(stored!.volumeIds.length).toBeGreaterThan(0);
+    }
   });
 
   it("hotpath entries exist for hierarchy prototypes after ingest", async () => {