Skip to content

Commit 8acc3d9

Browse files
pszymkowiakpszymkowiak
authored
perf: fix N+1 vector search, double embedding, and topic health queries (#38)
- search_by_embedding: batch fetch memories in single IN query instead of N+1 gets - tool_store: compute embedding once and reuse for both store and dedup check (-200ms/store) - topic_health: consolidate 6 separate queries into single aggregated SQL query - README: add LongMemEval benchmark results, remove competitor comparison table - Add LongMemEval benchmark script with LLM judge support (claude/ollama) - Add cross-LLM benchmark script (Claude stores, Gemini recalls)
1 parent 95a5d02 commit 8acc3d9

5 files changed

Lines changed: 890 additions & 107 deletions

File tree

‎README.md‎

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -356,16 +356,6 @@ ICM extracts memories automatically via three layers:
356356

357357
All 3 layers are installed automatically by `icm init --mode hook`.
358358

359-
### Comparison with alternatives
360-
361-
| System | Method | LLM cost | Latency | Captures compaction? |
362-
|--------|--------|----------|---------|---------------------|
363-
| **ICM** | 3-layer extraction | 0 to ~500 tok/session | 0ms | **Yes (PreCompact)** |
364-
| Mem0 | 2 LLM calls/message | ~2k tok/message | 200-2000ms | No |
365-
| claude-mem | PostToolUse + async | ~1-5k tok/session | 8ms hook | No |
366-
| MemGPT/Letta | Agent self-manages | 0 marginal | 0ms | No |
367-
| DiffMem | Git-based diffs | 0 | 0ms | No |
368-
369359
## Benchmarks
370360

371361
### Storage performance
@@ -442,6 +432,32 @@ qwen2.5:3b 3B 2% 58% +56%
442432

443433
`scripts/bench-ollama.sh qwen2.5:14b`
444434

435+
### LongMemEval (ICLR 2025)
436+
437+
Standard academic benchmark — 500 questions across 6 memory abilities, from the [LongMemEval paper](https://arxiv.org/abs/2410.10813) (ICLR 2025).
438+
439+
```
440+
LongMemEval Results — ICM (oracle variant, 500 questions)
441+
════════════════════════════════════════════════════════════════
442+
Category Retrieval Answer (Sonnet)
443+
────────────────────────────────────────────────────────────────
444+
single-session-user 100.0% 91.4%
445+
temporal-reasoning 100.0% 85.0%
446+
single-session-assistant 100.0% 83.9%
447+
multi-session 100.0% 81.2%
448+
knowledge-update 100.0% 80.8%
449+
single-session-preference 100.0% 50.0%
450+
────────────────────────────────────────────────────────────────
451+
OVERALL 100.0% 82.0%
452+
════════════════════════════════════════════════════════════════
453+
```
454+
455+
- **Retrieval** = does ICM find the right information? **100% across all categories.**
456+
- **Answer** = can the LLM produce the correct answer from retrieved context? Depends on the LLM, not ICM.
457+
- The retrieval score is the ICM benchmark. The answer score reflects the downstream LLM capability.
458+
459+
`scripts/bench-longmemeval.py --judge claude --workers 8`
460+
445461
### Test protocol
446462

447463
All benchmarks use **real API calls** — no mocks, no simulated responses, no cached answers.

‎crates/icm-mcp/src/tools.rs‎

Lines changed: 42 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -599,50 +599,56 @@ fn tool_store(
599599
}
600600

601601
// Auto-embed if embedder is available
602-
if let Some(emb) = embedder {
602+
let embed_vec = if let Some(emb) = embedder {
603603
let text = format!("{topic} {content}");
604604
match emb.embed(&text) {
605-
Ok(vec) => memory.embedding = Some(vec),
606-
Err(e) => tracing::warn!("embedding failed: {e}"),
605+
Ok(vec) => Some(vec),
606+
Err(e) => {
607+
tracing::warn!("embedding failed: {e}");
608+
None
609+
}
607610
}
611+
} else {
612+
None
613+
};
614+
615+
if let Some(ref vec) = embed_vec {
616+
memory.embedding = Some(vec.clone());
608617
}
609618

610619
// Dedup check: if a very similar memory exists in the same topic, update it instead
611-
if let Some(emb) = embedder {
620+
if let Some(ref query_emb) = embed_vec {
612621
let text = format!("{topic} {content}");
613-
if let Ok(query_emb) = emb.embed(&text) {
614-
if let Ok(similar) = store.search_hybrid(&text, &query_emb, 1) {
615-
if let Some((existing, score)) = similar.first() {
616-
if score > &0.85 && existing.topic == topic {
617-
// Very similar content in same topic — update instead of duplicate
618-
let mut updated = existing.clone();
619-
updated.summary = content.to_string();
620-
updated.updated_at = Utc::now();
621-
updated.weight = 1.0; // Reset weight on update
622-
if let Some(raw) = get_str(args, "raw_excerpt") {
623-
updated.raw_excerpt = Some(raw.into());
624-
}
625-
if let Some(keywords_arr) = args.get("keywords").and_then(|v| v.as_array())
626-
{
627-
updated.keywords = keywords_arr
628-
.iter()
629-
.filter_map(|v| v.as_str().map(String::from))
630-
.collect();
631-
}
632-
updated.importance = importance;
633-
updated.embedding = Some(query_emb);
634-
if let Err(e) = store.update(&updated) {
635-
return ToolResult::error(format!("failed to update: {e}"));
636-
}
637-
return if compact {
638-
ToolResult::text(format!("ok:{}", updated.id))
639-
} else {
640-
ToolResult::text(format!(
641-
"Updated existing memory (similarity {score:.2}): {}",
642-
updated.id
643-
))
644-
};
622+
if let Ok(similar) = store.search_hybrid(&text, query_emb, 1) {
623+
if let Some((existing, score)) = similar.first() {
624+
if score > &0.85 && existing.topic == topic {
625+
// Very similar content in same topic — update instead of duplicate
626+
let mut updated = existing.clone();
627+
updated.summary = content.to_string();
628+
updated.updated_at = Utc::now();
629+
updated.weight = 1.0; // Reset weight on update
630+
if let Some(raw) = get_str(args, "raw_excerpt") {
631+
updated.raw_excerpt = Some(raw.into());
632+
}
633+
if let Some(keywords_arr) = args.get("keywords").and_then(|v| v.as_array()) {
634+
updated.keywords = keywords_arr
635+
.iter()
636+
.filter_map(|v| v.as_str().map(String::from))
637+
.collect();
638+
}
639+
updated.importance = importance;
640+
updated.embedding = Some(query_emb.clone());
641+
if let Err(e) = store.update(&updated) {
642+
return ToolResult::error(format!("failed to update: {e}"));
645643
}
644+
return if compact {
645+
ToolResult::text(format!("ok:{}", updated.id))
646+
} else {
647+
ToolResult::text(format!(
648+
"Updated existing memory (similarity {score:.2}): {}",
649+
updated.id
650+
))
651+
};
646652
}
647653
}
648654
}

‎crates/icm-store/src/store.rs‎

Lines changed: 79 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -457,7 +457,7 @@ impl MemoryStore for SqliteStore {
457457
) -> IcmResult<Vec<(Memory, f32)>> {
458458
let query_blob = embedding_to_blob(embedding);
459459

460-
// First get the KNN results from vec_memories
460+
// KNN query on vec0 virtual table (requires LIMIT in the query itself)
461461
let mut knn_stmt = self
462462
.conn
463463
.prepare(
@@ -477,14 +477,42 @@ impl MemoryStore for SqliteStore {
477477
.filter_map(|r| r.ok())
478478
.collect();
479479

480-
// Then fetch the full memory for each result
481-
let mut results = Vec::new();
482-
for (id, distance) in &knn_rows {
483-
if let Some(memory) = self.get(id)? {
484-
let similarity = 1.0 - distance;
485-
results.push((memory, similarity));
486-
}
480+
if knn_rows.is_empty() {
481+
return Ok(Vec::new());
487482
}
483+
484+
// Batch fetch all memories in one query
485+
let placeholders: Vec<String> = (1..=knn_rows.len()).map(|i| format!("?{i}")).collect();
486+
let sql = format!(
487+
"SELECT {SELECT_COLS} FROM memories WHERE id IN ({})",
488+
placeholders.join(", ")
489+
);
490+
let mut stmt = self
491+
.conn
492+
.prepare(&sql)
493+
.map_err(|e| IcmError::Database(e.to_string()))?;
494+
495+
let ids: Vec<&str> = knn_rows.iter().map(|(id, _)| id.as_str()).collect();
496+
let params: Vec<&dyn rusqlite::types::ToSql> = ids
497+
.iter()
498+
.map(|id| id as &dyn rusqlite::types::ToSql)
499+
.collect();
500+
501+
let rows = stmt
502+
.query_map(&*params, row_to_memory)
503+
.map_err(|e| IcmError::Database(e.to_string()))?;
504+
505+
let mut memory_map: std::collections::HashMap<String, Memory> = HashMap::new();
506+
for row in rows.flatten() {
507+
memory_map.insert(row.id.clone(), row);
508+
}
509+
510+
// Reassemble in KNN order with similarity scores
511+
let results: Vec<(Memory, f32)> = knn_rows
512+
.into_iter()
513+
.filter_map(|(id, distance)| memory_map.remove(&id).map(|mem| (mem, 1.0 - distance)))
514+
.collect();
515+
488516
Ok(results)
489517
}
490518

@@ -696,73 +724,63 @@ impl MemoryStore for SqliteStore {
696724
}
697725

698726
fn topic_health(&self, topic: &str) -> IcmResult<TopicHealth> {
699-
let entry_count = self.count_by_topic(topic)?;
700-
if entry_count == 0 {
701-
return Err(IcmError::NotFound(format!("topic: {topic}")));
702-
}
703-
704-
let (avg_weight, avg_access): (f32, f32) = self
727+
let row = self
705728
.conn
706729
.query_row(
707-
"SELECT AVG(weight), AVG(CAST(access_count AS REAL)) FROM memories WHERE topic = ?1",
730+
"SELECT
731+
COUNT(*),
732+
AVG(weight),
733+
AVG(CAST(access_count AS REAL)),
734+
MIN(created_at),
735+
MAX(created_at),
736+
MAX(last_accessed),
737+
SUM(CASE WHEN weight < 0.5
738+
AND julianday('now') - julianday(last_accessed) > 14
739+
THEN 1 ELSE 0 END)
740+
FROM memories WHERE topic = ?1",
708741
params![topic],
709-
|row| Ok((row.get(0)?, row.get(1)?)),
742+
|row| {
743+
Ok((
744+
row.get::<_, usize>(0)?,
745+
row.get::<_, f32>(1)?,
746+
row.get::<_, f32>(2)?,
747+
row.get::<_, Option<String>>(3)?,
748+
row.get::<_, Option<String>>(4)?,
749+
row.get::<_, Option<String>>(5)?,
750+
row.get::<_, usize>(6)?,
751+
))
752+
},
710753
)
711754
.map_err(|e| IcmError::Database(e.to_string()))?;
712755

713-
let oldest: Option<DateTime<Utc>> = self
714-
.conn
715-
.query_row(
716-
"SELECT MIN(created_at) FROM memories WHERE topic = ?1",
717-
params![topic],
718-
|row| row.get::<_, Option<String>>(0),
719-
)
720-
.map_err(|e| IcmError::Database(e.to_string()))?
721-
.and_then(|s| DateTime::parse_from_rfc3339(&s).ok())
722-
.map(|d| d.with_timezone(&Utc));
723-
724-
let newest: Option<DateTime<Utc>> = self
725-
.conn
726-
.query_row(
727-
"SELECT MAX(created_at) FROM memories WHERE topic = ?1",
728-
params![topic],
729-
|row| row.get::<_, Option<String>>(0),
730-
)
731-
.map_err(|e| IcmError::Database(e.to_string()))?
732-
.and_then(|s| DateTime::parse_from_rfc3339(&s).ok())
733-
.map(|d| d.with_timezone(&Utc));
756+
let (
757+
entry_count,
758+
avg_weight,
759+
avg_access,
760+
oldest_str,
761+
newest_str,
762+
last_accessed_str,
763+
stale_count,
764+
) = row;
734765

735-
let last_accessed: Option<DateTime<Utc>> = self
736-
.conn
737-
.query_row(
738-
"SELECT MAX(last_accessed) FROM memories WHERE topic = ?1",
739-
params![topic],
740-
|row| row.get::<_, Option<String>>(0),
741-
)
742-
.map_err(|e| IcmError::Database(e.to_string()))?
743-
.and_then(|s| DateTime::parse_from_rfc3339(&s).ok())
744-
.map(|d| d.with_timezone(&Utc));
766+
if entry_count == 0 {
767+
return Err(IcmError::NotFound(format!("topic: {topic}")));
768+
}
745769

746-
// Stale = not accessed in 14 days and weight < 0.5
747-
let stale_count: usize = self
748-
.conn
749-
.query_row(
750-
"SELECT COUNT(*) FROM memories WHERE topic = ?1
751-
AND weight < 0.5
752-
AND julianday('now') - julianday(last_accessed) > 14",
753-
params![topic],
754-
|row| row.get(0),
755-
)
756-
.map_err(|e| IcmError::Database(e.to_string()))?;
770+
let parse_dt = |s: &str| -> Option<DateTime<Utc>> {
771+
DateTime::parse_from_rfc3339(s)
772+
.ok()
773+
.map(|d| d.with_timezone(&Utc))
774+
};
757775

758776
Ok(TopicHealth {
759777
topic: topic.to_string(),
760778
entry_count,
761779
avg_weight,
762780
avg_access_count: avg_access,
763-
oldest,
764-
newest,
765-
last_accessed,
781+
oldest: oldest_str.as_deref().and_then(parse_dt),
782+
newest: newest_str.as_deref().and_then(parse_dt),
783+
last_accessed: last_accessed_str.as_deref().and_then(parse_dt),
766784
needs_consolidation: entry_count > 5,
767785
stale_count,
768786
})

0 commit comments

Comments
 (0)