feat(chat): cache eviction on thumbs-down / faithfulness flag + hit-rate card

A cached answer otherwise lives for its full 7-day TTL even if it's wrong — a
thumbs-down or a hallucination flag couldn't clear it. Now each answer's ChatLog
row records the ChatCache id it's stored under (computeCacheId), so:

- recordFeedback: a thumbs-down (-1) evicts that entry.
- scoreFaithfulness: a flagged (likely-hallucinated) answer evicts it. The
  generate path now awaits storeCache BEFORE firing the async monitor so the
  entry exists when the judge (a multi-second call) decides to evict it.
- CacheHit carries its row id, so a thumbs-down on a *cached* answer evicts too.

Also surfaces a "Cache hit rate" card (cacheHits / chats over the window) on the
admin Chat dashboard.

Verified live: fresh answer cached → thumbs-down → ChatCache row deleted
(1 → 0), and the next identical ask regenerates (cached flag absent); cacheId
recorded on both generated and cache-hit ChatLog rows; admin card renders 6.6%.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Eijma5uKcPZi9tqV2aJBeh

Author: nizzlenitz

app/lib/admin.ts

@@ -87,6 +87,8 @@ export interface ChatAnalytics {
 		chats: number;
 		grounded: number;
 		groundedRate: number;
+		cacheHits: number; // answers served from the ChatCache (no generation)
+		cacheHitRate: number; // cacheHits / chats
 		avgLatencyMs: number;
 		windowDays: number;
 		avgFaithfulness: number; // over scored answers
@@ -401,6 +403,7 @@ export function renderChatDashboard(a: ChatAnalytics): string {
 			value: num(t.flaggedCount),
 			color: t.flaggedCount > 0 ? STATUS.rejected.color : undefined,
 		},
+		{ label: 'Cache hit rate', value: `${pct(t.cacheHitRate)}%`, color: STATUS.activated.color },
 		{ label: 'Avg latency', value: fmtDuration(t.avgLatencyMs) },
 		{ label: 'Window', value: `${t.windowDays} days` },
 	];

app/lib/chat.ts

@@ -203,6 +203,23 @@ function cacheKey(release: string, version: string, normQ: string): string {
 	return `${release}:${version}:${createHash('sha256').update(normQ).digest('hex').slice(0, 20)}`;
 }
 
+// The cache id a generated answer will be stored under — recorded on its ChatLog
+// row so a later thumbs-down or faithfulness flag can evict exactly that entry.
+export function computeCacheId(release: string, version: string, question: string): string {
+	return cacheKey(release, version, normalizeQuestion(question));
+}
+
+// Drop a cached answer (thumbs-down / faithfulness flag) so a bad answer is not
+// re-served for the rest of its TTL. Best-effort; a missing id is a no-op.
+export async function evictCache(cacheId: unknown): Promise<void> {
+	if (!CACHE_ENABLED || typeof cacheId !== 'string' || !cacheId) return;
+	try {
+		await ChatCache.delete(cacheId);
+	} catch {
+		/* best-effort: a missing/already-evicted row is fine */
+	}
+}
+
 function cosine(a: number[], b: number[]): number {
 	let dot = 0;
 	let na = 0;
@@ -217,6 +234,7 @@ function cosine(a: number[], b: number[]): number {
 }
 
 export interface CacheHit {
+	id: string; // the ChatCache row id — recorded on the hit's ChatLog row for eviction
 	answer: string;
 	sources: Source[];
 	model: string;
@@ -234,7 +252,7 @@ export async function lookupCache(version: string, release: string, question: st
 		const exact = await ChatCache.get(cacheKey(release, version, normQ));
 		if (exact && exact.release === release && exact.answer) {
 			bumpCacheHit(exact.id, exact.hitCount ?? 0);
-			return { answer: exact.answer, sources: safeSources(exact.sources), model: exact.model ?? 'cache', via: 'exact' };
+			return { id: exact.id, answer: exact.answer, sources: safeSources(exact.sources), model: exact.model ?? 'cache', via: 'exact' };
 		}
 	} catch {
 		/* fall through to semantic */
@@ -253,7 +271,7 @@ export async function lookupCache(version: string, release: string, question: st
 		})) {
 			if (row?.answer && Array.isArray(row.embedding) && cosine(qv, row.embedding) >= CACHE_SIM_THRESHOLD) {
 				bumpCacheHit(row.id, row.hitCount ?? 0);
-				return { answer: row.answer, sources: safeSources(row.sources), model: row.model ?? 'cache', via: 'semantic' };
+				return { id: row.id, answer: row.answer, sources: safeSources(row.sources), model: row.model ?? 'cache', via: 'semantic' };
 			}
 			break; // only the single nearest neighbor
 		}
@@ -506,6 +524,7 @@ export interface ChatLogInput {
 	sessionId: string;
 	ipHash: string;
 	cached?: boolean;
+	cacheId?: string; // the ChatCache entry this answer is stored under (for eviction)
 }
 
 export async function logChat(input: ChatLogInput): Promise<void> {
@@ -524,6 +543,7 @@ export async function logChat(input: ChatLogInput): Promise<void> {
 			ipHash: input.ipHash,
 			feedback: 0,
 			cached: Boolean(input.cached),
+			cacheId: input.cacheId ?? '',
 		});
 	} catch {
 		// observability is best-effort; never fail a chat on it
@@ -535,7 +555,7 @@ export async function logChat(input: ChatLogInput): Promise<void> {
 // the score + any unsupported claim on the ChatLog row. This catches hallucinations
 // (e.g. an invented Docker image) for review in the admin tab — WITHOUT adding
 // latency or cost to the live chat. Best-effort: never throws.
-export async function scoreFaithfulness(id: string, context: string, answer: string): Promise<void> {
+export async function scoreFaithfulness(id: string, context: string, answer: string, cacheId?: string): Promise<void> {
 	if (!hasLiveModel() || !context || !answer.trim()) return;
 	// Both context (doc content) and answer are treated as data to check.
 	const safe = (s: string, n: number) => s.replace(/<\/?(context|answer)>/gi, '').slice(0, n);
@@ -578,6 +598,9 @@ export async function scoreFaithfulness(id: string, context: string, answer: str
 			flagged,
 			flaggedNote: flagged ? note.slice(0, 300) : '',
 		});
+		// A flagged (likely-hallucinated) answer is evicted so it isn't re-served
+		// from cache while it sits in the review queue.
+		if (flagged && cacheId) void evictCache(cacheId);
 	} catch (err: any) {
 		console.error('[chat] faithfulness error', err?.message ?? err);
 	}
@@ -592,6 +615,9 @@ export async function recordFeedback(id: unknown, value: unknown): Promise<boole
 		const existing = await ChatLog.get(id);
 		if (!existing) return false;
 		await ChatLog.patch({ id, feedback: v });
+		// A thumbs-down evicts the cached answer so the next asker doesn't get the
+		// same disliked response for the rest of its TTL.
+		if (v < 0 && existing.cacheId) void evictCache(existing.cacheId);
 		return true;
 	} catch {
 		return false;

app/lib/metrics.ts

@@ -178,6 +178,7 @@ export async function chatAnalytics(): Promise<ChatAnalytics> {
 	const feedback = { up: 0, down: 0, none: 0 };
 	let chats = 0;
 	let grounded = 0;
+	let cacheHits = 0;
 	let latencySum = 0;
 	let latencyN = 0;
 	let faithSum = 0;
@@ -189,6 +190,7 @@ export async function chatAnalytics(): Promise<ChatAnalytics> {
 		if (now - created > windowMs) continue;
 		chats++;
 		if (row.grounded) grounded++;
+		if (row.cached) cacheHits++;
 		if (typeof row.faithfulness === 'number') {
 			faithSum += row.faithfulness;
 			faithN++;
@@ -247,6 +249,8 @@ export async function chatAnalytics(): Promise<ChatAnalytics> {
 			chats,
 			grounded,
 			groundedRate: chats ? grounded / chats : 0,
+			cacheHits,
+			cacheHitRate: chats ? cacheHits / chats : 0,
 			avgLatencyMs: latencyN ? latencySum / latencyN : 0,
 			windowDays: WINDOW_DAYS,
 			avgFaithfulness: faithN ? faithSum / faithN : 0,

app/resources/site.ts

@@ -23,6 +23,7 @@ import {
 	scoreFaithfulness,
 	lookupCache,
 	storeCache,
+	computeCacheId,
 	currentRelease,
 	condenseQuestion,
 	type CacheHit,
@@ -431,6 +432,12 @@ async function handleChat(request: HarperRequest): Promise<Response> {
 	const quota = await checkAndBumpQuota(ipHash);
 	if (!quota.ok) return jsonResponse({ error: 'daily quota exceeded', cap: quota.cap }, 429);
 
+	// The id this answer will be cached under (empty when uncacheable — dev stub or
+	// no release). Recorded on the ChatLog row so a thumbs-down or a faithfulness
+	// flag can evict exactly this entry.
+	const cacheable = modelId() !== 'stub' && Boolean(release);
+	const cacheId = cacheable ? computeCacheId(release, version, question) : '';
+
 	const started = Date.now();
 	const grounding = await retrieve(question, body.section ?? null, body.version ?? null);
 	const { system } = buildMessages(question, grounding.context);
@@ -462,6 +469,7 @@ async function handleChat(request: HarperRequest): Promise<Response> {
 					latencyMs: Date.now() - started,
 					sessionId,
 					ipHash,
+					cacheId: cacheable ? cacheId : undefined,
 				});
 			try {
 				send('sources', grounding.sources);
@@ -473,11 +481,13 @@ async function handleChat(request: HarperRequest): Promise<Response> {
 				// /api/chat-feedback can't arrive before the ChatLog row exists.
 				await writeLog();
 				send('done', { id: chatId, model: modelId(), latencyMs: Date.now() - started });
+				// Cache the answer for future repeats (skip the dev stub). Store BEFORE
+				// scoring so the faithfulness monitor can evict this exact entry if it
+				// flags the answer (the store is a fast put; the judge is a slow call).
+				if (cacheable) await storeCache(version, release, question, answer, grounding.sources, modelId());
 				// Fire-and-forget: score faithfulness AFTER responding, off the user
-				// path — patches the ChatLog row when it completes. Never awaited.
-				void scoreFaithfulness(chatId, grounding.context, answer);
-				// Cache the answer for future repeats (skip the dev stub). Off-path.
-				if (modelId() !== 'stub') void storeCache(version, release, question, answer, grounding.sources, modelId());
+				// path — patches the ChatLog row and evicts the cache entry if flagged.
+				void scoreFaithfulness(chatId, grounding.context, answer, cacheable ? cacheId : undefined);
 			} catch (err: any) {
 				// Log detail server-side; the client only gets a generic message.
 				console.error('[chat] generation error', err?.message ?? err);
@@ -543,6 +553,7 @@ function streamCachedAnswer(cached: CacheHit, question: string, sessionId: strin
 					sessionId,
 					ipHash,
 					cached: true,
+					cacheId: cached.id, // so a thumbs-down on a cached answer evicts it too
 				});
 			}
 		},

app/schemas/docs.graphql

@@ -194,6 +194,7 @@ type ChatLog @table(expiration: 7776000) {
 	ipHash: String # hashed client IP (never store raw IPs)
 	feedback: Int @indexed # -1 | 0 | +1 (thumbs), 0 = none
 	cached: Boolean @indexed # answer was served from the ChatCache (no generation)
+	cacheId: String # the ChatCache entry this answer is stored under (thumbs-down / flag evicts it)
 	# Async faithfulness monitor (scored after the answer streams — off the user path):
 	faithfulness: Float # 0..1, how well the answer is supported by the retrieved context
 	flagged: Boolean @indexed # faithfulness below the flag threshold (needs review)

app/test/unit/admin.test.ts

@@ -134,6 +134,8 @@ const SAMPLE_CHAT: ChatAnalytics = {
 		chats: 19,
 		grounded: 19,
 		groundedRate: 1,
+		cacheHits: 5,
+		cacheHitRate: 5 / 19,
 		avgLatencyMs: 390,
 		windowDays: 14,
 		avgFaithfulness: 0.94,
@@ -171,6 +173,9 @@ test('renderChatDashboard: active Chat tab, grounded %, recent rows', () => {
 	// Faithfulness monitor: card + flagged-answers panel.
 	assert.match(html, /Avg faithfulness/);
 	assert.match(html, /94\.0%/); // avgFaithfulness 0.94
+	// Cache hit-rate card: 5/19 → 26.3%.
+	assert.match(html, /Cache hit rate/);
+	assert.match(html, /26\.3%/);
 	assert.match(html, /Flagged answers/);
 	assert.match(html, /invented docker image/);
 });