fix: improve explain insights

Author: lindesvard

packages/ai/src/explain.ts

@@ -29,13 +29,27 @@ export interface BreakdownComparison {
   baseline: { name: string | null; sessions: number }[];
 }
 
+// Per-day sessions for the insight's own segment, so the model can read the
+// *shape* of the change (one-off spike on a date vs sustained growth) rather
+// than just current-vs-baseline totals.
+export interface DailyPoint {
+  date: string;
+  sessions: number;
+}
+
 export interface ExplainInsightInput {
   insight: {
     title: string;
     dimension: string;
     window: string;
     summary?: string;
   };
+  // The insight metric's daily series for its own segment.
+  dailySeries?: {
+    metric: string;
+    current: DailyPoint[];
+    baseline: DailyPoint[];
+  };
   breakdowns: BreakdownComparison[];
   references: { title: string; date: string }[];
 }
@@ -66,15 +80,21 @@ const explanationJsonSchema = z.toJSONSchema(explanationSchema, {
 
 const INSTRUCTION = `You explain WHY an analytics metric changed for a website/product owner.
 
-You receive: the insight (what changed), a decomposition of the change across dimensions (referrer, country, device, utm_source) as current-window vs baseline-window session breakdowns, and any manual references (off-platform events the owner logged) near the window.
+You receive: the insight (what changed); a dailySeries — the segment's own per-day values for the current window vs the baseline window; a decomposition of the change across dimensions (referrer, country, device, utm_source) as current-vs-baseline session breakdowns; and any manual references (off-platform events the owner logged) near the window.
+
+Read the dailySeries FIRST — it tells you the *shape* of the change, which the totals alone hide:
+- A one-off spike: most of the change is concentrated in one or a few days, then it returns toward baseline. Say so explicitly and name the peak date(s) and the peak value. A single-day spike inflating a window total is NOT sustained growth — do not describe it as "grew to X" as if it held.
+- A step change: it jumps to a new level and stays there.
+- Sustained/gradual growth or decline: it moves steadily across the window.
+If the series is flat except for a spike, lead with the spike.
 
 Produce:
-- summary: 1-2 plain sentences answering "why did this happen", grounded in the decomposition. Name the sub-segment(s) that account for most of the change (e.g. "most of the lift came from reddit.com referrals").
-- drivers: the concrete contributors, each a short label + a one-line detail with the numbers.
-- relatedReference: the title of a reference that plausibly explains the change, or "" if none fits. Do not force a connection.
-- confidence: low | medium | high — how clearly the decomposition explains the change.
+- summary: 1-2 plain sentences answering "why did this happen", grounded in the data. State the shape (spike / step / sustained) with the peak date when it's a spike, then name the sub-segment(s) that account for most of the change (e.g. "a one-off spike on May 28 — ~60 sessions vs a ~5/day baseline — drove the lift; traffic has since returned to baseline").
+- drivers: the concrete contributors, each a short label + a one-line detail with the numbers (include the peak date/value when relevant).
+- relatedReference: the title of a reference whose date lines up with the spike/change date, or "" if none fits. Prefer a reference dated on or just before the peak day. Do not force a connection.
+- confidence: low | medium | high — how clearly the data explains the change.
 
-Be honest and precise. You can only see what's in the data: explain the internal decomposition (which segment moved), not external causes you can't observe. If the breakdown doesn't clearly explain it, say so and set confidence low. Never invent numbers.`;
+Be honest and precise. You can only see what's in the data: explain the shape and the internal decomposition (which segment moved, when), not external causes you can't observe. If the data doesn't clearly explain it, say so and set confidence low. Never invent numbers or dates.`;
 
 let _app: ReturnType<typeof betterAgent> | null = null;
 function getApp() {

packages/db/src/services/overview.service.ts

@@ -1538,6 +1538,72 @@ export async function getTrafficBreakdownCore(input: {
   });
 }
 
+// Columns whose daily series we can derive from the sessions table. Page/entry
+// insights (path/origin) live on the events table and aren't covered here — the
+// caller degrades to no series for those.
+const SEGMENT_SERIES_COLUMNS: ReadonlySet<string> = new Set<TrafficColumn>([
+  'referrer',
+  'referrer_name',
+  'referrer_type',
+  'utm_source',
+  'utm_medium',
+  'utm_campaign',
+  'country',
+  'region',
+  'city',
+  'device',
+  'browser',
+  'os',
+]);
+
+export interface SegmentDailyPoint {
+  date: string;
+  sessions: number;
+  pageviews: number;
+}
+
+// Daily breakdown for a single segment value (e.g. the "Twitter" referrer),
+// so the explainer can see the *shape* of a change (one-off spike vs sustained
+// growth) instead of only current-vs-baseline totals. Returns one point per day
+// with zero-filled gaps; empty when the column isn't session-derived or the
+// value never appears in the window.
+export async function getSegmentDailySeriesCore(input: {
+  projectId: string;
+  column: string;
+  value: string;
+  startDate: string;
+  endDate: string;
+}): Promise<SegmentDailyPoint[]> {
+  if (!SEGMENT_SERIES_COLUMNS.has(input.column)) {
+    return [];
+  }
+
+  const { timezone } = await getSettingsForProject(input.projectId);
+  const { items } = await overviewService.getTopGenericSeries({
+    projectId: input.projectId,
+    filters: [],
+    startDate: input.startDate,
+    endDate: input.endDate,
+    column: input.column as TrafficColumn,
+    interval: 'day',
+    timezone,
+  });
+
+  // getTopGenericSeries reports empty values as null name; insights store the
+  // empty referrer as "direct". Match the segment leniently.
+  const target = input.value.toLowerCase();
+  const matched = items.find((item) => {
+    const name = (item.name ?? '').toLowerCase();
+    return name === target || (name === '' && target === 'direct');
+  });
+
+  return (matched?.data ?? []).map((point) => ({
+    date: point.date,
+    sessions: Number(point.sessions ?? 0),
+    pageviews: Number(point.pageviews ?? 0),
+  }));
+}
+
 export interface GetAnalyticsOverviewInput {
   projectId: string;
   startDate: string;

packages/trpc/src/routers/insight.ts

@@ -1,11 +1,21 @@
 import { generateInsightExplanation } from '@openpanel/ai';
-import { db, getTrafficBreakdownCore } from '@openpanel/db';
+import {
+  db,
+  getSegmentDailySeriesCore,
+  getTrafficBreakdownCore,
+} from '@openpanel/db';
+import { getRedisCache } from '@openpanel/redis';
+import type { InsightPayload } from '@openpanel/validation';
 import { z } from 'zod';
 import { getProjectAccess } from '../access';
 import { TRPCForbiddenError } from '../errors';
 import { createTRPCRouter, protectedProcedure } from '../trpc';
 
 const DAY_MS = 24 * 60 * 60 * 1000;
+// The explanation is a paid LLM call. Cache it keyed by the insight's
+// lastUpdatedAt so repeat clicks within the window are free, while any recompute
+// that changes the insight produces a new key and a fresh explanation.
+const EXPLAIN_CACHE_TTL_SEC = 24 * 60 * 60;
 const EXPLAIN_COLUMNS = [
   'referrer_name',
   'country',
@@ -117,7 +127,8 @@ export const insightRouter = createTRPCRouter({
 
   // Phase 5: the "why". Decompose the insight's change across referrer/country/
   // device/utm (current vs baseline window), pull nearby references, and have
-  // the AI explain which sub-segment drove it. On-demand (a button), not cached.
+  // the AI explain which sub-segment drove it. On-demand (a button); the result
+  // is cached per insight version so repeat clicks don't re-bill the LLM.
   explain: protectedProcedure
     .input(z.object({ insightId: z.string() }))
     .mutation(async ({ input: { insightId }, ctx }) => {
@@ -132,6 +143,8 @@ export const insightRouter = createTRPCRouter({
           windowKind: true,
           windowStart: true,
           windowEnd: true,
+          payload: true,
+          lastUpdatedAt: true,
         },
       });
 
@@ -144,6 +157,16 @@ export const insightRouter = createTRPCRouter({
         throw new TRPCForbiddenError('You do not have access to this project');
       }
 
+      // Serve a cached explanation if the insight hasn't changed since we
+      // computed it. Skips both the ClickHouse queries and the LLM call.
+      const cacheKey = `insight-explain:${insightId}:${insight.lastUpdatedAt.getTime()}`;
+      const cached = await getRedisCache().get(cacheKey);
+      if (cached) {
+        return JSON.parse(cached) as Awaited<
+          ReturnType<typeof generateInsightExplanation>
+        >;
+      }
+
       // Current window from the insight; baseline = same span immediately before.
       const end = insight.windowEnd ?? new Date();
       const start =
@@ -178,6 +201,54 @@ export const insightRouter = createTRPCRouter({
         }),
       );
 
+      // Daily series for the insight's own segment, so the model can read the
+      // shape of the change (a one-off spike vs sustained growth) instead of
+      // only the current-vs-baseline totals. Best-effort: skip on page/entry
+      // insights (events-table metrics) or anything we can't resolve.
+      const payload = insight.payload as InsightPayload | null;
+      const segment = payload?.dimensions?.[0];
+      const primaryMetric = payload?.primaryMetric ?? 'sessions';
+      let dailySeries:
+        | {
+            metric: string;
+            current: { date: string; sessions: number }[];
+            baseline: { date: string; sessions: number }[];
+          }
+        | undefined;
+
+      if (segment?.key && segment.value) {
+        const [curSeries, baseSeries] = await Promise.all([
+          getSegmentDailySeriesCore({
+            projectId: insight.projectId,
+            column: segment.key,
+            value: segment.value,
+            startDate: iso(start),
+            endDate: iso(end),
+          }),
+          getSegmentDailySeriesCore({
+            projectId: insight.projectId,
+            column: segment.key,
+            value: segment.value,
+            startDate: iso(baseStart),
+            endDate: iso(baseEnd),
+          }),
+        ]);
+
+        const toMetric = (points: typeof curSeries) =>
+          points.map((p) => ({
+            date: p.date.slice(0, 10),
+            sessions: primaryMetric === 'pageviews' ? p.pageviews : p.sessions,
+          }));
+
+        if (curSeries.length > 0) {
+          dailySeries = {
+            metric: primaryMetric,
+            current: toMetric(curSeries),
+            baseline: toMetric(baseSeries),
+          };
+        }
+      }
+
       const references = await db.reference.findMany({
         where: {
           projectId: insight.projectId,
@@ -191,18 +262,31 @@ export const insightRouter = createTRPCRouter({
         select: { title: true, date: true },
       });
 
-      return generateInsightExplanation({
+      const explanation = await generateInsightExplanation({
         insight: {
           title: insight.aiSummary ?? insight.title,
           dimension: insight.dimensionKey,
           window: insight.windowKind,
           summary: insight.summary ?? undefined,
         },
+        dailySeries,
         breakdowns,
         references: references.map((r) => ({
           title: r.title,
           date: r.date.toISOString().slice(0, 10),
         })),
       });
+
+      // Only cache a successful explanation — a null is a transient LLM failure
+      // and should be retried on the next click.
+      if (explanation) {
+        await getRedisCache().setex(
+          cacheKey,
+          EXPLAIN_CACHE_TTL_SEC,
+          JSON.stringify(explanation),
+        );
+      }
+
+      return explanation;
     }),
 });