Skip to content

Commit d824e68

Browse files
zydozydo
committed
Add /monitor observability page and rich per-engine metrics
Surface scrape behavior on a built-in ops page (no Prometheus — this is a single-user Docker-Compose deployment, so no extra services). - GET /api/v1/metrics?window= extends the existing 4-field summary to a superset (original fields kept): overall rollup + per-engine breakdown (health label, success rate, avg/p50/p95 fetch latency, items, 0-parse selector-rot count, blocks, failures, last status + breakdown), plus recent cycles and recent failures. Aggregation in SQL (percentile_cont); the per-engine health label is a heuristic classify_engine helper. - /monitor page (linked from the wire masthead): KPI strip, engines table with color-coded health dots, cycle-duration sparkline + list, recent-failures log. Reuses the wire-desk palette; textContent rendering throughout. - db.ensure_schema() also runs in the API lifespan so the schema evolves regardless of which process starts first. Tests: tests/test_metrics.py (classify_engine), integration metrics test. Docs: docs/OBSERVABILITY.md; API_REFERENCE.md updated.
1 parent 04d2459 commit d824e68

10 files changed

Lines changed: 1071 additions & 19 deletions

File tree

‎api/main.py‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,10 @@
1717
from fastapi.staticfiles import StaticFiles
1818
from fastapi.middleware.cors import CORSMiddleware
1919
from pathlib import Path
20+
from starlette.concurrency import run_in_threadpool
2021
from starlette.middleware.base import BaseHTTPMiddleware
2122

22-
from common.database import close_pool
23+
from common.database import close_pool, ensure_schema
2324
from common.logging_config import configure_logging
2425
from common.settings import settings
2526
from .exceptions import TopicStreamsException
@@ -117,6 +118,9 @@ async def dispatch(self, request: Request, call_next):
117118

118119
@asynccontextmanager
119120
async def lifespan(__app: FastAPI):
121+
# Evolve the schema for an existing volume (adds scraper_logs.duration_ms
122+
# and the scraper_cycles table). Shared with the scraper process; idempotent.
123+
await run_in_threadpool(ensure_schema)
120124
websocket_manager.start_listener()
121125
yield
122126
await websocket_manager.stop_listener()
@@ -178,6 +182,21 @@ async def read_root():
178182
)
179183

180184

185+
@app.get("/monitor")
186+
async def read_monitor():
187+
"""Serve the scrape-observability monitor page."""
188+
index_file = static_dir / "monitor.html"
189+
if index_file.exists():
190+
return FileResponse(index_file)
191+
return JSONResponse(
192+
status_code=404,
193+
content={
194+
"error": "Monitor page not found",
195+
"message": "Static files not available",
196+
},
197+
)
198+
199+
181200
app.include_router(v1_router)
182201

183202

‎api/static/index.html‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
<span class="sep">·</span>
4141
<span><span id="total-news">–</span> filed</span>
4242
<button id="theme-toggle" class="chip" type="button" aria-label="Toggle theme">☾</button>
43+
<a class="chip" href="/monitor">monitor</a>
4344
<a class="chip" href="https://github.com/zydo/topicstreams" target="_blank"
4445
rel="noopener noreferrer">github ↗</a>
4546
</div>

‎api/static/monitor.html‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
4+
<head>
5+
<meta charset="UTF-8">
6+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
7+
<title>TopicStreams — Ops / Monitor</title>
8+
<link rel="icon" type="image/svg+xml" href="/static/favicon.svg">
9+
<link rel="icon" type="image/png" sizes="32x32" href="/static/favicon-32.png">
10+
<link rel="apple-touch-icon" href="/static/apple-touch-icon.png">
11+
<link rel="stylesheet" href="/static/styles.css">
12+
<script>
13+
// Set theme before first paint to avoid a flash of the wrong palette.
14+
(function () {
15+
try {
16+
var t = localStorage.getItem('ts-theme');
17+
if (!t) t = matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light';
18+
document.documentElement.dataset.theme = t;
19+
} catch (e) {
20+
console.debug('theme init fell back to light:', e);
21+
document.documentElement.dataset.theme = 'light';
22+
}
23+
})();
24+
</script>
25+
</head>
26+
27+
<body>
28+
<div class="sheet monitor">
29+
<!-- Masthead -->
30+
<header class="masthead">
31+
<div class="brand">TOPICSTREAMS <span>/ ops</span></div>
32+
<div class="status">
33+
<span class="monitor__updated"><span id="last-updated">—</span></span>
34+
<span class="sep">·</span>
35+
<label class="filter">Window
36+
<select id="window-select">
37+
<option value="3600">1h</option>
38+
<option value="21600">6h</option>
39+
<option value="86400">24h</option>
40+
</select>
41+
</label>
42+
<button id="theme-toggle" class="chip" type="button" aria-label="Toggle theme">☾</button>
43+
<a class="chip" href="/">← the wire</a>
44+
</div>
45+
</header>
46+
47+
<!-- Overall strip -->
48+
<section class="kpis" id="kpis">
49+
<div class="kpi"><div class="kpi__n" id="kpi-topics">–</div><div class="kpi__l">active topics</div></div>
50+
<div class="kpi"><div class="kpi__n" id="kpi-filed">–</div><div class="kpi__l">filed</div></div>
51+
<div class="kpi"><div class="kpi__n" id="kpi-success">–</div><div class="kpi__l">scrape success</div></div>
52+
<div class="kpi"><div class="kpi__n" id="kpi-fresh">–</div><div class="kpi__l">feed freshness</div></div>
53+
<div class="kpi"><div class="kpi__n" id="kpi-cycle">–</div><div class="kpi__l">last cycle</div></div>
54+
<div class="kpi"><div class="kpi__n" id="kpi-scrapes">–</div><div class="kpi__l">scrapes (blocked / fail)</div></div>
55+
</section>
56+
57+
<!-- Engines -->
58+
<section class="panel">
59+
<h2>Engines</h2>
60+
<div class="table-wrap">
61+
<table class="engines" id="engines-table">
62+
<thead>
63+
<tr>
64+
<th class="al">engine</th>
65+
<th>health</th>
66+
<th>scrapes</th>
67+
<th>success</th>
68+
<th>latency avg / p95</th>
69+
<th>items</th>
70+
<th title="Successful scrapes that parsed 0 items — selector-rot signal">0-parse</th>
71+
<th title="Failures with a 429/403/503 status">blocks</th>
72+
<th>fails</th>
73+
<th>last status</th>
74+
<th>last scrape</th>
75+
</tr>
76+
</thead>
77+
<tbody id="engines-body">
78+
<tr><td colspan="11" class="muted center">loading…</td></tr>
79+
</tbody>
80+
</table>
81+
</div>
82+
</section>
83+
84+
<!-- Cycles -->
85+
<section class="panel">
86+
<h2>Recent cycles</h2>
87+
<div class="spark" id="cycle-spark" aria-hidden="true"></div>
88+
<div class="cycles" id="cycles-list">
89+
<div class="muted">loading…</div>
90+
</div>
91+
</section>
92+
93+
<!-- Failures -->
94+
<section class="panel">
95+
<h2>Recent failures</h2>
96+
<div class="failures" id="failures-list">
97+
<div class="muted">loading…</div>
98+
</div>
99+
</section>
100+
</div>
101+
102+
<!-- Engine row template -->
103+
<template id="engine-row-template">
104+
<tr>
105+
<td class="c-engine"><span class="edot"></span><span class="ename"></span></td>
106+
<td class="c-health"><span class="ehealth"></span></td>
107+
<td class="c-scrapes num r"></td>
108+
<td class="c-success num r"></td>
109+
<td class="c-latency num r"></td>
110+
<td class="c-items num r"></td>
111+
<td class="c-zparse num r"></td>
112+
<td class="c-blocks num r"></td>
113+
<td class="c-fails num r"></td>
114+
<td class="c-status r"><span class="estatus"></span></td>
115+
<td class="c-last r"><span class="elast"></span></td>
116+
</tr>
117+
</template>
118+
119+
<!-- Cycle row template -->
120+
<template id="cycle-row-template">
121+
<div class="cycle">
122+
<span class="cycle__dot"></span>
123+
<span class="cycle__when"></span>
124+
<span class="cycle__dur"></span>
125+
<span class="cycle__meta"></span>
126+
</div>
127+
</template>
128+
129+
<!-- Failure row template -->
130+
<template id="failure-row-template">
131+
<div class="failure">
132+
<span class="failure__time"></span>
133+
<span class="engine-badge failure__engine"></span>
134+
<span class="failure__topic"></span>
135+
<span class="failure__status"></span>
136+
<span class="failure__msg"></span>
137+
</div>
138+
</template>
139+
140+
<script src="/static/monitor.js"></script>
141+
<script>
142+
// Theme toggle chrome (mirrors index.html).
143+
(function () {
144+
var root = document.documentElement;
145+
var themeBtn = document.getElementById('theme-toggle');
146+
function syncTheme() {
147+
var dark = root.dataset.theme === 'dark';
148+
themeBtn.textContent = dark ? '☀' : '☾';
149+
themeBtn.title = dark ? 'Switch to light' : 'Switch to dark';
150+
}
151+
syncTheme();
152+
themeBtn.addEventListener('click', function () {
153+
var next = root.dataset.theme === 'dark' ? 'light' : 'dark';
154+
root.dataset.theme = next;
155+
try { localStorage.setItem('ts-theme', next); } catch (e) { console.debug('could not persist theme preference:', e); }
156+
syncTheme();
157+
});
158+
})();
159+
</script>
160+
</body>
161+
162+
</html>

0 commit comments

Comments
 (0)