diff --git a/README.md b/README.md index bdf9452..bac56a6 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,16 @@ das komplette serverjonas search siehe [search/README.md](search/README.md) - `web/` – Statistik-Webseite (Port 8081, `jonassearch-stats.service`). Seiten werden alle 30 Min. gezählt; Bilder alle 2 h exakt, dazwischen nur - geschätzt. Alle Diagramme zeigen ausschließlich die exakten Messpunkte. + geschätzt. Die Image-Diagramme zeigen standardmäßig nur exakte Messpunkte; + ein umschaltbarer, im Browser gespeicherter Schalter kann Schätzwerte in allen + Image-Diagrammen, Records und der Heatmap einblenden. + Mit jeder Messung wird zusätzlich die Anzahl der `pending`-Einträge in + `crawl_queue` erfasst (exakter `COUNT`, keine Schätzung) und mit dem + kompletten Diagrammsatz (Verlauf, Change/5 min, kumulativ, pro Stunde, + Tagesstunden, Records, Heatmap) dargestellt. + Die Hauptseite hat drei Tabs mit Symbolen (Pages, Images, Queue): Jeder Tab + zeigt nur die Kennzahlen und Diagramme seines Datentyps, die Auswahl wird im + Browser gespeichert. CSV/JSON-Export folgt dem aktiven Tab. - `web/tui.py` – Terminal-Dashboard für dieselben Statistiken (siehe unten) - `priorityweb/` – Vorschlagsverfeinerer (Port 8082, `jonassearch-priority.service`) - `search/jonassearch-engine.service` – systemd-Unit für die Suchmaschine diff --git a/web/app.py b/web/app.py index 9b05a38..7cb3af1 100644 --- a/web/app.py +++ b/web/app.py @@ -147,6 +147,61 @@ HTML = r""" .ranges { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 18px; } + .image-options { display: flex; align-items: center; gap: 9px; margin: 0 0 12px; color: var(--muted); font-size: 0.88rem; } + .image-options input { accent-color: var(--accent); } + .image-options label { cursor: pointer; } + .image-options-hint { color: var(--muted); font-size: 0.78rem; margin-bottom: 14px; } + + .metric-tabs { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); + gap: 10px; + margin: 22px 0 16px; + } + + .metric-btn { + display: flex; + align-items: center; + gap: 12px; + text-align: left; + background: var(--card); + border: 1px solid var(--border); + color: var(--muted); + border-radius: 15px; + padding: 13px 16px; + font-size: 1rem; + font-weight: 600; + font-family: inherit; + cursor: pointer; + transition: border-color .15s, color .15s, background .15s; + } + + .metric-btn:hover { border-color: var(--accent); color: var(--text); } + + .metric-btn.active { + color: var(--text); + border-color: var(--accent); + background: linear-gradient(180deg, rgba(79,140,255,0.18), rgba(79,140,255,0.04)); + box-shadow: inset 0 0 0 1px rgba(79,140,255,0.35); + } + + .metric-icon { font-size: 1.6rem; line-height: 1; } + + .metric-name { display: block; } + + .metric-count { + display: block; + font-size: 0.78rem; + font-weight: 400; + color: var(--muted); + margin-top: 2px; + } + + .metric-panel { display: none; } + .metric-panel.active { display: block; } + + .section-title-first { margin-top: 6px; } + .range-btn { background: var(--card); border: 1px solid var(--border); @@ -253,6 +308,8 @@ HTML = r""" .section-title { font-weight: 600; margin: 30px 0 12px; font-size: 1.05rem; } @media (max-width: 600px) { + .metric-btn { padding: 10px 12px; font-size: 0.9rem; gap: 9px; } + .metric-icon { font-size: 1.3rem; } .current { font-size: 2.4rem; } .container { padding: 20px 14px 40px; } .chart-container { padding: 14px; } @@ -266,7 +323,7 @@ HTML = r"""

JonasSearch

-
Indexed Pages – Live Statistics
+
Pages – Live Statistics
Export CSV Export JSON @@ -274,19 +331,28 @@ HTML = r"""
-
-
-
- –Pages -
-
Loading…
-
-
-
- –Images -
-
Loading…
-
+
+ + +
@@ -305,6 +371,16 @@ HTML = r"""
+
+
+
+
+ –Pages +
+
Loading…
+
+
+
Growth / hour
@@ -406,21 +482,39 @@ HTML = r"""
-
Images – all graphs
-
Only exact measurements (counted every 2 h) are plotted – the estimates in between are excluded.
+
+ +
+
+
+
+ –Images +
+
Loading…
+
+
+ +
+ +
Images – all graphs
+
+ + +
+
Only exact measurements (counted every 2 h) are plotted. This setting is saved in this browser.
Images over time
-
Image URLs found while crawling · exact count every 2 h · only exact measurements are plotted (no estimates) · dashed line = trend & 24 h forecast
+
Image URLs found while crawling · dashed line = trend & 24 h forecast
Change per 5 minutes
-
Normalized to a 5-minute interval · spans between two exact counts (2 h) · grey = raw · colored = smoothed (moving average over 5 measurements)
+
Normalized to a 5-minute interval between available image measurements · grey = raw · colored = smoothed (moving average over 5 measurements)
@@ -455,6 +549,70 @@ HTML = r"""
+
+ +
+
+
+
+ –Pending +
+
Loading…
+
+
+ +
+ +
Queue – all graphs
+
URLs waiting in crawl_queue (status = 'pending'). Counted with every 30-minute sample (exact COUNT). Falling graphs mean the crawler is working through the queue.
+ + +
+
Pending URLs over time
+
Queued URLs not yet crawled · dashed line = trend & 24 h forecast
+
+
+ +
+
Change per 5 minutes
+
Normalized to a 5-minute interval · negative = the crawler worked the queue down · grey = raw · colored = smoothed (moving average over 5 measurements)
+
+
+ +
+
Cumulative net change
+
Total queued URLs gained/lost since the start of the selected range · linear time axis
+
+
+ +
+
Queue change per hour (last 48 h)
+
Sum of changes per hour bucket · green = queue grew, red = queue shrank
+
+
+ +
+
+
Queue change by hour of day
+
Average change per 5 min, grouped by hour of day (whole history)
+
+
+
+
Records – biggest queue changes
+
Largest gains and losses between two consecutive measurements (selected range)
+
+
+
+ +
Activity heatmap – queue change per hour of day
+
+
Rows = days · columns = hours (0–23) · green = queue grew, red = queue shrank, grey = no data
+
+
+
+
@@ -467,6 +625,7 @@ if (window.Chart && Chart.registerables) { const ACCENT = '#4f8cff'; const ACCENT2 = '#b47aff'; +const ACCENT3 = '#f5a623'; const UP = '#2ecc71'; const DOWN = '#ff6b6b'; const FALLBACK_INTERVAL_MS = 15 * 60 * 1000; @@ -476,6 +635,8 @@ const REFRESH_S = 30; let rawValues = []; let intervalMs = FALLBACK_INTERVAL_MS; let range = 'all'; +let activeMetric = 'pages'; +let showEstimatedImages = false; let charts = {}; let refreshLeft = REFRESH_S; @@ -603,27 +764,36 @@ function parsePoints(values) { .map(function (v) { const images = (v.images === null || v.images === undefined) ? null : Number(v.images); + const pending = (v.pending === null || v.pending === undefined) + ? null : Number(v.pending); return { x: new Date(v.timestamp).getTime(), y: v.count, images: Number.isFinite(images) ? images : null, - imagesExact: v.images_exact === true + imagesExact: v.images_exact === true, + pending: Number.isFinite(pending) ? pending : null }; }) .filter(function (p) { return !isNaN(p.x); }) .sort(function (a, b) { return a.x - b.x; }); } -// Nur exakte Bildzählungen (alle 2 h). Die dazwischen geschätzten Werte -// werden in keinem Diagramm dargestellt, damit alle Graphen ausschließlich -// echte Messpunkte zeigen. -function toImagePoints(points) { +// Estimates are excluded by default; the Images section toggle can include them. +function toImagePoints(points, includeEstimated) { return (points || []) .filter(function (p) { return p.images !== null && p.images !== undefined; }) - .filter(function (p) { return p.imagesExact; }) + .filter(function (p) { return includeEstimated || p.imagesExact; }) .map(function (p) { return { x: p.x, y: p.images }; }); } +// Pending queue entries are counted with every sample (exact COUNT), so +// no filtering beyond "value present" is needed. +function toPendingPoints(points) { + return (points || []) + .filter(function (p) { return p.pending !== null && p.pending !== undefined; }) + .map(function (p) { return { x: p.x, y: p.pending }; }); +} + const RANGE_SPANS = { '6h': 6 * 3600000, '24h': 24 * 3600000, @@ -673,6 +843,17 @@ document.getElementById('btn-theme').addEventListener('click', function () { applyTheme(document.documentElement.dataset.theme === 'light' ? 'dark' : 'light'); }); +(function () { + const toggle = document.getElementById('show-estimated-images'); + try { showEstimatedImages = localStorage.getItem('js-show-estimated-images') === 'true'; } catch (e) {} + toggle.checked = showEstimatedImages; + toggle.addEventListener('change', function () { + showEstimatedImages = toggle.checked; + try { localStorage.setItem('js-show-estimated-images', String(showEstimatedImages)); } catch (e) {} + if (rawValues.length) renderAll(); + }); +})(); + (function () { let saved = null; try { saved = localStorage.getItem('js-theme'); } catch (e) {} @@ -680,6 +861,44 @@ document.getElementById('btn-theme').addEventListener('click', function () { document.getElementById('btn-theme').textContent = saved === 'light' ? '🌙 Dark' : '☀️ Light'; })(); +/* ---------- metric tabs (pages / images / queue) ---------- */ + +const METRIC_LABELS = { pages: 'Pages', images: 'Images', queue: 'Queue' }; + +// Hidden panels must not render Chart.js instances: a canvas inside a +// display:none container measures 0 px and breaks on later resize. +function isVisible(el) { + const panel = el.closest('.metric-panel'); + return !panel || panel.classList.contains('active'); +} + +function setMetric(next, persist) { + activeMetric = METRIC_LABELS[next] ? next : 'pages'; + + document.querySelectorAll('.metric-btn').forEach(function (btn) { + const on = btn.dataset.metric === activeMetric; + btn.classList.toggle('active', on); + btn.setAttribute('aria-selected', on ? 'true' : 'false'); + }); + document.querySelectorAll('.metric-panel').forEach(function (panel) { + panel.classList.toggle('active', panel.dataset.metric === activeMetric); + }); + + const label = METRIC_LABELS[activeMetric]; + document.getElementById('page-subtitle').textContent = label + ' – Live Statistics'; + document.title = 'JonasSearch – ' + label; + + if (persist !== false) { + try { localStorage.setItem('js-metric', activeMetric); } catch (e) {} + } + + if (rawValues.length) renderAll(); +} + +document.querySelectorAll('.metric-btn').forEach(function (btn) { + btn.addEventListener('click', function () { setMetric(btn.dataset.metric); }); +}); + /* ---------- range buttons ---------- */ document.querySelectorAll('.range-btn').forEach(function (btn) { @@ -702,16 +921,50 @@ function download(name, text, mime) { URL.revokeObjectURL(a.href); } +// Export what the selected tab shows, so the CSV/JSON match the graphs. +function exportSeries() { + const all = parsePoints(rawValues); + + if (activeMetric === 'images') { + const values = all + .filter(function (p) { return p.images !== null; }) + .map(function (p) { + return { timestamp: new Date(p.x).toISOString(), images: p.images, images_exact: p.imagesExact }; + }); + return { name: 'jonassearch_images', values: values }; + } + + if (activeMetric === 'queue') { + const values = all + .filter(function (p) { return p.pending !== null; }) + .map(function (p) { + return { timestamp: new Date(p.x).toISOString(), pending: p.pending }; + }); + return { name: 'jonassearch_queue', values: values }; + } + + return { name: 'jonassearch_pages', values: rawValues }; +} + document.getElementById('btn-csv').addEventListener('click', function (e) { e.preventDefault(); - const lines = ['timestamp;count']; - rawValues.forEach(function (v) { lines.push(v.timestamp + ';' + v.count); }); - download('jonassearch_pages.csv', lines.join('\n'), 'text/csv'); + const series = exportSeries(); + const keys = series.values.length ? Object.keys(series.values[0]) : ['timestamp']; + const lines = [keys.join(';')]; + series.values.forEach(function (row) { + lines.push(keys.map(function (k) { return row[k] === undefined ? '' : row[k]; }).join(';')); + }); + download(series.name + '.csv', lines.join('\n'), 'text/csv'); }); document.getElementById('btn-json').addEventListener('click', function (e) { e.preventDefault(); - download('jonassearch_pages.json', JSON.stringify({ values: rawValues }, null, 2), 'application/json'); + const series = exportSeries(); + download( + series.name + '.json', + JSON.stringify({ metric: activeMetric, values: series.values }, null, 2), + 'application/json' + ); }); /* ---------- chart helpers ---------- */ @@ -767,6 +1020,9 @@ function gradientFill(colorTop, colorBottom) { /* ---------- charts ---------- */ function renderCharts(cfg) { + const target = document.getElementById(chartId(cfg.prefix, 'MainChart')); + if (!target || !isVisible(target)) return; + const prefix = cfg.prefix; const points = cfg.points; const deltas = cfg.deltas; @@ -1052,6 +1308,7 @@ function renderCharts(cfg) { function renderRecords(deltas, id, unit) { const el = document.getElementById(id); + if (!el || !isVisible(el)) return; if (deltas.length < 2) { el.innerHTML = '
Not enough data yet.
'; return; @@ -1078,6 +1335,7 @@ function renderRecords(deltas, id, unit) { function renderHeatmap(points, id, unit) { const el = document.getElementById(id); + if (!el || !isVisible(el)) return; if (!points.length) { el.innerHTML = ''; return; } function localDayKey(d) { @@ -1137,6 +1395,84 @@ function renderHeatmap(points, id, unit) { el.innerHTML = html; } +/* ---------- summary cards for images / queue ---------- */ + +// The pages tab keeps its static card grid (ids like stat-delta-6h); +// images and queue get the same key numbers generated from their series. +function renderSummary(containerId, all, points) { + const el = document.getElementById(containerId); + if (!el || !isVisible(el)) return; + + if (!points.length) { + el.innerHTML = '
No data in the selected range yet.
'; + return; + } + + const anchor = all[all.length - 1]; + const first = points[0]; + const hours = (anchor.x - first.x) / 3600000; + const perHour = hours > 0 ? (anchor.y - first.y) / hours : null; + const deltas = computeDeltas(points).deltas; + + let peak = all[0], low = all[0]; + all.forEach(function (p) { + if (p.y > peak.y) peak = p; + if (p.y < low.y) low = p; + }); + + const cards = []; + + function signedCard(label, value, hint) { + const cls = value === null ? '' : (value >= 0 ? 'pos' : 'neg'); + cards.push('
' + label + + '
' + + (value === null ? '–' : fmtSigned(value)) + + '
' + hint + '
'); + } + + function plainCard(label, value, hint) { + cards.push('
' + label + + '
' + value + + '
' + hint + '
'); + } + + signedCard('Growth / hour', perHour, 'Average over full history'); + signedCard('Growth / day', perHour === null ? null : perHour * 24, 'Average over full history'); + + const windows = [ + ['6 h', 6 * 3600000, Math.max(3 * intervalMs, 3600000)], + ['24 h', 24 * 3600000, Math.max(6 * intervalMs, 3 * 3600000)], + ['7 days', 7 * 24 * 3600000, Math.max(12 * intervalMs, 6 * 3600000)] + ]; + windows.forEach(function (win) { + const base = closestBefore(all, anchor.x - win[1], win[2]); + signedCard('Growth ' + win[0], base ? anchor.y - base.y : null, + base ? 'vs. measurement ' + fmtDur(anchor.x - base.x) + ' ago' : 'Not enough data yet'); + }); + + let sum24 = 0; + const windowStart = Math.max(first.x, anchor.x - 24 * 3600000); + const coveredH = (anchor.x - windowStart) / 3600000; + deltas.forEach(function (d) { + if (d.x > anchor.x - 24 * 3600000) sum24 += d.diff; + }); + signedCard('Avg / h (24 h)', coveredH >= 0.5 ? sum24 / coveredH : null, 'Mean of last 24 h changes'); + + plainCard('Peak', fmt(peak.y), 'On ' + fmtDateFull(peak.x)); + plainCard('Lowest', fmt(low.y), 'On ' + fmtDateFull(low.x)); + + const regWindow = points.filter(function (p) { return p.x >= anchor.x - 2 * 3600000; }).slice(-96); + const reg = regress(regWindow); + if (reg) { + plainCard('Forecast next hour', fmt(anchor.y + reg.slope), + 'Trend ' + fmtSigned(reg.slope) + '/h · R² = ' + reg.r2.toFixed(3)); + } else { + plainCard('Forecast next hour', '–', 'Needs at least 2 measurements'); + } + + el.innerHTML = cards.join(''); +} + /* ---------- main update ---------- */ function renderAll() { @@ -1147,6 +1483,8 @@ function renderAll() { const meta = document.getElementById('current-meta'); const elImages = document.getElementById('current-images'); const imagesMeta = document.getElementById('current-images-meta'); + const elPending = document.getElementById('current-pending'); + const pendingMeta = document.getElementById('current-pending-meta'); document.getElementById('badge-count').textContent = 'Measurements: ' + all.length; @@ -1155,6 +1493,8 @@ function renderAll() { meta.textContent = 'No measurements yet – the collector stores one every 30 minutes.'; elImages.textContent = '–'; imagesMeta.textContent = 'No image count yet'; + elPending.textContent = '–'; + pendingMeta.textContent = 'No queue count yet'; setStatus('offline', 'No data'); return; } @@ -1181,6 +1521,21 @@ function renderAll() { ' · ' + fmtDateFull(last.x); } + if (last.pending === null) { + elPending.textContent = '–'; + pendingMeta.textContent = 'No queue count yet'; + } else { + elPending.textContent = fmt(last.pending); + pendingMeta.textContent = 'Waiting in queue · exact count · ' + fmtDateFull(last.x); + } + + /* tab badges: always show the latest value of each data type */ + document.getElementById('tab-count-pages').textContent = fmt(last.y) + ' total'; + document.getElementById('tab-count-images').textContent = last.images === null + ? 'no data yet' : fmt(last.images) + ' total'; + document.getElementById('tab-count-queue').textContent = last.pending === null + ? 'no data yet' : fmt(last.pending) + ' pending'; + document.getElementById('badge-since').textContent = 'First measurement: ' + fmtDateFull(first.x); document.getElementById('badge-updated').textContent = 'Last update: ' + fmtDate(last.x); @@ -1295,19 +1650,42 @@ function renderAll() { elF.classList.add(fc >= fpoints[fpoints.length - 1].y ? 'pos' : 'neg'); document.getElementById('stat-forecast-hint').textContent = 'Trend ' + fmtSigned(reg.slope) + '/h · R² = ' + reg.r2.toFixed(3); - document.getElementById('footer').textContent = - 'Forecast: linear least-squares regression over the last ' + regWindow.length + - ' measurements (' + fmtDur(fpoints[fpoints.length - 1].x - regWindow[0].x) + - '), extrapolated +24 h · R² = ' + reg.r2.toFixed(3) + - ' · Range: ' + (range === 'all' ? 'all' : range) + ' · All statistics are computed in the browser.'; } else { document.getElementById('stat-forecast').textContent = '–'; document.getElementById('stat-forecast-hint').textContent = 'Needs at least 2 measurements'; - document.getElementById('footer').textContent = 'Forecast available once there are at least 2 measurements.'; } - /* images: same graphs, but only exact measurements (every 2 h) */ - const imageAll = toImagePoints(all); + document.getElementById('footer').textContent = + 'Showing ' + METRIC_LABELS[activeMetric] + + ' · Range: ' + (range === 'all' ? 'all data' : range) + + ' · All statistics are computed in the browser.'; + + /* queue: same graphs for the pending URLs in crawl_queue */ + const pendingAll = toPendingPoints(all); + const pendingFpoints = filterRange(pendingAll, last.x); + const pendingDeltas = computeDeltas(pendingFpoints).deltas; + const pendingSmoothY = movingAverage(pendingDeltas.map(function (d) { return d.y; }), DELTA_SMOOTH_WINDOW); + const pendingSmoothedDeltas = pendingDeltas.map(function (d, i) { + return { x: d.x, y: pendingSmoothY[i], raw: d.y, dt: d.dt, diff: d.diff }; + }); + const pendingEmpty = document.getElementById('pending-empty'); + if (!pendingAll.length) { + pendingEmpty.textContent = all.some(function (p) { return p.pending !== null; }) + ? 'No queue counts in the selected range.' + : 'No queue counts yet – they appear after the next collector run.'; + pendingEmpty.style.display = ''; + } else if (!pendingFpoints.length) { + pendingEmpty.textContent = 'No queue counts in the selected range.'; + pendingEmpty.style.display = ''; + } else { + pendingEmpty.style.display = 'none'; + } + + /* Images: estimates are optional and controlled by the Images-section toggle. */ + const imageAll = toImagePoints(all, showEstimatedImages); + document.getElementById('images-mode-note').textContent = showEstimatedImages + ? 'Exact and estimated measurements are plotted. Estimates come from a 1% table sample; this setting is saved in this browser.' + : 'Only exact measurements (counted every 2 h) are plotted. This setting is saved in this browser.'; const imageFpoints = filterRange(imageAll, last.x); const imageDeltas = computeDeltas(imageFpoints).deltas; const imageSmoothY = movingAverage(imageDeltas.map(function (d) { return d.y; }), DELTA_SMOOTH_WINDOW); @@ -1318,11 +1696,11 @@ function renderAll() { const hasImageValues = all.some(function (p) { return p.images !== null; }); if (!imageAll.length) { imagesEmpty.textContent = hasImageValues - ? 'Only estimated image counts so far – the graphs show exact counts only (every 2 h).' + ? 'Only estimated image counts so far – turn on “Include estimated image counts in graphs” to show them.' : 'No image counts yet – they appear after the next collector run with the image migration in place.'; imagesEmpty.style.display = ''; } else if (!imageFpoints.length) { - imagesEmpty.textContent = 'No exact image counts in the selected range (counted every 2 h).'; + imagesEmpty.textContent = 'No image counts in the selected range.'; imagesEmpty.style.display = ''; } else { imagesEmpty.style.display = 'none'; @@ -1351,10 +1729,25 @@ function renderAll() { gradTop: 'rgba(180,122,255,0.28)', gradBottom: 'rgba(180,122,255,0.02)' }); + renderCharts({ + prefix: 'pending', + points: pendingFpoints, + deltas: pendingDeltas, + smoothedDeltas: pendingSmoothedDeltas, + unit: 'Pending', + accent: ACCENT3, + accentBar: 'rgba(245,166,35,0.65)', + gradTop: 'rgba(245,166,35,0.28)', + gradBottom: 'rgba(245,166,35,0.02)' + }); renderRecords(deltas, 'records', 'Pages'); renderRecords(imageDeltas, 'records-images', 'Images'); + renderRecords(pendingDeltas, 'records-pending', 'Pending'); renderHeatmap(all, 'heatmap', 'Pages'); renderHeatmap(imageAll, 'heatmap-images', 'Images'); + renderHeatmap(pendingAll, 'heatmap-pending', 'Pending'); + renderSummary('summary-images', imageAll, imageFpoints); + renderSummary('summary-queue', pendingAll, pendingFpoints); } async function update() { @@ -1380,6 +1773,13 @@ setInterval(function () { update(); setInterval(update, REFRESH_S * 1000); + +// Restore the last selected tab (pages / images / queue). +(function () { + let saved = 'pages'; + try { saved = localStorage.getItem('js-metric') || 'pages'; } catch (e) {} + setMetric(saved, false); +})(); @@ -1427,6 +1827,10 @@ ESTIMATE_IMAGE_SQL = ( percent=IMAGE_SAMPLE_PERCENT, ) +# URLs waiting in the crawler queue. Nutzt den Index +# (status, priority, discovered_at); kein Sampling, der Wert ist exakt. +PENDING_QUEUE_SQL = "SELECT COUNT(*) FROM crawl_queue WHERE status = 'pending';" + def _psql_scalar(sql): """Eine einzelne psql-Zeile ("count|images") abholen.""" @@ -1518,12 +1922,32 @@ def get_stats(exact_images): return count, images, False +def get_pending(): + """ + Anzahl der URLs mit status='pending' in der crawl_queue. + + Ist die Tabelle (noch) nicht erreichbar, wird None geliefert - die + Messung wird trotzdem gespeichert, nur ohne Queue-Wert. + """ + try: + return int(_psql_scalar(PENDING_QUEUE_SQL).strip()) + except subprocess.CalledProcessError as exc: + detail = ((exc.stderr or "").strip().splitlines() or [str(exc)])[-1] + print(f"Queue-Statistik nicht verfügbar ({detail}); zähle nur Seiten/Bilder.") + return None + except (TypeError, ValueError): + # Leere/überraschende psql-Ausgabe darf die übrigen Werte nicht kosten. + print("Queue-Statistik nicht verfügbar (unerwartete psql-Ausgabe).") + return None + + def collect(): while True: try: data = load_data() exact = images_exact_due(data) count, images, images_exact = get_stats(exact) + pending = get_pending() if images is None: # Stichprobe ohne Treffer (praktisch nur bei sehr großen @@ -1538,6 +1962,7 @@ def collect(): "count": count, "images": images, "images_exact": images_exact, + "pending": pending, } data.append(entry) @@ -1546,7 +1971,8 @@ def collect(): print( f"[{entry['timestamp']}] " f"Pages: {count} · Images: {images} " - f"({'exakt' if images_exact else 'geschätzt'})" + f"({'exakt' if images_exact else 'geschätzt'}) " + f"· Pending: {pending if pending is not None else 'n/a'}" ) except Exception as e: diff --git a/web/test_stats.py b/web/test_stats.py index 5f72473..5e19312 100644 --- a/web/test_stats.py +++ b/web/test_stats.py @@ -117,6 +117,59 @@ class GetStatsTest(unittest.TestCase): "SELECT COUNT(*) FROM pages;") +class GetPendingTest(unittest.TestCase): + + @mock.patch.object(stats_app, "_psql_scalar") + def test_counts_pending_queue_rows(self, psql): + psql.return_value = "1384711" + self.assertEqual(stats_app.get_pending(), 1384711) + self.assertIn("FROM crawl_queue", stats_app.PENDING_QUEUE_SQL) + self.assertIn("status = 'pending'", stats_app.PENDING_QUEUE_SQL) + + @mock.patch.object(stats_app, "_psql_scalar") + def test_unavailable_queue_returns_none(self, psql): + psql.side_effect = subprocess.CalledProcessError( + 1, ["psql"], stderr='ERROR: relation "crawl_queue" does not exist' + ) + self.assertIsNone(stats_app.get_pending()) + + @mock.patch.object(stats_app, "_psql_scalar") + def test_unexpected_output_returns_none(self, psql): + psql.return_value = "" + self.assertIsNone(stats_app.get_pending()) + + +class CollectTest(unittest.TestCase): + + def _run_collect(self, get_stats, get_pending): + with mock.patch.object(stats_app, "load_data", return_value=[]), \ + mock.patch.object(stats_app, "save_data") as save_data, \ + mock.patch.object(stats_app, "get_stats", get_stats), \ + mock.patch.object(stats_app, "get_pending", get_pending), \ + mock.patch.object(stats_app, "time") as time_mock: + time_mock.sleep.side_effect = KeyboardInterrupt + with self.assertRaises(KeyboardInterrupt): + stats_app.collect() + return save_data.call_args[0][0][0] + + def test_pending_is_stored_with_every_sample(self): + entry = self._run_collect( + mock.Mock(return_value=(1000, 2000, True)), + mock.Mock(return_value=4711), + ) + self.assertEqual(entry["pending"], 4711) + self.assertEqual(entry["count"], 1000) + self.assertEqual(entry["images"], 2000) + + def test_missing_queue_does_not_break_the_sample(self): + entry = self._run_collect( + mock.Mock(return_value=(1000, 2000, False)), + mock.Mock(return_value=None), + ) + self.assertIsNone(entry["pending"]) + self.assertEqual(entry["count"], 1000) + + class StatsPageTest(unittest.TestCase): def test_page_contains_image_counter(self): @@ -159,15 +212,27 @@ class StatsPageTest(unittest.TestCase): self.assertIn('id="heatmap-images"', html) self.assertIn('id="images-empty"', html) - # the image series skips entries without an image count (old data) - # and only plots exact measurements - no estimates between the 2 h counts - self.assertIn("function toImagePoints(points)", html) + # Image estimates are excluded by default and can be included with the persistent toggle. + self.assertIn('id="show-estimated-images"', html) + self.assertIn('type="checkbox" id="show-estimated-images"', html) + self.assertIn("Include estimated image counts in graphs", html) + self.assertIn("function toImagePoints(points, includeEstimated)", html) self.assertIn("p.images !== null && p.images !== undefined", html) - self.assertIn(".filter(function (p) { return p.imagesExact; })", html) - self.assertIn("only exact measurements are plotted (no estimates)", html) + self.assertIn("includeEstimated || p.imagesExact", html) + self.assertIn("localStorage.getItem('js-show-estimated-images')", html) + self.assertIn("localStorage.setItem('js-show-estimated-images'", html) + self.assertIn("toImagePoints(all, showEstimatedImages)", html) self.assertIn("Only exact measurements (counted every 2 h) are plotted", html) + self.assertIn("Exact and estimated measurements are plotted", html) self.assertIn("Only estimated image counts so far", html) - self.assertIn("No exact image counts in the selected range", html) + self.assertIn("No image counts in the selected range", html) + # The headline counter stays the latest collected value; only graphs toggle estimates. + self.assertIn("elImages.textContent = fmt(last.images)", html) + # Images estimates affect graphs only; other series keep their own points. + self.assertIn("renderCharts({\n prefix: '',\n points: fpoints", html) + self.assertIn("renderCharts({\n prefix: 'pending',\n points: pendingFpoints", html) + self.assertIn("renderHeatmap(imageAll, 'heatmap-images', 'Images')", html) + self.assertIn("renderRecords(imageDeltas, 'records-images', 'Images')", html) # existing page chart ids must stay lowercase (mainChart, not MainChart) self.assertIn("function chartId(prefix, suffix)", html) self.assertIn("chartId(prefix, 'MainChart')", html) @@ -182,6 +247,79 @@ class StatsPageTest(unittest.TestCase): self.assertIn("renderHeatmap(all, 'heatmap', 'Pages')", html) self.assertIn("renderHeatmap(imageAll, 'heatmap-images', 'Images')", html) + def test_page_has_metric_tabs_with_one_panel_each(self): + client = stats_app.app.test_client() + response = client.get("/") + self.assertEqual(response.status_code, 200) + html = response.get_data(as_text=True) + + # three equally styled tab buttons (pages / images / queue) with icons + self.assertIn('class="metric-btn', html) + for metric in ("pages", "images", "queue"): + self.assertIn('data-metric="' + metric + '"', html) + self.assertIn("metric-icon", html) + self.assertIn("📄", html) + self.assertIn("🖼️", html) + self.assertIn("⏳", html) + + # every metric has its own panel; only the initial one is active + self.assertEqual(html.count('class="metric-panel active" data-metric="pages"'), 1) + for metric in ("images", "queue"): + self.assertIn('class="metric-panel" data-metric="' + metric + '"', html) + self.assertIn('id="summary-images"', html) + self.assertIn('id="summary-queue"', html) + + # switching tabs hides the other panels and is remembered + self.assertIn("function setMetric(next, persist)", html) + self.assertIn("panel.classList.toggle('active', panel.dataset.metric === activeMetric)", html) + self.assertIn("localStorage.setItem('js-metric', activeMetric)", html) + self.assertIn("localStorage.getItem('js-metric')", html) + + # hidden panels must not create Chart.js instances (canvas has 0 px) + self.assertIn("function isVisible(el)", html) + self.assertIn("if (!target || !isVisible(target)) return;", html) + + # summary cards are generated for the non-page tabs + self.assertIn("function renderSummary(containerId, all, points)", html) + self.assertIn("renderSummary('summary-images', imageAll, imageFpoints)", html) + self.assertIn("renderSummary('summary-queue', pendingAll, pendingFpoints)", html) + + # exports follow the selected tab + self.assertIn("function exportSeries()", html) + self.assertIn("return { name: 'jonassearch_images'", html) + self.assertIn("return { name: 'jonassearch_queue'", html) + + def test_page_contains_pending_queue_counter_and_graphs(self): + client = stats_app.app.test_client() + response = client.get("/") + self.assertEqual(response.status_code, 200) + html = response.get_data(as_text=True) + + self.assertIn('id="current-pending"', html) + self.assertIn('id="current-pending-meta"', html) + self.assertIn("Queue – all graphs", html) + for canvas in ( + "pendingMainChart", + "pendingDeltaChart", + "pendingCumChart", + "pendingHourlyChart", + "pendingHodChart", + ): + self.assertIn('id="' + canvas + '"', html) + self.assertIn('id="records-pending"', html) + self.assertIn('id="heatmap-pending"', html) + self.assertIn('id="pending-empty"', html) + + # pending values are parsed and plotted without sampling/filtering + self.assertIn("function toPendingPoints(points)", html) + self.assertIn("p.pending !== null && p.pending !== undefined", html) + self.assertIn("filterRange(pendingAll, last.x)", html) + self.assertIn("unit: 'Pending'", html) + self.assertIn("renderRecords(pendingDeltas, 'records-pending', 'Pending')", html) + self.assertIn("renderHeatmap(pendingAll, 'heatmap-pending', 'Pending')", html) + self.assertIn("No queue counts yet", html) + self.assertIn("No queue counts in the selected range.", html) + if __name__ == "__main__": unittest.main(verbosity=2)