Serverjonas/search
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

make the stats mor ereadaable

Browse Source
This commit is contained in:
jhkaktus
2026-10-21 18:32:31 +02:00
parent 3ebfeca7cc
commit 530c2b1ecc
3 changed files with 619 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/app.py
+465 -39
View File
@@ -147,6 +147,61 @@ HTML = r"""
.ranges { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 18px; }
.image-options { display: flex; align-items: center; gap: 9px; margin: 0 0 12px; color: var(--muted); font-size: 0.88rem; }
.image-options input { accent-color: var(--accent); }
.image-options label { cursor: pointer; }
.image-options-hint { color: var(--muted); font-size: 0.78rem; margin-bottom: 14px; }
.metric-tabs {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(160px, 1fr));
gap: 10px;
margin: 22px 0 16px;
}
.metric-btn {
display: flex;
align-items: center;
gap: 12px;
text-align: left;
background: var(--card);
border: 1px solid var(--border);
color: var(--muted);
border-radius: 15px;
padding: 13px 16px;
font-size: 1rem;
font-weight: 600;
font-family: inherit;
cursor: pointer;
transition: border-color .15s, color .15s, background .15s;
}
.metric-btn:hover { border-color: var(--accent); color: var(--text); }
.metric-btn.active {
color: var(--text);
border-color: var(--accent);
background: linear-gradient(180deg, rgba(79,140,255,0.18), rgba(79,140,255,0.04));
box-shadow: inset 0 0 0 1px rgba(79,140,255,0.35);
}
.metric-icon { font-size: 1.6rem; line-height: 1; }
.metric-name { display: block; }
.metric-count {
display: block;
font-size: 0.78rem;
font-weight: 400;
color: var(--muted);
margin-top: 2px;
}
.metric-panel { display: none; }
.metric-panel.active { display: block; }
.section-title-first { margin-top: 6px; }
.range-btn {
background: var(--card);
border: 1px solid var(--border);
@@ -253,6 +308,8 @@ HTML = r"""
.section-title { font-weight: 600; margin: 30px 0 12px; font-size: 1.05rem; }
@media (max-width: 600px) {
.metric-btn { padding: 10px 12px; font-size: 0.9rem; gap: 9px; }
.metric-icon { font-size: 1.3rem; }
.current { font-size: 2.4rem; }
.container { padding: 20px 14px 40px; }
.chart-container { padding: 14px; }
@@ -266,7 +323,7 @@ HTML = r"""
<div class="container">
<header>
<h1>JonasSearch</h1>
<div class="subtitle">Indexed Pages – Live Statistics</div>
<div class="subtitle" id="page-subtitle">Pages – Live Statistics</div>
<div class="header-right">
<a class="btn" id="btn-csv" href="#">Export CSV</a>
<a class="btn" id="btn-json" href="#">Export JSON</a>
@@ -274,19 +331,28 @@ HTML = r"""
</div>
</header>
<div class="current-row">
<div>
<div class="current">
<span id="current">–</span><span class="unit">Pages</span>
</div>
<div class="current-meta" id="current-meta">Loading…</div>
</div>
<div>
<div class="current">
<span id="current-images">–</span><span class="unit">Images</span>
</div>
<div class="current-meta" id="current-images-meta">Loading…</div>
</div>
<div class="metric-tabs" role="tablist" aria-label="Data type">
<button class="metric-btn active" data-metric="pages" role="tab" aria-selected="true">
<span class="metric-icon" aria-hidden="true">📄</span>
<span>
<span class="metric-name">Pages</span>
<span class="metric-count" id="tab-count-pages">– total</span>
</span>
</button>
<button class="metric-btn" data-metric="images" role="tab" aria-selected="false">
<span class="metric-icon" aria-hidden="true">🖼️</span>
<span>
<span class="metric-name">Images</span>
<span class="metric-count" id="tab-count-images">– total</span>
</span>
</button>
<button class="metric-btn" data-metric="queue" role="tab" aria-selected="false">
<span class="metric-icon" aria-hidden="true">⏳</span>
<span>
<span class="metric-name">Queue</span>
<span class="metric-count" id="tab-count-queue">– pending</span>
</span>
</button>
</div>
<div class="badges">
@@ -305,6 +371,16 @@ HTML = r"""
<button class="range-btn" data-range="6h">Last 6 h</button>
</div>
<div class="metric-panel active" data-metric="pages" role="tabpanel">
<div class="current-row">
<div>
<div class="current">
<span id="current">–</span><span class="unit">Pages</span>
</div>
<div class="current-meta" id="current-meta">Loading…</div>
</div>
</div>
<div class="grid">
<div class="card">
<div class="label">Growth / hour</div>
@@ -406,21 +482,39 @@ HTML = r"""
<div id="heatmap"></div>
</div>
<div class="section-title">Images – all graphs</div>
<div class="chart-sub">Only exact measurements (counted every 2 h) are plotted – the estimates in between are excluded.</div>
</div>
<div class="metric-panel" data-metric="images" role="tabpanel">
<div class="current-row">
<div>
<div class="current">
<span id="current-images">–</span><span class="unit">Images</span>
</div>
<div class="current-meta" id="current-images-meta">Loading…</div>
</div>
</div>
<div class="grid" id="summary-images"></div>
<div class="section-title section-title-first">Images – all graphs</div>
<div class="image-options">
<input type="checkbox" id="show-estimated-images">
<label for="show-estimated-images">Include estimated image counts in graphs</label>
</div>
<div class="image-options-hint" id="images-mode-note">Only exact measurements (counted every 2 h) are plotted. This setting is saved in this browser.</div>
<div class="chart-sub" id="images-empty" style="display:none">
No image counts yet – they appear after the next collector run with the image migration in place.
</div>
<div class="chart-container">
<div class="chart-title">Images over time</div>
<div class="chart-sub">Image URLs found while crawling · exact count every 2 h · only exact measurements are plotted (no estimates) · dashed line = trend &amp; 24 h forecast</div>
<div class="chart-sub">Image URLs found while crawling · dashed line = trend &amp; 24 h forecast</div>
<div class="chart-box"><canvas id="imagesMainChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Change per 5 minutes</div>
<div class="chart-sub">Normalized to a 5-minute interval · spans between two exact counts (2 h) · grey = raw · colored = smoothed (moving average over 5 measurements)</div>
<div class="chart-sub">Normalized to a 5-minute interval between available image measurements · grey = raw · colored = smoothed (moving average over 5 measurements)</div>
<div class="chart-box small"><canvas id="imagesDeltaChart"></canvas></div>
</div>
@@ -455,6 +549,70 @@ HTML = r"""
<div id="heatmap-images"></div>
</div>
</div>
<div class="metric-panel" data-metric="queue" role="tabpanel">
<div class="current-row">
<div>
<div class="current">
<span id="current-pending">–</span><span class="unit">Pending</span>
</div>
<div class="current-meta" id="current-pending-meta">Loading…</div>
</div>
</div>
<div class="grid" id="summary-queue"></div>
<div class="section-title section-title-first">Queue – all graphs</div>
<div class="chart-sub">URLs waiting in crawl_queue (status = 'pending'). Counted with every 30-minute sample (exact COUNT). Falling graphs mean the crawler is working through the queue.</div>
<div class="chart-sub" id="pending-empty" style="display:none">
No queue counts yet – they appear after the next collector run.
</div>
<div class="chart-container">
<div class="chart-title">Pending URLs over time</div>
<div class="chart-sub">Queued URLs not yet crawled · dashed line = trend &amp; 24 h forecast</div>
<div class="chart-box"><canvas id="pendingMainChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Change per 5 minutes</div>
<div class="chart-sub">Normalized to a 5-minute interval · negative = the crawler worked the queue down · grey = raw · colored = smoothed (moving average over 5 measurements)</div>
<div class="chart-box small"><canvas id="pendingDeltaChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Cumulative net change</div>
<div class="chart-sub">Total queued URLs gained/lost since the start of the selected range · linear time axis</div>
<div class="chart-box small"><canvas id="pendingCumChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Queue change per hour (last 48 h)</div>
<div class="chart-sub">Sum of changes per hour bucket · green = queue grew, red = queue shrank</div>
<div class="chart-box small"><canvas id="pendingHourlyChart"></canvas></div>
</div>
<div class="chart-row">
<div class="chart-container">
<div class="chart-title">Queue change by hour of day</div>
<div class="chart-sub">Average change per 5 min, grouped by hour of day (whole history)</div>
<div class="chart-box small"><canvas id="pendingHodChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Records – biggest queue changes</div>
<div class="chart-sub">Largest gains and losses between two consecutive measurements (selected range)</div>
<div id="records-pending"></div>
</div>
</div>
<div class="section-title">Activity heatmap – queue change per hour of day</div>
<div class="chart-container heatmap">
<div class="chart-sub">Rows = days · columns = hours (0–23) · green = queue grew, red = queue shrank, grey = no data</div>
<div id="heatmap-pending"></div>
</div>
</div>
<div class="footer" id="footer"></div>
</div>
@@ -467,6 +625,7 @@ if (window.Chart && Chart.registerables) {
const ACCENT = '#4f8cff';
const ACCENT2 = '#b47aff';
const ACCENT3 = '#f5a623';
const UP = '#2ecc71';
const DOWN = '#ff6b6b';
const FALLBACK_INTERVAL_MS = 15 * 60 * 1000;
@@ -476,6 +635,8 @@ const REFRESH_S = 30;
let rawValues = [];
let intervalMs = FALLBACK_INTERVAL_MS;
let range = 'all';
let activeMetric = 'pages';
let showEstimatedImages = false;
let charts = {};
let refreshLeft = REFRESH_S;
@@ -603,27 +764,36 @@ function parsePoints(values) {
.map(function (v) {
const images = (v.images === null || v.images === undefined)
? null : Number(v.images);
const pending = (v.pending === null || v.pending === undefined)
? null : Number(v.pending);
return {
x: new Date(v.timestamp).getTime(),
y: v.count,
images: Number.isFinite(images) ? images : null,
imagesExact: v.images_exact === true
imagesExact: v.images_exact === true,
pending: Number.isFinite(pending) ? pending : null
};
})
.filter(function (p) { return !isNaN(p.x); })
.sort(function (a, b) { return a.x - b.x; });
}
// Nur exakte Bildzählungen (alle 2 h). Die dazwischen geschätzten Werte
// werden in keinem Diagramm dargestellt, damit alle Graphen ausschließlich
// echte Messpunkte zeigen.
function toImagePoints(points) {
// Estimates are excluded by default; the Images section toggle can include them.
function toImagePoints(points, includeEstimated) {
return (points || [])
.filter(function (p) { return p.images !== null && p.images !== undefined; })
.filter(function (p) { return p.imagesExact; })
.filter(function (p) { return includeEstimated || p.imagesExact; })
.map(function (p) { return { x: p.x, y: p.images }; });
}
// Pending queue entries are counted with every sample (exact COUNT), so
// no filtering beyond "value present" is needed.
function toPendingPoints(points) {
return (points || [])
.filter(function (p) { return p.pending !== null && p.pending !== undefined; })
.map(function (p) { return { x: p.x, y: p.pending }; });
}
const RANGE_SPANS = {
'6h': 6 * 3600000,
'24h': 24 * 3600000,
@@ -673,6 +843,17 @@ document.getElementById('btn-theme').addEventListener('click', function () {
applyTheme(document.documentElement.dataset.theme === 'light' ? 'dark' : 'light');
});
(function () {
const toggle = document.getElementById('show-estimated-images');
try { showEstimatedImages = localStorage.getItem('js-show-estimated-images') === 'true'; } catch (e) {}
toggle.checked = showEstimatedImages;
toggle.addEventListener('change', function () {
showEstimatedImages = toggle.checked;
try { localStorage.setItem('js-show-estimated-images', String(showEstimatedImages)); } catch (e) {}
if (rawValues.length) renderAll();
});
})();
(function () {
let saved = null;
try { saved = localStorage.getItem('js-theme'); } catch (e) {}
@@ -680,6 +861,44 @@ document.getElementById('btn-theme').addEventListener('click', function () {
document.getElementById('btn-theme').textContent = saved === 'light' ? '🌙 Dark' : '☀️ Light';
})();
/* ---------- metric tabs (pages / images / queue) ---------- */
const METRIC_LABELS = { pages: 'Pages', images: 'Images', queue: 'Queue' };
// Hidden panels must not render Chart.js instances: a canvas inside a
// display:none container measures 0 px and breaks on later resize.
function isVisible(el) {
const panel = el.closest('.metric-panel');
return !panel || panel.classList.contains('active');
}
function setMetric(next, persist) {
activeMetric = METRIC_LABELS[next] ? next : 'pages';
document.querySelectorAll('.metric-btn').forEach(function (btn) {
const on = btn.dataset.metric === activeMetric;
btn.classList.toggle('active', on);
btn.setAttribute('aria-selected', on ? 'true' : 'false');
});
document.querySelectorAll('.metric-panel').forEach(function (panel) {
panel.classList.toggle('active', panel.dataset.metric === activeMetric);
});
const label = METRIC_LABELS[activeMetric];
document.getElementById('page-subtitle').textContent = label + ' – Live Statistics';
document.title = 'JonasSearch – ' + label;
if (persist !== false) {
try { localStorage.setItem('js-metric', activeMetric); } catch (e) {}
}
if (rawValues.length) renderAll();
}
document.querySelectorAll('.metric-btn').forEach(function (btn) {
btn.addEventListener('click', function () { setMetric(btn.dataset.metric); });
});
/* ---------- range buttons ---------- */
document.querySelectorAll('.range-btn').forEach(function (btn) {
@@ -702,16 +921,50 @@ function download(name, text, mime) {
URL.revokeObjectURL(a.href);
}
// Export what the selected tab shows, so the CSV/JSON match the graphs.
function exportSeries() {
const all = parsePoints(rawValues);
if (activeMetric === 'images') {
const values = all
.filter(function (p) { return p.images !== null; })
.map(function (p) {
return { timestamp: new Date(p.x).toISOString(), images: p.images, images_exact: p.imagesExact };
});
return { name: 'jonassearch_images', values: values };
}
if (activeMetric === 'queue') {
const values = all
.filter(function (p) { return p.pending !== null; })
.map(function (p) {
return { timestamp: new Date(p.x).toISOString(), pending: p.pending };
});
return { name: 'jonassearch_queue', values: values };
}
return { name: 'jonassearch_pages', values: rawValues };
}
document.getElementById('btn-csv').addEventListener('click', function (e) {
e.preventDefault();
const lines = ['timestamp;count'];
rawValues.forEach(function (v) { lines.push(v.timestamp + ';' + v.count); });
download('jonassearch_pages.csv', lines.join('\n'), 'text/csv');
const series = exportSeries();
const keys = series.values.length ? Object.keys(series.values[0]) : ['timestamp'];
const lines = [keys.join(';')];
series.values.forEach(function (row) {
lines.push(keys.map(function (k) { return row[k] === undefined ? '' : row[k]; }).join(';'));
});
download(series.name + '.csv', lines.join('\n'), 'text/csv');
});
document.getElementById('btn-json').addEventListener('click', function (e) {
e.preventDefault();
download('jonassearch_pages.json', JSON.stringify({ values: rawValues }, null, 2), 'application/json');
const series = exportSeries();
download(
series.name + '.json',
JSON.stringify({ metric: activeMetric, values: series.values }, null, 2),
'application/json'
);
});
/* ---------- chart helpers ---------- */
@@ -767,6 +1020,9 @@ function gradientFill(colorTop, colorBottom) {
/* ---------- charts ---------- */
function renderCharts(cfg) {
const target = document.getElementById(chartId(cfg.prefix, 'MainChart'));
if (!target || !isVisible(target)) return;
const prefix = cfg.prefix;
const points = cfg.points;
const deltas = cfg.deltas;
@@ -1052,6 +1308,7 @@ function renderCharts(cfg) {
function renderRecords(deltas, id, unit) {
const el = document.getElementById(id);
if (!el || !isVisible(el)) return;
if (deltas.length < 2) {
el.innerHTML = '<div class="chart-sub">Not enough data yet.</div>';
return;
@@ -1078,6 +1335,7 @@ function renderRecords(deltas, id, unit) {
function renderHeatmap(points, id, unit) {
const el = document.getElementById(id);
if (!el || !isVisible(el)) return;
if (!points.length) { el.innerHTML = ''; return; }
function localDayKey(d) {
@@ -1137,6 +1395,84 @@ function renderHeatmap(points, id, unit) {
el.innerHTML = html;
}
/* ---------- summary cards for images / queue ---------- */
// The pages tab keeps its static card grid (ids like stat-delta-6h);
// images and queue get the same key numbers generated from their series.
function renderSummary(containerId, all, points) {
const el = document.getElementById(containerId);
if (!el || !isVisible(el)) return;
if (!points.length) {
el.innerHTML = '<div class="chart-sub">No data in the selected range yet.</div>';
return;
}
const anchor = all[all.length - 1];
const first = points[0];
const hours = (anchor.x - first.x) / 3600000;
const perHour = hours > 0 ? (anchor.y - first.y) / hours : null;
const deltas = computeDeltas(points).deltas;
let peak = all[0], low = all[0];
all.forEach(function (p) {
if (p.y > peak.y) peak = p;
if (p.y < low.y) low = p;
});
const cards = [];
function signedCard(label, value, hint) {
const cls = value === null ? '' : (value >= 0 ? 'pos' : 'neg');
cards.push('<div class="card"><div class="label">' + label +
'</div><div class="value ' + cls + '">' +
(value === null ? '–' : fmtSigned(value)) +
'</div><div class="hint">' + hint + '</div></div>');
}
function plainCard(label, value, hint) {
cards.push('<div class="card"><div class="label">' + label +
'</div><div class="value">' + value +
'</div><div class="hint">' + hint + '</div></div>');
}
signedCard('Growth / hour', perHour, 'Average over full history');
signedCard('Growth / day', perHour === null ? null : perHour * 24, 'Average over full history');
const windows = [
['6 h', 6 * 3600000, Math.max(3 * intervalMs, 3600000)],
['24 h', 24 * 3600000, Math.max(6 * intervalMs, 3 * 3600000)],
['7 days', 7 * 24 * 3600000, Math.max(12 * intervalMs, 6 * 3600000)]
];
windows.forEach(function (win) {
const base = closestBefore(all, anchor.x - win[1], win[2]);
signedCard('Growth ' + win[0], base ? anchor.y - base.y : null,
base ? 'vs. measurement ' + fmtDur(anchor.x - base.x) + ' ago' : 'Not enough data yet');
});
let sum24 = 0;
const windowStart = Math.max(first.x, anchor.x - 24 * 3600000);
const coveredH = (anchor.x - windowStart) / 3600000;
deltas.forEach(function (d) {
if (d.x > anchor.x - 24 * 3600000) sum24 += d.diff;
});
signedCard('Avg / h (24 h)', coveredH >= 0.5 ? sum24 / coveredH : null, 'Mean of last 24 h changes');
plainCard('Peak', fmt(peak.y), 'On ' + fmtDateFull(peak.x));
plainCard('Lowest', fmt(low.y), 'On ' + fmtDateFull(low.x));
const regWindow = points.filter(function (p) { return p.x >= anchor.x - 2 * 3600000; }).slice(-96);
const reg = regress(regWindow);
if (reg) {
plainCard('Forecast next hour', fmt(anchor.y + reg.slope),
'Trend ' + fmtSigned(reg.slope) + '/h · R² = ' + reg.r2.toFixed(3));
} else {
plainCard('Forecast next hour', '–', 'Needs at least 2 measurements');
}
el.innerHTML = cards.join('');
}
/* ---------- main update ---------- */
function renderAll() {
@@ -1147,6 +1483,8 @@ function renderAll() {
const meta = document.getElementById('current-meta');
const elImages = document.getElementById('current-images');
const imagesMeta = document.getElementById('current-images-meta');
const elPending = document.getElementById('current-pending');
const pendingMeta = document.getElementById('current-pending-meta');
document.getElementById('badge-count').textContent = 'Measurements: ' + all.length;
@@ -1155,6 +1493,8 @@ function renderAll() {
meta.textContent = 'No measurements yet – the collector stores one every 30 minutes.';
elImages.textContent = '–';
imagesMeta.textContent = 'No image count yet';
elPending.textContent = '–';
pendingMeta.textContent = 'No queue count yet';
setStatus('offline', 'No data');
return;
}
@@ -1181,6 +1521,21 @@ function renderAll() {
' · ' + fmtDateFull(last.x);
}
if (last.pending === null) {
elPending.textContent = '–';
pendingMeta.textContent = 'No queue count yet';
} else {
elPending.textContent = fmt(last.pending);
pendingMeta.textContent = 'Waiting in queue · exact count · ' + fmtDateFull(last.x);
}
/* tab badges: always show the latest value of each data type */
document.getElementById('tab-count-pages').textContent = fmt(last.y) + ' total';
document.getElementById('tab-count-images').textContent = last.images === null
? 'no data yet' : fmt(last.images) + ' total';
document.getElementById('tab-count-queue').textContent = last.pending === null
? 'no data yet' : fmt(last.pending) + ' pending';
document.getElementById('badge-since').textContent = 'First measurement: ' + fmtDateFull(first.x);
document.getElementById('badge-updated').textContent = 'Last update: ' + fmtDate(last.x);
@@ -1295,19 +1650,42 @@ function renderAll() {
elF.classList.add(fc >= fpoints[fpoints.length - 1].y ? 'pos' : 'neg');
document.getElementById('stat-forecast-hint').textContent =
'Trend ' + fmtSigned(reg.slope) + '/h · R² = ' + reg.r2.toFixed(3);
document.getElementById('footer').textContent =
'Forecast: linear least-squares regression over the last ' + regWindow.length +
' measurements (' + fmtDur(fpoints[fpoints.length - 1].x - regWindow[0].x) +
'), extrapolated +24 h · R² = ' + reg.r2.toFixed(3) +
' · Range: ' + (range === 'all' ? 'all' : range) + ' · All statistics are computed in the browser.';
} else {
document.getElementById('stat-forecast').textContent = '–';
document.getElementById('stat-forecast-hint').textContent = 'Needs at least 2 measurements';
document.getElementById('footer').textContent = 'Forecast available once there are at least 2 measurements.';
}
/* images: same graphs, but only exact measurements (every 2 h) */
const imageAll = toImagePoints(all);
document.getElementById('footer').textContent =
'Showing ' + METRIC_LABELS[activeMetric] +
' · Range: ' + (range === 'all' ? 'all data' : range) +
' · All statistics are computed in the browser.';
/* queue: same graphs for the pending URLs in crawl_queue */
const pendingAll = toPendingPoints(all);
const pendingFpoints = filterRange(pendingAll, last.x);
const pendingDeltas = computeDeltas(pendingFpoints).deltas;
const pendingSmoothY = movingAverage(pendingDeltas.map(function (d) { return d.y; }), DELTA_SMOOTH_WINDOW);
const pendingSmoothedDeltas = pendingDeltas.map(function (d, i) {
return { x: d.x, y: pendingSmoothY[i], raw: d.y, dt: d.dt, diff: d.diff };
});
const pendingEmpty = document.getElementById('pending-empty');
if (!pendingAll.length) {
pendingEmpty.textContent = all.some(function (p) { return p.pending !== null; })
? 'No queue counts in the selected range.'
: 'No queue counts yet – they appear after the next collector run.';
pendingEmpty.style.display = '';
} else if (!pendingFpoints.length) {
pendingEmpty.textContent = 'No queue counts in the selected range.';
pendingEmpty.style.display = '';
} else {
pendingEmpty.style.display = 'none';
}
/* Images: estimates are optional and controlled by the Images-section toggle. */
const imageAll = toImagePoints(all, showEstimatedImages);
document.getElementById('images-mode-note').textContent = showEstimatedImages
? 'Exact and estimated measurements are plotted. Estimates come from a 1% table sample; this setting is saved in this browser.'
: 'Only exact measurements (counted every 2 h) are plotted. This setting is saved in this browser.';
const imageFpoints = filterRange(imageAll, last.x);
const imageDeltas = computeDeltas(imageFpoints).deltas;
const imageSmoothY = movingAverage(imageDeltas.map(function (d) { return d.y; }), DELTA_SMOOTH_WINDOW);
@@ -1318,11 +1696,11 @@ function renderAll() {
const hasImageValues = all.some(function (p) { return p.images !== null; });
if (!imageAll.length) {
imagesEmpty.textContent = hasImageValues
? 'Only estimated image counts so far – the graphs show exact counts only (every 2 h).'
? 'Only estimated image counts so far – turn on “Include estimated image counts in graphs” to show them.'
: 'No image counts yet – they appear after the next collector run with the image migration in place.';
imagesEmpty.style.display = '';
} else if (!imageFpoints.length) {
imagesEmpty.textContent = 'No exact image counts in the selected range (counted every 2 h).';
imagesEmpty.textContent = 'No image counts in the selected range.';
imagesEmpty.style.display = '';
} else {
imagesEmpty.style.display = 'none';
@@ -1351,10 +1729,25 @@ function renderAll() {
gradTop: 'rgba(180,122,255,0.28)',
gradBottom: 'rgba(180,122,255,0.02)'
});
renderCharts({
prefix: 'pending',
points: pendingFpoints,
deltas: pendingDeltas,
smoothedDeltas: pendingSmoothedDeltas,
unit: 'Pending',
accent: ACCENT3,
accentBar: 'rgba(245,166,35,0.65)',
gradTop: 'rgba(245,166,35,0.28)',
gradBottom: 'rgba(245,166,35,0.02)'
});
renderRecords(deltas, 'records', 'Pages');
renderRecords(imageDeltas, 'records-images', 'Images');
renderRecords(pendingDeltas, 'records-pending', 'Pending');
renderHeatmap(all, 'heatmap', 'Pages');
renderHeatmap(imageAll, 'heatmap-images', 'Images');
renderHeatmap(pendingAll, 'heatmap-pending', 'Pending');
renderSummary('summary-images', imageAll, imageFpoints);
renderSummary('summary-queue', pendingAll, pendingFpoints);
}
async function update() {
@@ -1380,6 +1773,13 @@ setInterval(function () {
update();
setInterval(update, REFRESH_S * 1000);
// Restore the last selected tab (pages / images / queue).
(function () {
let saved = 'pages';
try { saved = localStorage.getItem('js-metric') || 'pages'; } catch (e) {}
setMetric(saved, false);
})();
</script>
</body>
@@ -1427,6 +1827,10 @@ ESTIMATE_IMAGE_SQL = (
percent=IMAGE_SAMPLE_PERCENT,
)
# URLs waiting in the crawler queue. Nutzt den Index
# (status, priority, discovered_at); kein Sampling, der Wert ist exakt.
PENDING_QUEUE_SQL = "SELECT COUNT(*) FROM crawl_queue WHERE status = 'pending';"
def _psql_scalar(sql):
"""Eine einzelne psql-Zeile ("count|images") abholen."""
@@ -1518,12 +1922,32 @@ def get_stats(exact_images):
return count, images, False
def get_pending():
"""
Anzahl der URLs mit status='pending' in der crawl_queue.
Ist die Tabelle (noch) nicht erreichbar, wird None geliefert - die
Messung wird trotzdem gespeichert, nur ohne Queue-Wert.
"""
try:
return int(_psql_scalar(PENDING_QUEUE_SQL).strip())
except subprocess.CalledProcessError as exc:
detail = ((exc.stderr or "").strip().splitlines() or [str(exc)])[-1]
print(f"Queue-Statistik nicht verfügbar ({detail}); zähle nur Seiten/Bilder.")
return None
except (TypeError, ValueError):
# Leere/überraschende psql-Ausgabe darf die übrigen Werte nicht kosten.
print("Queue-Statistik nicht verfügbar (unerwartete psql-Ausgabe).")
return None
def collect():
while True:
try:
data = load_data()
exact = images_exact_due(data)
count, images, images_exact = get_stats(exact)
pending = get_pending()
if images is None:
# Stichprobe ohne Treffer (praktisch nur bei sehr großen
@@ -1538,6 +1962,7 @@ def collect():
"count": count,
"images": images,
"images_exact": images_exact,
"pending": pending,
}
data.append(entry)
@@ -1546,7 +1971,8 @@ def collect():
print(
f"[{entry['timestamp']}] "
f"Pages: {count} · Images: {images} "
f"({'exakt' if images_exact else 'geschätzt'})"
f"({'exakt' if images_exact else 'geschätzt'}) "
f"· Pending: {pending if pending is not None else 'n/a'}"
)
except Exception as e:
web/test_stats.py
+144 -6
View File
@@ -117,6 +117,59 @@ class GetStatsTest(unittest.TestCase):
"SELECT COUNT(*) FROM pages;")
class GetPendingTest(unittest.TestCase):
@mock.patch.object(stats_app, "_psql_scalar")
def test_counts_pending_queue_rows(self, psql):
psql.return_value = "1384711"
self.assertEqual(stats_app.get_pending(), 1384711)
self.assertIn("FROM crawl_queue", stats_app.PENDING_QUEUE_SQL)
self.assertIn("status = 'pending'", stats_app.PENDING_QUEUE_SQL)
@mock.patch.object(stats_app, "_psql_scalar")
def test_unavailable_queue_returns_none(self, psql):
psql.side_effect = subprocess.CalledProcessError(
1, ["psql"], stderr='ERROR: relation "crawl_queue" does not exist'
)
self.assertIsNone(stats_app.get_pending())
@mock.patch.object(stats_app, "_psql_scalar")
def test_unexpected_output_returns_none(self, psql):
psql.return_value = ""
self.assertIsNone(stats_app.get_pending())
class CollectTest(unittest.TestCase):
def _run_collect(self, get_stats, get_pending):
with mock.patch.object(stats_app, "load_data", return_value=[]), \
mock.patch.object(stats_app, "save_data") as save_data, \
mock.patch.object(stats_app, "get_stats", get_stats), \
mock.patch.object(stats_app, "get_pending", get_pending), \
mock.patch.object(stats_app, "time") as time_mock:
time_mock.sleep.side_effect = KeyboardInterrupt
with self.assertRaises(KeyboardInterrupt):
stats_app.collect()
return save_data.call_args[0][0][0]
def test_pending_is_stored_with_every_sample(self):
entry = self._run_collect(
mock.Mock(return_value=(1000, 2000, True)),
mock.Mock(return_value=4711),
)
self.assertEqual(entry["pending"], 4711)
self.assertEqual(entry["count"], 1000)
self.assertEqual(entry["images"], 2000)
def test_missing_queue_does_not_break_the_sample(self):
entry = self._run_collect(
mock.Mock(return_value=(1000, 2000, False)),
mock.Mock(return_value=None),
)
self.assertIsNone(entry["pending"])
self.assertEqual(entry["count"], 1000)
class StatsPageTest(unittest.TestCase):
def test_page_contains_image_counter(self):
@@ -159,15 +212,27 @@ class StatsPageTest(unittest.TestCase):
self.assertIn('id="heatmap-images"', html)
self.assertIn('id="images-empty"', html)
# the image series skips entries without an image count (old data)
# and only plots exact measurements - no estimates between the 2 h counts
self.assertIn("function toImagePoints(points)", html)
# Image estimates are excluded by default and can be included with the persistent toggle.
self.assertIn('id="show-estimated-images"', html)
self.assertIn('type="checkbox" id="show-estimated-images"', html)
self.assertIn("Include estimated image counts in graphs", html)
self.assertIn("function toImagePoints(points, includeEstimated)", html)
self.assertIn("p.images !== null && p.images !== undefined", html)
self.assertIn(".filter(function (p) { return p.imagesExact; })", html)
self.assertIn("only exact measurements are plotted (no estimates)", html)
self.assertIn("includeEstimated || p.imagesExact", html)
self.assertIn("localStorage.getItem('js-show-estimated-images')", html)
self.assertIn("localStorage.setItem('js-show-estimated-images'", html)
self.assertIn("toImagePoints(all, showEstimatedImages)", html)
self.assertIn("Only exact measurements (counted every 2 h) are plotted", html)
self.assertIn("Exact and estimated measurements are plotted", html)
self.assertIn("Only estimated image counts so far", html)
self.assertIn("No exact image counts in the selected range", html)
self.assertIn("No image counts in the selected range", html)
# The headline counter stays the latest collected value; only graphs toggle estimates.
self.assertIn("elImages.textContent = fmt(last.images)", html)
# Images estimates affect graphs only; other series keep their own points.
self.assertIn("renderCharts({\n prefix: '',\n points: fpoints", html)
self.assertIn("renderCharts({\n prefix: 'pending',\n points: pendingFpoints", html)
self.assertIn("renderHeatmap(imageAll, 'heatmap-images', 'Images')", html)
self.assertIn("renderRecords(imageDeltas, 'records-images', 'Images')", html)
# existing page chart ids must stay lowercase (mainChart, not MainChart)
self.assertIn("function chartId(prefix, suffix)", html)
self.assertIn("chartId(prefix, 'MainChart')", html)
@@ -182,6 +247,79 @@ class StatsPageTest(unittest.TestCase):
self.assertIn("renderHeatmap(all, 'heatmap', 'Pages')", html)
self.assertIn("renderHeatmap(imageAll, 'heatmap-images', 'Images')", html)
def test_page_has_metric_tabs_with_one_panel_each(self):
client = stats_app.app.test_client()
response = client.get("/")
self.assertEqual(response.status_code, 200)
html = response.get_data(as_text=True)
# three equally styled tab buttons (pages / images / queue) with icons
self.assertIn('class="metric-btn', html)
for metric in ("pages", "images", "queue"):
self.assertIn('data-metric="' + metric + '"', html)
self.assertIn("metric-icon", html)
self.assertIn("📄", html)
self.assertIn("🖼️", html)
self.assertIn("⏳", html)
# every metric has its own panel; only the initial one is active
self.assertEqual(html.count('class="metric-panel active" data-metric="pages"'), 1)
for metric in ("images", "queue"):
self.assertIn('class="metric-panel" data-metric="' + metric + '"', html)
self.assertIn('id="summary-images"', html)
self.assertIn('id="summary-queue"', html)
# switching tabs hides the other panels and is remembered
self.assertIn("function setMetric(next, persist)", html)
self.assertIn("panel.classList.toggle('active', panel.dataset.metric === activeMetric)", html)
self.assertIn("localStorage.setItem('js-metric', activeMetric)", html)
self.assertIn("localStorage.getItem('js-metric')", html)
# hidden panels must not create Chart.js instances (canvas has 0 px)
self.assertIn("function isVisible(el)", html)
self.assertIn("if (!target || !isVisible(target)) return;", html)
# summary cards are generated for the non-page tabs
self.assertIn("function renderSummary(containerId, all, points)", html)
self.assertIn("renderSummary('summary-images', imageAll, imageFpoints)", html)
self.assertIn("renderSummary('summary-queue', pendingAll, pendingFpoints)", html)
# exports follow the selected tab
self.assertIn("function exportSeries()", html)
self.assertIn("return { name: 'jonassearch_images'", html)
self.assertIn("return { name: 'jonassearch_queue'", html)
def test_page_contains_pending_queue_counter_and_graphs(self):
client = stats_app.app.test_client()
response = client.get("/")
self.assertEqual(response.status_code, 200)
html = response.get_data(as_text=True)
self.assertIn('id="current-pending"', html)
self.assertIn('id="current-pending-meta"', html)
self.assertIn("Queue – all graphs", html)
for canvas in (
"pendingMainChart",
"pendingDeltaChart",
"pendingCumChart",
"pendingHourlyChart",
"pendingHodChart",
):
self.assertIn('id="' + canvas + '"', html)
self.assertIn('id="records-pending"', html)
self.assertIn('id="heatmap-pending"', html)
self.assertIn('id="pending-empty"', html)
# pending values are parsed and plotted without sampling/filtering
self.assertIn("function toPendingPoints(points)", html)
self.assertIn("p.pending !== null && p.pending !== undefined", html)
self.assertIn("filterRange(pendingAll, last.x)", html)
self.assertIn("unit: 'Pending'", html)
self.assertIn("renderRecords(pendingDeltas, 'records-pending', 'Pending')", html)
self.assertIn("renderHeatmap(pendingAll, 'heatmap-pending', 'Pending')", html)
self.assertIn("No queue counts yet", html)
self.assertIn("No queue counts in the selected range.", html)
if __name__ == "__main__":
unittest.main(verbosity=2)