Serverjonas/search
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

make the stats mor ereadaable

Browse Source
This commit is contained in:
jhkaktus
2026-10-21 18:32:31 +02:00
parent 3ebfeca7cc
commit 530c2b1ecc
3 changed files with 619 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/app.py
+465 -39
View File
@@ -147,6 +147,61 @@ HTML = r"""
.ranges { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 18px; }
.image-options { display: flex; align-items: center; gap: 9px; margin: 0 0 12px; color: var(--muted); font-size: 0.88rem; }
.image-options input { accent-color: var(--accent); }
.image-options label { cursor: pointer; }
.image-options-hint { color: var(--muted); font-size: 0.78rem; margin-bottom: 14px; }
.metric-tabs {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(160px, 1fr));
gap: 10px;
margin: 22px 0 16px;
}
.metric-btn {
display: flex;
align-items: center;
gap: 12px;
text-align: left;
background: var(--card);
border: 1px solid var(--border);
color: var(--muted);
border-radius: 15px;
padding: 13px 16px;
font-size: 1rem;
font-weight: 600;
font-family: inherit;
cursor: pointer;
transition: border-color .15s, color .15s, background .15s;
}
.metric-btn:hover { border-color: var(--accent); color: var(--text); }
.metric-btn.active {
color: var(--text);
border-color: var(--accent);
background: linear-gradient(180deg, rgba(79,140,255,0.18), rgba(79,140,255,0.04));
box-shadow: inset 0 0 0 1px rgba(79,140,255,0.35);
}
.metric-icon { font-size: 1.6rem; line-height: 1; }
.metric-name { display: block; }
.metric-count {
display: block;
font-size: 0.78rem;
font-weight: 400;
color: var(--muted);
margin-top: 2px;
}
.metric-panel { display: none; }
.metric-panel.active { display: block; }
.section-title-first { margin-top: 6px; }
.range-btn {
background: var(--card);
border: 1px solid var(--border);
@@ -253,6 +308,8 @@ HTML = r"""
.section-title { font-weight: 600; margin: 30px 0 12px; font-size: 1.05rem; }
@media (max-width: 600px) {
.metric-btn { padding: 10px 12px; font-size: 0.9rem; gap: 9px; }
.metric-icon { font-size: 1.3rem; }
.current { font-size: 2.4rem; }
.container { padding: 20px 14px 40px; }
.chart-container { padding: 14px; }
@@ -266,7 +323,7 @@ HTML = r"""
<div class="container">
<header>
<h1>JonasSearch</h1>
<div class="subtitle">Indexed Pages – Live Statistics</div>
<div class="subtitle" id="page-subtitle">Pages – Live Statistics</div>
<div class="header-right">
<a class="btn" id="btn-csv" href="#">Export CSV</a>
<a class="btn" id="btn-json" href="#">Export JSON</a>
@@ -274,19 +331,28 @@ HTML = r"""
</div>
</header>
<div class="current-row">
<div>
<div class="current">
<span id="current">–</span><span class="unit">Pages</span>
</div>
<div class="current-meta" id="current-meta">Loading…</div>
</div>
<div>
<div class="current">
<span id="current-images">–</span><span class="unit">Images</span>
</div>
<div class="current-meta" id="current-images-meta">Loading…</div>
</div>
<div class="metric-tabs" role="tablist" aria-label="Data type">
<button class="metric-btn active" data-metric="pages" role="tab" aria-selected="true">
<span class="metric-icon" aria-hidden="true">📄</span>
<span>
<span class="metric-name">Pages</span>
<span class="metric-count" id="tab-count-pages">– total</span>
</span>
</button>
<button class="metric-btn" data-metric="images" role="tab" aria-selected="false">
<span class="metric-icon" aria-hidden="true">🖼️</span>
<span>
<span class="metric-name">Images</span>
<span class="metric-count" id="tab-count-images">– total</span>
</span>
</button>
<button class="metric-btn" data-metric="queue" role="tab" aria-selected="false">
<span class="metric-icon" aria-hidden="true">⏳</span>
<span>
<span class="metric-name">Queue</span>
<span class="metric-count" id="tab-count-queue">– pending</span>
</span>
</button>
</div>
<div class="badges">
@@ -305,6 +371,16 @@ HTML = r"""
<button class="range-btn" data-range="6h">Last 6 h</button>
</div>
<div class="metric-panel active" data-metric="pages" role="tabpanel">
<div class="current-row">
<div>
<div class="current">
<span id="current">–</span><span class="unit">Pages</span>
</div>
<div class="current-meta" id="current-meta">Loading…</div>
</div>
</div>
<div class="grid">
<div class="card">
<div class="label">Growth / hour</div>
@@ -406,21 +482,39 @@ HTML = r"""
<div id="heatmap"></div>
</div>
<div class="section-title">Images – all graphs</div>
<div class="chart-sub">Only exact measurements (counted every 2 h) are plotted – the estimates in between are excluded.</div>
</div>
<div class="metric-panel" data-metric="images" role="tabpanel">
<div class="current-row">
<div>
<div class="current">
<span id="current-images">–</span><span class="unit">Images</span>
</div>
<div class="current-meta" id="current-images-meta">Loading…</div>
</div>
</div>
<div class="grid" id="summary-images"></div>
<div class="section-title section-title-first">Images – all graphs</div>
<div class="image-options">
<input type="checkbox" id="show-estimated-images">
<label for="show-estimated-images">Include estimated image counts in graphs</label>
</div>
<div class="image-options-hint" id="images-mode-note">Only exact measurements (counted every 2 h) are plotted. This setting is saved in this browser.</div>
<div class="chart-sub" id="images-empty" style="display:none">
No image counts yet – they appear after the next collector run with the image migration in place.
</div>
<div class="chart-container">
<div class="chart-title">Images over time</div>
<div class="chart-sub">Image URLs found while crawling · exact count every 2 h · only exact measurements are plotted (no estimates) · dashed line = trend &amp; 24 h forecast</div>
<div class="chart-sub">Image URLs found while crawling · dashed line = trend &amp; 24 h forecast</div>
<div class="chart-box"><canvas id="imagesMainChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Change per 5 minutes</div>
<div class="chart-sub">Normalized to a 5-minute interval · spans between two exact counts (2 h) · grey = raw · colored = smoothed (moving average over 5 measurements)</div>
<div class="chart-sub">Normalized to a 5-minute interval between available image measurements · grey = raw · colored = smoothed (moving average over 5 measurements)</div>
<div class="chart-box small"><canvas id="imagesDeltaChart"></canvas></div>
</div>
@@ -455,6 +549,70 @@ HTML = r"""
<div id="heatmap-images"></div>
</div>
</div>
<div class="metric-panel" data-metric="queue" role="tabpanel">
<div class="current-row">
<div>
<div class="current">
<span id="current-pending">–</span><span class="unit">Pending</span>
</div>
<div class="current-meta" id="current-pending-meta">Loading…</div>
</div>
</div>
<div class="grid" id="summary-queue"></div>
<div class="section-title section-title-first">Queue – all graphs</div>
<div class="chart-sub">URLs waiting in crawl_queue (status = 'pending'). Counted with every 30-minute sample (exact COUNT). Falling graphs mean the crawler is working through the queue.</div>
<div class="chart-sub" id="pending-empty" style="display:none">
No queue counts yet – they appear after the next collector run.
</div>
<div class="chart-container">
<div class="chart-title">Pending URLs over time</div>
<div class="chart-sub">Queued URLs not yet crawled · dashed line = trend &amp; 24 h forecast</div>
<div class="chart-box"><canvas id="pendingMainChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Change per 5 minutes</div>
<div class="chart-sub">Normalized to a 5-minute interval · negative = the crawler worked the queue down · grey = raw · colored = smoothed (moving average over 5 measurements)</div>
<div class="chart-box small"><canvas id="pendingDeltaChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Cumulative net change</div>
<div class="chart-sub">Total queued URLs gained/lost since the start of the selected range · linear time axis</div>
<div class="chart-box small"><canvas id="pendingCumChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Queue change per hour (last 48 h)</div>
<div class="chart-sub">Sum of changes per hour bucket · green = queue grew, red = queue shrank</div>
<div class="chart-box small"><canvas id="pendingHourlyChart"></canvas></div>
</div>
<div class="chart-row">
<div class="chart-container">
<div class="chart-title">Queue change by hour of day</div>
<div class="chart-sub">Average change per 5 min, grouped by hour of day (whole history)</div>
<div class="chart-box small"><canvas id="pendingHodChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Records – biggest queue changes</div>
<div class="chart-sub">Largest gains and losses between two consecutive measurements (selected range)</div>
<div id="records-pending"></div>
</div>
</div>
<div class="section-title">Activity heatmap – queue change per hour of day</div>
<div class="chart-container heatmap">
<div class="chart-sub">Rows = days · columns = hours (0–23) · green = queue grew, red = queue shrank, grey = no data</div>
<div id="heatmap-pending"></div>
</div>
</div>
<div class="footer" id="footer"></div>
</div>
@@ -467,6 +625,7 @@ if (window.Chart && Chart.registerables) {
const ACCENT = '#4f8cff';
const ACCENT2 = '#b47aff';
const ACCENT3 = '#f5a623';
const UP = '#2ecc71';
const DOWN = '#ff6b6b';
const FALLBACK_INTERVAL_MS = 15 * 60 * 1000;
@@ -476,6 +635,8 @@ const REFRESH_S = 30;
let rawValues = [];
let intervalMs = FALLBACK_INTERVAL_MS;
let range = 'all';
let activeMetric = 'pages';
let showEstimatedImages = false;
let charts = {};
let refreshLeft = REFRESH_S;
@@ -603,27 +764,36 @@ function parsePoints(values) {
.map(function (v) {
const images = (v.images === null || v.images === undefined)
? null : Number(v.images);
const pending = (v.pending === null || v.pending === undefined)
? null : Number(v.pending);
return {
x: new Date(v.timestamp).getTime(),
y: v.count,
images: Number.isFinite(images) ? images : null,
imagesExact: v.images_exact === true
imagesExact: v.images_exact === true,
pending: Number.isFinite(pending) ? pending : null
};
})
.filter(function (p) { return !isNaN(p.x); })
.sort(function (a, b) { return a.x - b.x; });
}
// Nur exakte Bildzählungen (alle 2 h). Die dazwischen geschätzten Werte
// werden in keinem Diagramm dargestellt, damit alle Graphen ausschließlich
// echte Messpunkte zeigen.
function toImagePoints(points) {
// Estimates are excluded by default; the Images section toggle can include them.
function toImagePoints(points, includeEstimated) {
return (points || [])
.filter(function (p) { return p.images !== null && p.images !== undefined; })
.filter(function (p) { return p.imagesExact; })
.filter(function (p) { return includeEstimated || p.imagesExact; })
.map(function (p) { return { x: p.x, y: p.images }; });
}
// Pending queue entries are counted with every sample (exact COUNT), so
// no filtering beyond "value present" is needed.
function toPendingPoints(points) {
return (points || [])
.filter(function (p) { return p.pending !== null && p.pending !== undefined; })
.map(function (p) { return { x: p.x, y: p.pending }; });
}
const RANGE_SPANS = {
'6h': 6 * 3600000,
'24h': 24 * 3600000,
@@ -673,6 +843,17 @@ document.getElementById('btn-theme').addEventListener('click', function () {
applyTheme(document.documentElement.dataset.theme === 'light' ? 'dark' : 'light');
});
(function () {
const toggle = document.getElementById('show-estimated-images');
try { showEstimatedImages = localStorage.getItem('js-show-estimated-images') === 'true'; } catch (e) {}
toggle.checked = showEstimatedImages;
toggle.addEventListener('change', function () {
showEstimatedImages = toggle.checked;
try { localStorage.setItem('js-show-estimated-images', String(showEstimatedImages)); } catch (e) {}
if (rawValues.length) renderAll();
});
})();
(function () {
let saved = null;
try { saved = localStorage.getItem('js-theme'); } catch (e) {}
@@ -680,6 +861,44 @@ document.getElementById('btn-theme').addEventListener('click', function () {
document.getElementById('btn-theme').textContent = saved === 'light' ? '🌙 Dark' : '☀️ Light';
})();
/* ---------- metric tabs (pages / images / queue) ---------- */
const METRIC_LABELS = { pages: 'Pages', images: 'Images', queue: 'Queue' };
// Hidden panels must not render Chart.js instances: a canvas inside a
// display:none container measures 0 px and breaks on later resize.
function isVisible(el) {
const panel = el.closest('.metric-panel');
return !panel || panel.classList.contains('active');
}
function setMetric(next, persist) {
activeMetric = METRIC_LABELS[next] ? next : 'pages';
document.querySelectorAll('.metric-btn').forEach(function (btn) {
const on = btn.dataset.metric === activeMetric;
btn.classList.toggle('active', on);
btn.setAttribute('aria-selected', on ? 'true' : 'false');
});
document.querySelectorAll('.metric-panel').forEach(function (panel) {
panel.classList.toggle('active', panel.dataset.metric === activeMetric);
});
const label = METRIC_LABELS[activeMetric];
document.getElementById('page-subtitle').textContent = label + ' – Live Statistics';
document.title = 'JonasSearch – ' + label;
if (persist !== false) {
try { localStorage.setItem('js-metric', activeMetric); } catch (e) {}
}
if (rawValues.length) renderAll();
}
document.querySelectorAll('.metric-btn').forEach(function (btn) {
btn.addEventListener('click', function () { setMetric(btn.dataset.metric); });
});
/* ---------- range buttons ---------- */
document.querySelectorAll('.range-btn').forEach(function (btn) {
@@ -702,16 +921,50 @@ function download(name, text, mime) {
URL.revokeObjectURL(a.href);
}
// Export what the selected tab shows, so the CSV/JSON match the graphs.
function exportSeries() {
const all = parsePoints(rawValues);
if (activeMetric === 'images') {
const values = all
.filter(function (p) { return p.images !== null; })
.map(function (p) {
return { timestamp: new Date(p.x).toISOString(), images: p.images, images_exact: p.imagesExact };
});
return { name: 'jonassearch_images', values: values };
}
if (activeMetric === 'queue') {
const values = all
.filter(function (p) { return p.pending !== null; })
.map(function (p) {
return { timestamp: new Date(p.x).toISOString(), pending: p.pending };
});
return { name: 'jonassearch_queue', values: values };
}
return { name: 'jonassearch_pages', values: rawValues };
}
document.getElementById('btn-csv').addEventListener('click', function (e) {
e.preventDefault();
const lines = ['timestamp;count'];
rawValues.forEach(function (v) { lines.push(v.timestamp + ';' + v.count); });
download('jonassearch_pages.csv', lines.join('\n'), 'text/csv');
const series = exportSeries();
const keys = series.values.length ? Object.keys(series.values[0]) : ['timestamp'];
const lines = [keys.join(';')];
series.values.forEach(function (row) {
lines.push(keys.map(function (k) { return row[k] === undefined ? '' : row[k]; }).join(';'));
});
download(series.name + '.csv', lines.join('\n'), 'text/csv');
});
document.getElementById('btn-json').addEventListener('click', function (e) {
e.preventDefault();
download('jonassearch_pages.json', JSON.stringify({ values: rawValues }, null, 2), 'application/json');
const series = exportSeries();
download(
series.name + '.json',
JSON.stringify({ metric: activeMetric, values: series.values }, null, 2),
'application/json'
);
});
/* ---------- chart helpers ---------- */
@@ -767,6 +1020,9 @@ function gradientFill(colorTop, colorBottom) {
/* ---------- charts ---------- */
function renderCharts(cfg) {
const target = document.getElementById(chartId(cfg.prefix, 'MainChart'));
if (!target || !isVisible(target)) return;
const prefix = cfg.prefix;
const points = cfg.points;
const deltas = cfg.deltas;
@@ -1052,6 +1308,7 @@ function renderCharts(cfg) {
function renderRecords(deltas, id, unit) {
const el = document.getElementById(id);
if (!el || !isVisible(el)) return;
if (deltas.length < 2) {
el.innerHTML = '<div class="chart-sub">Not enough data yet.</div>';
return;
@@ -1078,6 +1335,7 @@ function renderRecords(deltas, id, unit) {
function renderHeatmap(points, id, unit) {
const el = document.getElementById(id);
if (!el || !isVisible(el)) return;
if (!points.length) { el.innerHTML = ''; return; }
function localDayKey(d) {
@@ -1137,6 +1395,84 @@ function renderHeatmap(points, id, unit) {
el.innerHTML = html;
}
/* ---------- summary cards for images / queue ---------- */
// The pages tab keeps its static card grid (ids like stat-delta-6h);
// images and queue get the same key numbers generated from their series.
function renderSummary(containerId, all, points) {
const el = document.getElementById(containerId);
if (!el || !isVisible(el)) return;
if (!points.length) {
el.innerHTML = '<div class="chart-sub">No data in the selected range yet.</div>';
return;
}
const anchor = all[all.length - 1];
const first = points[0];
const hours = (anchor.x - first.x) / 3600000;
const perHour = hours > 0 ? (anchor.y - first.y) / hours : null;
const deltas = computeDeltas(points).deltas;
let peak = all[0], low = all[0];
all.forEach(function (p) {
if (p.y > peak.y) peak = p;
if (p.y < low.y) low = p;
});
const cards = [];
function signedCard(label, value, hint) {
const cls = value === null ? '' : (value >= 0 ? 'pos' : 'neg');
cards.push('<div class="card"><div class="label">' + label +
'</div><div class="value ' + cls + '">' +
(value === null ? '–' : fmtSigned(value)) +
'</div><div class="hint">' + hint + '</div></div>');
}
function plainCard(label, value, hint) {
cards.push('<div class="card"><div class="label">' + label +
'</div><div class="value">' + value +
'</div><div class="hint">' + hint + '</div></div>');
}
signedCard('Growth / hour', perHour, 'Average over full history');
signedCard('Growth / day', perHour === null ? null : perHour * 24, 'Average over full history');
const windows = [
['6 h', 6 * 3600000, Math.max(3 * intervalMs, 3600000)],
['24 h', 24 * 3600000, Math.max(6 * intervalMs, 3 * 3600000)],
['7 days', 7 * 24 * 3600000, Math.max(12 * intervalMs, 6 * 3600000)]
];
windows.forEach(function (win) {
const base = closestBefore(all, anchor.x - win[1], win[2]);
signedCard('Growth ' + win[0], base ? anchor.y - base.y : null,
base ? 'vs. measurement ' + fmtDur(anchor.x - base.x) + ' ago' : 'Not enough data yet');
});
let sum24 = 0;
const windowStart = Math.max(first.x, anchor.x - 24 * 3600000);
const coveredH = (anchor.x - windowStart) / 3600000;
deltas.forEach(function (d) {
if (d.x > anchor.x - 24 * 3600000) sum24 += d.diff;
});
signedCard('Avg / h (24 h)', coveredH >= 0.5 ? sum24 / coveredH : null, 'Mean of last 24 h changes');
plainCard('Peak', fmt(peak.y), 'On ' + fmtDateFull(peak.x));
plainCard('Lowest', fmt(low.y), 'On ' + fmtDateFull(low.x));
const regWindow = points.filter(function (p) { return p.x >= anchor.x - 2 * 3600000; }).slice(-96);
const reg = regress(regWindow);
if (reg) {
plainCard('Forecast next hour', fmt(anchor.y + reg.slope),
'Trend ' + fmtSigned(reg.slope) + '/h · R² = ' + reg.r2.toFixed(3));
} else {
plainCard('Forecast next hour', '–', 'Needs at least 2 measurements');
}
el.innerHTML = cards.join('');
}
/* ---------- main update ---------- */
function renderAll() {
@@ -1147,6 +1483,8 @@ function renderAll() {
const meta = document.getElementById('current-meta');
const elImages = document.getElementById('current-images');
const imagesMeta = document.getElementById('current-images-meta');
const elPending = document.getElementById('current-pending');
const pendingMeta = document.getElementById('current-pending-meta');
document.getElementById('badge-count').textContent = 'Measurements: ' + all.length;
@@ -1155,6 +1493,8 @@ function renderAll() {
meta.textContent = 'No measurements yet – the collector stores one every 30 minutes.';
elImages.textContent = '–';
imagesMeta.textContent = 'No image count yet';
elPending.textContent = '–';
pendingMeta.textContent = 'No queue count yet';
setStatus('offline', 'No data');
return;
}
@@ -1181,6 +1521,21 @@ function renderAll() {
' · ' + fmtDateFull(last.x);
}
if (last.pending === null) {
elPending.textContent = '–';
pendingMeta.textContent = 'No queue count yet';
} else {
elPending.textContent = fmt(last.pending);
pendingMeta.textContent = 'Waiting in queue · exact count · ' + fmtDateFull(last.x);
}
/* tab badges: always show the latest value of each data type */
document.getElementById('tab-count-pages').textContent = fmt(last.y) + ' total';
document.getElementById('tab-count-images').textContent = last.images === null
? 'no data yet' : fmt(last.images) + ' total';
document.getElementById('tab-count-queue').textContent = last.pending === null
? 'no data yet' : fmt(last.pending) + ' pending';
document.getElementById('badge-since').textContent = 'First measurement: ' + fmtDateFull(first.x);
document.getElementById('badge-updated').textContent = 'Last update: ' + fmtDate(last.x);
@@ -1295,19 +1650,42 @@ function renderAll() {
elF.classList.add(fc >= fpoints[fpoints.length - 1].y ? 'pos' : 'neg');
document.getElementById('stat-forecast-hint').textContent =
'Trend ' + fmtSigned(reg.slope) + '/h · R² = ' + reg.r2.toFixed(3);
document.getElementById('footer').textContent =
'Forecast: linear least-squares regression over the last ' + regWindow.length +
' measurements (' + fmtDur(fpoints[fpoints.length - 1].x - regWindow[0].x) +
'), extrapolated +24 h · R² = ' + reg.r2.toFixed(3) +
' · Range: ' + (range === 'all' ? 'all' : range) + ' · All statistics are computed in the browser.';
} else {
document.getElementById('stat-forecast').textContent = '–';
document.getElementById('stat-forecast-hint').textContent = 'Needs at least 2 measurements';
document.getElementById('footer').textContent = 'Forecast available once there are at least 2 measurements.';
}
/* images: same graphs, but only exact measurements (every 2 h) */
const imageAll = toImagePoints(all);
document.getElementById('footer').textContent =
'Showing ' + METRIC_LABELS[activeMetric] +
' · Range: ' + (range === 'all' ? 'all data' : range) +
' · All statistics are computed in the browser.';
/* queue: same graphs for the pending URLs in crawl_queue */
const pendingAll = toPendingPoints(all);
const pendingFpoints = filterRange(pendingAll, last.x);
const pendingDeltas = computeDeltas(pendingFpoints).deltas;
const pendingSmoothY = movingAverage(pendingDeltas.map(function (d) { return d.y; }), DELTA_SMOOTH_WINDOW);
const pendingSmoothedDeltas = pendingDeltas.map(function (d, i) {
return { x: d.x, y: pendingSmoothY[i], raw: d.y, dt: d.dt, diff: d.diff };
});
const pendingEmpty = document.getElementById('pending-empty');
if (!pendingAll.length) {
pendingEmpty.textContent = all.some(function (p) { return p.pending !== null; })
? 'No queue counts in the selected range.'
: 'No queue counts yet – they appear after the next collector run.';
pendingEmpty.style.display = '';
} else if (!pendingFpoints.length) {
pendingEmpty.textContent = 'No queue counts in the selected range.';
pendingEmpty.style.display = '';
} else {
pendingEmpty.style.display = 'none';
}
/* Images: estimates are optional and controlled by the Images-section toggle. */
const imageAll = toImagePoints(all, showEstimatedImages);
document.getElementById('images-mode-note').textContent = showEstimatedImages
? 'Exact and estimated measurements are plotted. Estimates come from a 1% table sample; this setting is saved in this browser.'
: 'Only exact measurements (counted every 2 h) are plotted. This setting is saved in this browser.';
const imageFpoints = filterRange(imageAll, last.x);
const imageDeltas = computeDeltas(imageFpoints).deltas;
const imageSmoothY = movingAverage(imageDeltas.map(function (d) { return d.y; }), DELTA_SMOOTH_WINDOW);
@@ -1318,11 +1696,11 @@ function renderAll() {
const hasImageValues = all.some(function (p) { return p.images !== null; });
if (!imageAll.length) {
imagesEmpty.textContent = hasImageValues
? 'Only estimated image counts so far – the graphs show exact counts only (every 2 h).'
? 'Only estimated image counts so far – turn on “Include estimated image counts in graphs” to show them.'
: 'No image counts yet – they appear after the next collector run with the image migration in place.';
imagesEmpty.style.display = '';
} else if (!imageFpoints.length) {
imagesEmpty.textContent = 'No exact image counts in the selected range (counted every 2 h).';
imagesEmpty.textContent = 'No image counts in the selected range.';
imagesEmpty.style.display = '';
} else {
imagesEmpty.style.display = 'none';
@@ -1351,10 +1729,25 @@ function renderAll() {
gradTop: 'rgba(180,122,255,0.28)',
gradBottom: 'rgba(180,122,255,0.02)'
});
renderCharts({
prefix: 'pending',
points: pendingFpoints,
deltas: pendingDeltas,
smoothedDeltas: pendingSmoothedDeltas,
unit: 'Pending',
accent: ACCENT3,
accentBar: 'rgba(245,166,35,0.65)',
gradTop: 'rgba(245,166,35,0.28)',
gradBottom: 'rgba(245,166,35,0.02)'
});
renderRecords(deltas, 'records', 'Pages');
renderRecords(imageDeltas, 'records-images', 'Images');
renderRecords(pendingDeltas, 'records-pending', 'Pending');
renderHeatmap(all, 'heatmap', 'Pages');
renderHeatmap(imageAll, 'heatmap-images', 'Images');
renderHeatmap(pendingAll, 'heatmap-pending', 'Pending');
renderSummary('summary-images', imageAll, imageFpoints);
renderSummary('summary-queue', pendingAll, pendingFpoints);
}
async function update() {
@@ -1380,6 +1773,13 @@ setInterval(function () {
update();
setInterval(update, REFRESH_S * 1000);
// Restore the last selected tab (pages / images / queue).
(function () {
let saved = 'pages';
try { saved = localStorage.getItem('js-metric') || 'pages'; } catch (e) {}
setMetric(saved, false);
})();
</script>
</body>
@@ -1427,6 +1827,10 @@ ESTIMATE_IMAGE_SQL = (
percent=IMAGE_SAMPLE_PERCENT,
)
# URLs waiting in the crawler queue. Nutzt den Index
# (status, priority, discovered_at); kein Sampling, der Wert ist exakt.
PENDING_QUEUE_SQL = "SELECT COUNT(*) FROM crawl_queue WHERE status = 'pending';"
def _psql_scalar(sql):
"""Eine einzelne psql-Zeile ("count|images") abholen."""
@@ -1518,12 +1922,32 @@ def get_stats(exact_images):
return count, images, False
def get_pending():
"""
Anzahl der URLs mit status='pending' in der crawl_queue.
Ist die Tabelle (noch) nicht erreichbar, wird None geliefert - die
Messung wird trotzdem gespeichert, nur ohne Queue-Wert.
"""
try:
return int(_psql_scalar(PENDING_QUEUE_SQL).strip())
except subprocess.CalledProcessError as exc:
detail = ((exc.stderr or "").strip().splitlines() or [str(exc)])[-1]
print(f"Queue-Statistik nicht verfügbar ({detail}); zähle nur Seiten/Bilder.")
return None
except (TypeError, ValueError):
# Leere/überraschende psql-Ausgabe darf die übrigen Werte nicht kosten.
print("Queue-Statistik nicht verfügbar (unerwartete psql-Ausgabe).")
return None
def collect():
while True:
try:
data = load_data()
exact = images_exact_due(data)
count, images, images_exact = get_stats(exact)
pending = get_pending()
if images is None:
# Stichprobe ohne Treffer (praktisch nur bei sehr großen
@@ -1538,6 +1962,7 @@ def collect():
"count": count,
"images": images,
"images_exact": images_exact,
"pending": pending,
}
data.append(entry)
@@ -1546,7 +1971,8 @@ def collect():
print(
f"[{entry['timestamp']}] "
f"Pages: {count} · Images: {images} "
f"({'exakt' if images_exact else 'geschätzt'})"
f"({'exakt' if images_exact else 'geschätzt'}) "
f"· Pending: {pending if pending is not None else 'n/a'}"
)
except Exception as e: