make the stats mor ereadaable

README.md

@@ -8,7 +8,16 @@ das komplette serverjonas search
   siehe [search/README.md](search/README.md)
 - `web/` – Statistik-Webseite (Port 8081, `jonassearch-stats.service`).
   Seiten werden alle 30 Min. gezählt; Bilder alle 2 h exakt, dazwischen nur
-  geschätzt. Alle Diagramme zeigen ausschließlich die exakten Messpunkte.
+  geschätzt. Die Image-Diagramme zeigen standardmäßig nur exakte Messpunkte;
+  ein umschaltbarer, im Browser gespeicherter Schalter kann Schätzwerte in allen
+  Image-Diagrammen, Records und der Heatmap einblenden.
+  Mit jeder Messung wird zusätzlich die Anzahl der `pending`-Einträge in
+  `crawl_queue` erfasst (exakter `COUNT`, keine Schätzung) und mit dem
+  kompletten Diagrammsatz (Verlauf, Change/5 min, kumulativ, pro Stunde,
+  Tagesstunden, Records, Heatmap) dargestellt.
+  Die Hauptseite hat drei Tabs mit Symbolen (Pages, Images, Queue): Jeder Tab
+  zeigt nur die Kennzahlen und Diagramme seines Datentyps, die Auswahl wird im
+  Browser gespeichert. CSV/JSON-Export folgt dem aktiven Tab.
 - `web/tui.py` – Terminal-Dashboard für dieselben Statistiken (siehe unten)
 - `priorityweb/` – Vorschlagsverfeinerer (Port 8082, `jonassearch-priority.service`)
 - `search/jonassearch-engine.service` – systemd-Unit für die Suchmaschine

web/app.py

@@ -147,6 +147,61 @@ HTML = r"""
 
         .ranges { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 18px; }
 
+        .image-options { display: flex; align-items: center; gap: 9px; margin: 0 0 12px; color: var(--muted); font-size: 0.88rem; }
+        .image-options input { accent-color: var(--accent); }
+        .image-options label { cursor: pointer; }
+        .image-options-hint { color: var(--muted); font-size: 0.78rem; margin-bottom: 14px; }
+
+        .metric-tabs {
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(160px, 1fr));
+            gap: 10px;
+            margin: 22px 0 16px;
+        }
+
+        .metric-btn {
+            display: flex;
+            align-items: center;
+            gap: 12px;
+            text-align: left;
+            background: var(--card);
+            border: 1px solid var(--border);
+            color: var(--muted);
+            border-radius: 15px;
+            padding: 13px 16px;
+            font-size: 1rem;
+            font-weight: 600;
+            font-family: inherit;
+            cursor: pointer;
+            transition: border-color .15s, color .15s, background .15s;
+        }
+
+        .metric-btn:hover { border-color: var(--accent); color: var(--text); }
+
+        .metric-btn.active {
+            color: var(--text);
+            border-color: var(--accent);
+            background: linear-gradient(180deg, rgba(79,140,255,0.18), rgba(79,140,255,0.04));
+            box-shadow: inset 0 0 0 1px rgba(79,140,255,0.35);
+        }
+
+        .metric-icon { font-size: 1.6rem; line-height: 1; }
+
+        .metric-name { display: block; }
+
+        .metric-count {
+            display: block;
+            font-size: 0.78rem;
+            font-weight: 400;
+            color: var(--muted);
+            margin-top: 2px;
+        }
+
+        .metric-panel { display: none; }
+        .metric-panel.active { display: block; }
+
+        .section-title-first { margin-top: 6px; }
+
         .range-btn {
             background: var(--card);
             border: 1px solid var(--border);
@@ -253,6 +308,8 @@ HTML = r"""
         .section-title { font-weight: 600; margin: 30px 0 12px; font-size: 1.05rem; }
 
         @media (max-width: 600px) {
+            .metric-btn { padding: 10px 12px; font-size: 0.9rem; gap: 9px; }
+            .metric-icon { font-size: 1.3rem; }
             .current { font-size: 2.4rem; }
             .container { padding: 20px 14px 40px; }
             .chart-container { padding: 14px; }
@@ -266,7 +323,7 @@ HTML = r"""
 <div class="container">
     <header>
         <h1>JonasSearch</h1>
-        <div class="subtitle">Indexed Pages – Live Statistics</div>
+        <div class="subtitle" id="page-subtitle">Pages – Live Statistics</div>
         <div class="header-right">
             <a class="btn" id="btn-csv" href="#">Export CSV</a>
             <a class="btn" id="btn-json" href="#">Export JSON</a>
@@ -274,19 +331,28 @@ HTML = r"""
         </div>
     </header>
 
-    <div class="current-row">
+    <div class="metric-tabs" role="tablist" aria-label="Data type">
-        <div>
+        <button class="metric-btn active" data-metric="pages" role="tab" aria-selected="true">
-            <div class="current">
+            <span class="metric-icon" aria-hidden="true">📄</span>
-                <span id="current">–</span><span class="unit">Pages</span>
+            <span>
-            </div>
+                <span class="metric-name">Pages</span>
-            <div class="current-meta" id="current-meta">Loading…</div>
+                <span class="metric-count" id="tab-count-pages">– total</span>
-        </div>
+            </span>
-        <div>
+        </button>
-            <div class="current">
+        <button class="metric-btn" data-metric="images" role="tab" aria-selected="false">
-                <span id="current-images">–</span><span class="unit">Images</span>
+            <span class="metric-icon" aria-hidden="true">🖼️</span>
-            </div>
+            <span>
-            <div class="current-meta" id="current-images-meta">Loading…</div>
+                <span class="metric-name">Images</span>
-        </div>
+                <span class="metric-count" id="tab-count-images">– total</span>
+            </span>
+        </button>
+        <button class="metric-btn" data-metric="queue" role="tab" aria-selected="false">
+            <span class="metric-icon" aria-hidden="true">⏳</span>
+            <span>
+                <span class="metric-name">Queue</span>
+                <span class="metric-count" id="tab-count-queue">– pending</span>
+            </span>
+        </button>
     </div>
 
     <div class="badges">
@@ -305,6 +371,16 @@ HTML = r"""
         <button class="range-btn" data-range="6h">Last 6 h</button>
     </div>
 
+    <div class="metric-panel active" data-metric="pages" role="tabpanel">
+    <div class="current-row">
+        <div>
+            <div class="current">
+                <span id="current">–</span><span class="unit">Pages</span>
+            </div>
+            <div class="current-meta" id="current-meta">Loading…</div>
+        </div>
+    </div>
+
     <div class="grid">
         <div class="card">
             <div class="label">Growth / hour</div>
@@ -406,21 +482,39 @@ HTML = r"""
         <div id="heatmap"></div>
     </div>
 
-    <div class="section-title">Images – all graphs</div>
+    </div>
-    <div class="chart-sub">Only exact measurements (counted every 2 h) are plotted – the estimates in between are excluded.</div>
+
+    <div class="metric-panel" data-metric="images" role="tabpanel">
+    <div class="current-row">
+        <div>
+            <div class="current">
+                <span id="current-images">–</span><span class="unit">Images</span>
+            </div>
+            <div class="current-meta" id="current-images-meta">Loading…</div>
+        </div>
+    </div>
+
+    <div class="grid" id="summary-images"></div>
+
+    <div class="section-title section-title-first">Images – all graphs</div>
+    <div class="image-options">
+        <input type="checkbox" id="show-estimated-images">
+        <label for="show-estimated-images">Include estimated image counts in graphs</label>
+    </div>
+    <div class="image-options-hint" id="images-mode-note">Only exact measurements (counted every 2 h) are plotted. This setting is saved in this browser.</div>
     <div class="chart-sub" id="images-empty" style="display:none">
         No image counts yet – they appear after the next collector run with the image migration in place.
     </div>
 
     <div class="chart-container">
         <div class="chart-title">Images over time</div>
-        <div class="chart-sub">Image URLs found while crawling · exact count every 2 h · only exact measurements are plotted (no estimates) · dashed line = trend &amp; 24 h forecast</div>
+        <div class="chart-sub">Image URLs found while crawling · dashed line = trend &amp; 24 h forecast</div>
         <div class="chart-box"><canvas id="imagesMainChart"></canvas></div>
     </div>
 
     <div class="chart-container">
         <div class="chart-title">Change per 5 minutes</div>
-        <div class="chart-sub">Normalized to a 5-minute interval · spans between two exact counts (2 h) · grey = raw · colored = smoothed (moving average over 5 measurements)</div>
+        <div class="chart-sub">Normalized to a 5-minute interval between available image measurements · grey = raw · colored = smoothed (moving average over 5 measurements)</div>
         <div class="chart-box small"><canvas id="imagesDeltaChart"></canvas></div>
     </div>
 
@@ -455,6 +549,70 @@ HTML = r"""
         <div id="heatmap-images"></div>
     </div>
 
+    </div>
+
+    <div class="metric-panel" data-metric="queue" role="tabpanel">
+    <div class="current-row">
+        <div>
+            <div class="current">
+                <span id="current-pending">–</span><span class="unit">Pending</span>
+            </div>
+            <div class="current-meta" id="current-pending-meta">Loading…</div>
+        </div>
+    </div>
+
+    <div class="grid" id="summary-queue"></div>
+
+    <div class="section-title section-title-first">Queue – all graphs</div>
+    <div class="chart-sub">URLs waiting in crawl_queue (status = 'pending'). Counted with every 30-minute sample (exact COUNT). Falling graphs mean the crawler is working through the queue.</div>
+    <div class="chart-sub" id="pending-empty" style="display:none">
+        No queue counts yet – they appear after the next collector run.
+    </div>
+
+    <div class="chart-container">
+        <div class="chart-title">Pending URLs over time</div>
+        <div class="chart-sub">Queued URLs not yet crawled · dashed line = trend &amp; 24 h forecast</div>
+        <div class="chart-box"><canvas id="pendingMainChart"></canvas></div>
+    </div>
+
+    <div class="chart-container">
+        <div class="chart-title">Change per 5 minutes</div>
+        <div class="chart-sub">Normalized to a 5-minute interval · negative = the crawler worked the queue down · grey = raw · colored = smoothed (moving average over 5 measurements)</div>
+        <div class="chart-box small"><canvas id="pendingDeltaChart"></canvas></div>
+    </div>
+
+    <div class="chart-container">
+        <div class="chart-title">Cumulative net change</div>
+        <div class="chart-sub">Total queued URLs gained/lost since the start of the selected range · linear time axis</div>
+        <div class="chart-box small"><canvas id="pendingCumChart"></canvas></div>
+    </div>
+
+    <div class="chart-container">
+        <div class="chart-title">Queue change per hour (last 48 h)</div>
+        <div class="chart-sub">Sum of changes per hour bucket · green = queue grew, red = queue shrank</div>
+        <div class="chart-box small"><canvas id="pendingHourlyChart"></canvas></div>
+    </div>
+
+    <div class="chart-row">
+        <div class="chart-container">
+            <div class="chart-title">Queue change by hour of day</div>
+            <div class="chart-sub">Average change per 5 min, grouped by hour of day (whole history)</div>
+            <div class="chart-box small"><canvas id="pendingHodChart"></canvas></div>
+        </div>
+        <div class="chart-container">
+            <div class="chart-title">Records – biggest queue changes</div>
+            <div class="chart-sub">Largest gains and losses between two consecutive measurements (selected range)</div>
+            <div id="records-pending"></div>
+        </div>
+    </div>
+
+    <div class="section-title">Activity heatmap – queue change per hour of day</div>
+    <div class="chart-container heatmap">
+        <div class="chart-sub">Rows = days · columns = hours (0–23) · green = queue grew, red = queue shrank, grey = no data</div>
+        <div id="heatmap-pending"></div>
+    </div>
+    </div>
+
     <div class="footer" id="footer"></div>
 </div>
 
@@ -467,6 +625,7 @@ if (window.Chart && Chart.registerables) {
 
 const ACCENT = '#4f8cff';
 const ACCENT2 = '#b47aff';
+const ACCENT3 = '#f5a623';
 const UP = '#2ecc71';
 const DOWN = '#ff6b6b';
 const FALLBACK_INTERVAL_MS = 15 * 60 * 1000;
@@ -476,6 +635,8 @@ const REFRESH_S = 30;
 let rawValues = [];
 let intervalMs = FALLBACK_INTERVAL_MS;
 let range = 'all';
+let activeMetric = 'pages';
+let showEstimatedImages = false;
 let charts = {};
 let refreshLeft = REFRESH_S;
 
@@ -603,27 +764,36 @@ function parsePoints(values) {
         .map(function (v) {
             const images = (v.images === null || v.images === undefined)
                 ? null : Number(v.images);
+            const pending = (v.pending === null || v.pending === undefined)
+                ? null : Number(v.pending);
             return {
                 x: new Date(v.timestamp).getTime(),
                 y: v.count,
                 images: Number.isFinite(images) ? images : null,
-                imagesExact: v.images_exact === true
+                imagesExact: v.images_exact === true,
+                pending: Number.isFinite(pending) ? pending : null
             };
         })
         .filter(function (p) { return !isNaN(p.x); })
         .sort(function (a, b) { return a.x - b.x; });
 }
 
-// Nur exakte Bildzählungen (alle 2 h). Die dazwischen geschätzten Werte
+// Estimates are excluded by default; the Images section toggle can include them.
-// werden in keinem Diagramm dargestellt, damit alle Graphen ausschließlich
+function toImagePoints(points, includeEstimated) {
-// echte Messpunkte zeigen.
-function toImagePoints(points) {
     return (points || [])
         .filter(function (p) { return p.images !== null && p.images !== undefined; })
-        .filter(function (p) { return p.imagesExact; })
+        .filter(function (p) { return includeEstimated || p.imagesExact; })
         .map(function (p) { return { x: p.x, y: p.images }; });
 }
 
+// Pending queue entries are counted with every sample (exact COUNT), so
+// no filtering beyond "value present" is needed.
+function toPendingPoints(points) {
+    return (points || [])
+        .filter(function (p) { return p.pending !== null && p.pending !== undefined; })
+        .map(function (p) { return { x: p.x, y: p.pending }; });
+}
+
 const RANGE_SPANS = {
     '6h': 6 * 3600000,
     '24h': 24 * 3600000,
@@ -673,6 +843,17 @@ document.getElementById('btn-theme').addEventListener('click', function () {
     applyTheme(document.documentElement.dataset.theme === 'light' ? 'dark' : 'light');
 });
 
+(function () {
+    const toggle = document.getElementById('show-estimated-images');
+    try { showEstimatedImages = localStorage.getItem('js-show-estimated-images') === 'true'; } catch (e) {}
+    toggle.checked = showEstimatedImages;
+    toggle.addEventListener('change', function () {
+        showEstimatedImages = toggle.checked;
+        try { localStorage.setItem('js-show-estimated-images', String(showEstimatedImages)); } catch (e) {}
+        if (rawValues.length) renderAll();
+    });
+})();
+
 (function () {
     let saved = null;
     try { saved = localStorage.getItem('js-theme'); } catch (e) {}
@@ -680,6 +861,44 @@ document.getElementById('btn-theme').addEventListener('click', function () {
     document.getElementById('btn-theme').textContent = saved === 'light' ? '🌙 Dark' : '☀️ Light';
 })();
 
+/* ---------- metric tabs (pages / images / queue) ---------- */
+
+const METRIC_LABELS = { pages: 'Pages', images: 'Images', queue: 'Queue' };
+
+// Hidden panels must not render Chart.js instances: a canvas inside a
+// display:none container measures 0 px and breaks on later resize.
+function isVisible(el) {
+    const panel = el.closest('.metric-panel');
+    return !panel || panel.classList.contains('active');
+}
+
+function setMetric(next, persist) {
+    activeMetric = METRIC_LABELS[next] ? next : 'pages';
+
+    document.querySelectorAll('.metric-btn').forEach(function (btn) {
+        const on = btn.dataset.metric === activeMetric;
+        btn.classList.toggle('active', on);
+        btn.setAttribute('aria-selected', on ? 'true' : 'false');
+    });
+    document.querySelectorAll('.metric-panel').forEach(function (panel) {
+        panel.classList.toggle('active', panel.dataset.metric === activeMetric);
+    });
+
+    const label = METRIC_LABELS[activeMetric];
+    document.getElementById('page-subtitle').textContent = label + ' – Live Statistics';
+    document.title = 'JonasSearch – ' + label;
+
+    if (persist !== false) {
+        try { localStorage.setItem('js-metric', activeMetric); } catch (e) {}
+    }
+
+    if (rawValues.length) renderAll();
+}
+
+document.querySelectorAll('.metric-btn').forEach(function (btn) {
+    btn.addEventListener('click', function () { setMetric(btn.dataset.metric); });
+});
+
 /* ---------- range buttons ---------- */
 
 document.querySelectorAll('.range-btn').forEach(function (btn) {
@@ -702,16 +921,50 @@ function download(name, text, mime) {
     URL.revokeObjectURL(a.href);
 }
 
+// Export what the selected tab shows, so the CSV/JSON match the graphs.
+function exportSeries() {
+    const all = parsePoints(rawValues);
+
+    if (activeMetric === 'images') {
+        const values = all
+            .filter(function (p) { return p.images !== null; })
+            .map(function (p) {
+                return { timestamp: new Date(p.x).toISOString(), images: p.images, images_exact: p.imagesExact };
+            });
+        return { name: 'jonassearch_images', values: values };
+    }
+
+    if (activeMetric === 'queue') {
+        const values = all
+            .filter(function (p) { return p.pending !== null; })
+            .map(function (p) {
+                return { timestamp: new Date(p.x).toISOString(), pending: p.pending };
+            });
+        return { name: 'jonassearch_queue', values: values };
+    }
+
+    return { name: 'jonassearch_pages', values: rawValues };
+}
+
 document.getElementById('btn-csv').addEventListener('click', function (e) {
     e.preventDefault();
-    const lines = ['timestamp;count'];
+    const series = exportSeries();
-    rawValues.forEach(function (v) { lines.push(v.timestamp + ';' + v.count); });
+    const keys = series.values.length ? Object.keys(series.values[0]) : ['timestamp'];
-    download('jonassearch_pages.csv', lines.join('\n'), 'text/csv');
+    const lines = [keys.join(';')];
+    series.values.forEach(function (row) {
+        lines.push(keys.map(function (k) { return row[k] === undefined ? '' : row[k]; }).join(';'));
+    });
+    download(series.name + '.csv', lines.join('\n'), 'text/csv');
 });
 
 document.getElementById('btn-json').addEventListener('click', function (e) {
     e.preventDefault();
-    download('jonassearch_pages.json', JSON.stringify({ values: rawValues }, null, 2), 'application/json');
+    const series = exportSeries();
+    download(
+        series.name + '.json',
+        JSON.stringify({ metric: activeMetric, values: series.values }, null, 2),
+        'application/json'
+    );
 });
 
 /* ---------- chart helpers ---------- */
@@ -767,6 +1020,9 @@ function gradientFill(colorTop, colorBottom) {
 /* ---------- charts ---------- */
 
 function renderCharts(cfg) {
+    const target = document.getElementById(chartId(cfg.prefix, 'MainChart'));
+    if (!target || !isVisible(target)) return;
+
     const prefix = cfg.prefix;
     const points = cfg.points;
     const deltas = cfg.deltas;
@@ -1052,6 +1308,7 @@ function renderCharts(cfg) {
 
 function renderRecords(deltas, id, unit) {
     const el = document.getElementById(id);
+    if (!el || !isVisible(el)) return;
     if (deltas.length < 2) {
         el.innerHTML = '<div class="chart-sub">Not enough data yet.</div>';
         return;
@@ -1078,6 +1335,7 @@ function renderRecords(deltas, id, unit) {
 
 function renderHeatmap(points, id, unit) {
     const el = document.getElementById(id);
+    if (!el || !isVisible(el)) return;
     if (!points.length) { el.innerHTML = ''; return; }
 
     function localDayKey(d) {
@@ -1137,6 +1395,84 @@ function renderHeatmap(points, id, unit) {
     el.innerHTML = html;
 }
 
+/* ---------- summary cards for images / queue ---------- */
+
+// The pages tab keeps its static card grid (ids like stat-delta-6h);
+// images and queue get the same key numbers generated from their series.
+function renderSummary(containerId, all, points) {
+    const el = document.getElementById(containerId);
+    if (!el || !isVisible(el)) return;
+
+    if (!points.length) {
+        el.innerHTML = '<div class="chart-sub">No data in the selected range yet.</div>';
+        return;
+    }
+
+    const anchor = all[all.length - 1];
+    const first = points[0];
+    const hours = (anchor.x - first.x) / 3600000;
+    const perHour = hours > 0 ? (anchor.y - first.y) / hours : null;
+    const deltas = computeDeltas(points).deltas;
+
+    let peak = all[0], low = all[0];
+    all.forEach(function (p) {
+        if (p.y > peak.y) peak = p;
+        if (p.y < low.y) low = p;
+    });
+
+    const cards = [];
+
+    function signedCard(label, value, hint) {
+        const cls = value === null ? '' : (value >= 0 ? 'pos' : 'neg');
+        cards.push('<div class="card"><div class="label">' + label +
+            '</div><div class="value ' + cls + '">' +
+            (value === null ? '–' : fmtSigned(value)) +
+            '</div><div class="hint">' + hint + '</div></div>');
+    }
+
+    function plainCard(label, value, hint) {
+        cards.push('<div class="card"><div class="label">' + label +
+            '</div><div class="value">' + value +
+            '</div><div class="hint">' + hint + '</div></div>');
+    }
+
+    signedCard('Growth / hour', perHour, 'Average over full history');
+    signedCard('Growth / day', perHour === null ? null : perHour * 24, 'Average over full history');
+
+    const windows = [
+        ['6 h', 6 * 3600000, Math.max(3 * intervalMs, 3600000)],
+        ['24 h', 24 * 3600000, Math.max(6 * intervalMs, 3 * 3600000)],
+        ['7 days', 7 * 24 * 3600000, Math.max(12 * intervalMs, 6 * 3600000)]
+    ];
+    windows.forEach(function (win) {
+        const base = closestBefore(all, anchor.x - win[1], win[2]);
+        signedCard('Growth ' + win[0], base ? anchor.y - base.y : null,
+            base ? 'vs. measurement ' + fmtDur(anchor.x - base.x) + ' ago' : 'Not enough data yet');
+    });
+
+    let sum24 = 0;
+    const windowStart = Math.max(first.x, anchor.x - 24 * 3600000);
+    const coveredH = (anchor.x - windowStart) / 3600000;
+    deltas.forEach(function (d) {
+        if (d.x > anchor.x - 24 * 3600000) sum24 += d.diff;
+    });
+    signedCard('Avg / h (24 h)', coveredH >= 0.5 ? sum24 / coveredH : null, 'Mean of last 24 h changes');
+
+    plainCard('Peak', fmt(peak.y), 'On ' + fmtDateFull(peak.x));
+    plainCard('Lowest', fmt(low.y), 'On ' + fmtDateFull(low.x));
+
+    const regWindow = points.filter(function (p) { return p.x >= anchor.x - 2 * 3600000; }).slice(-96);
+    const reg = regress(regWindow);
+    if (reg) {
+        plainCard('Forecast next hour', fmt(anchor.y + reg.slope),
+            'Trend ' + fmtSigned(reg.slope) + '/h · R² = ' + reg.r2.toFixed(3));
+    } else {
+        plainCard('Forecast next hour', '–', 'Needs at least 2 measurements');
+    }
+
+    el.innerHTML = cards.join('');
+}
+
 /* ---------- main update ---------- */
 
 function renderAll() {
@@ -1147,6 +1483,8 @@ function renderAll() {
     const meta = document.getElementById('current-meta');
     const elImages = document.getElementById('current-images');
     const imagesMeta = document.getElementById('current-images-meta');
+    const elPending = document.getElementById('current-pending');
+    const pendingMeta = document.getElementById('current-pending-meta');
 
     document.getElementById('badge-count').textContent = 'Measurements: ' + all.length;
 
@@ -1155,6 +1493,8 @@ function renderAll() {
         meta.textContent = 'No measurements yet – the collector stores one every 30 minutes.';
         elImages.textContent = '–';
         imagesMeta.textContent = 'No image count yet';
+        elPending.textContent = '–';
+        pendingMeta.textContent = 'No queue count yet';
         setStatus('offline', 'No data');
         return;
     }
@@ -1181,6 +1521,21 @@ function renderAll() {
             ' · ' + fmtDateFull(last.x);
     }
 
+    if (last.pending === null) {
+        elPending.textContent = '–';
+        pendingMeta.textContent = 'No queue count yet';
+    } else {
+        elPending.textContent = fmt(last.pending);
+        pendingMeta.textContent = 'Waiting in queue · exact count · ' + fmtDateFull(last.x);
+    }
+
+    /* tab badges: always show the latest value of each data type */
+    document.getElementById('tab-count-pages').textContent = fmt(last.y) + ' total';
+    document.getElementById('tab-count-images').textContent = last.images === null
+        ? 'no data yet' : fmt(last.images) + ' total';
+    document.getElementById('tab-count-queue').textContent = last.pending === null
+        ? 'no data yet' : fmt(last.pending) + ' pending';
+
     document.getElementById('badge-since').textContent = 'First measurement: ' + fmtDateFull(first.x);
     document.getElementById('badge-updated').textContent = 'Last update: ' + fmtDate(last.x);
 
@@ -1295,19 +1650,42 @@ function renderAll() {
         elF.classList.add(fc >= fpoints[fpoints.length - 1].y ? 'pos' : 'neg');
         document.getElementById('stat-forecast-hint').textContent =
             'Trend ' + fmtSigned(reg.slope) + '/h · R² = ' + reg.r2.toFixed(3);
-        document.getElementById('footer').textContent =
-            'Forecast: linear least-squares regression over the last ' + regWindow.length +
-            ' measurements (' + fmtDur(fpoints[fpoints.length - 1].x - regWindow[0].x) +
-            '), extrapolated +24 h · R² = ' + reg.r2.toFixed(3) +
-            ' · Range: ' + (range === 'all' ? 'all' : range) + ' · All statistics are computed in the browser.';
     } else {
         document.getElementById('stat-forecast').textContent = '–';
         document.getElementById('stat-forecast-hint').textContent = 'Needs at least 2 measurements';
-        document.getElementById('footer').textContent = 'Forecast available once there are at least 2 measurements.';
     }
 
-    /* images: same graphs, but only exact measurements (every 2 h) */
+    document.getElementById('footer').textContent =
-    const imageAll = toImagePoints(all);
+        'Showing ' + METRIC_LABELS[activeMetric] +
+        ' · Range: ' + (range === 'all' ? 'all data' : range) +
+        ' · All statistics are computed in the browser.';
+
+    /* queue: same graphs for the pending URLs in crawl_queue */
+    const pendingAll = toPendingPoints(all);
+    const pendingFpoints = filterRange(pendingAll, last.x);
+    const pendingDeltas = computeDeltas(pendingFpoints).deltas;
+    const pendingSmoothY = movingAverage(pendingDeltas.map(function (d) { return d.y; }), DELTA_SMOOTH_WINDOW);
+    const pendingSmoothedDeltas = pendingDeltas.map(function (d, i) {
+        return { x: d.x, y: pendingSmoothY[i], raw: d.y, dt: d.dt, diff: d.diff };
+    });
+    const pendingEmpty = document.getElementById('pending-empty');
+    if (!pendingAll.length) {
+        pendingEmpty.textContent = all.some(function (p) { return p.pending !== null; })
+            ? 'No queue counts in the selected range.'
+            : 'No queue counts yet – they appear after the next collector run.';
+        pendingEmpty.style.display = '';
+    } else if (!pendingFpoints.length) {
+        pendingEmpty.textContent = 'No queue counts in the selected range.';
+        pendingEmpty.style.display = '';
+    } else {
+        pendingEmpty.style.display = 'none';
+    }
+
+    /* Images: estimates are optional and controlled by the Images-section toggle. */
+    const imageAll = toImagePoints(all, showEstimatedImages);
+    document.getElementById('images-mode-note').textContent = showEstimatedImages
+        ? 'Exact and estimated measurements are plotted. Estimates come from a 1% table sample; this setting is saved in this browser.'
+        : 'Only exact measurements (counted every 2 h) are plotted. This setting is saved in this browser.';
     const imageFpoints = filterRange(imageAll, last.x);
     const imageDeltas = computeDeltas(imageFpoints).deltas;
     const imageSmoothY = movingAverage(imageDeltas.map(function (d) { return d.y; }), DELTA_SMOOTH_WINDOW);
@@ -1318,11 +1696,11 @@ function renderAll() {
     const hasImageValues = all.some(function (p) { return p.images !== null; });
     if (!imageAll.length) {
         imagesEmpty.textContent = hasImageValues
-            ? 'Only estimated image counts so far – the graphs show exact counts only (every 2 h).'
+            ? 'Only estimated image counts so far – turn on “Include estimated image counts in graphs” to show them.'
             : 'No image counts yet – they appear after the next collector run with the image migration in place.';
         imagesEmpty.style.display = '';
     } else if (!imageFpoints.length) {
-        imagesEmpty.textContent = 'No exact image counts in the selected range (counted every 2 h).';
+        imagesEmpty.textContent = 'No image counts in the selected range.';
         imagesEmpty.style.display = '';
     } else {
         imagesEmpty.style.display = 'none';
@@ -1351,10 +1729,25 @@ function renderAll() {
         gradTop: 'rgba(180,122,255,0.28)',
         gradBottom: 'rgba(180,122,255,0.02)'
     });
+    renderCharts({
+        prefix: 'pending',
+        points: pendingFpoints,
+        deltas: pendingDeltas,
+        smoothedDeltas: pendingSmoothedDeltas,
+        unit: 'Pending',
+        accent: ACCENT3,
+        accentBar: 'rgba(245,166,35,0.65)',
+        gradTop: 'rgba(245,166,35,0.28)',
+        gradBottom: 'rgba(245,166,35,0.02)'
+    });
     renderRecords(deltas, 'records', 'Pages');
     renderRecords(imageDeltas, 'records-images', 'Images');
+    renderRecords(pendingDeltas, 'records-pending', 'Pending');
     renderHeatmap(all, 'heatmap', 'Pages');
     renderHeatmap(imageAll, 'heatmap-images', 'Images');
+    renderHeatmap(pendingAll, 'heatmap-pending', 'Pending');
+    renderSummary('summary-images', imageAll, imageFpoints);
+    renderSummary('summary-queue', pendingAll, pendingFpoints);
 }
 
 async function update() {
@@ -1380,6 +1773,13 @@ setInterval(function () {
 
 update();
 setInterval(update, REFRESH_S * 1000);
+
+// Restore the last selected tab (pages / images / queue).
+(function () {
+    let saved = 'pages';
+    try { saved = localStorage.getItem('js-metric') || 'pages'; } catch (e) {}
+    setMetric(saved, false);
+})();
 </script>
 
 </body>
@@ -1427,6 +1827,10 @@ ESTIMATE_IMAGE_SQL = (
     percent=IMAGE_SAMPLE_PERCENT,
 )
 
+# URLs waiting in the crawler queue. Nutzt den Index
+# (status, priority, discovered_at); kein Sampling, der Wert ist exakt.
+PENDING_QUEUE_SQL = "SELECT COUNT(*) FROM crawl_queue WHERE status = 'pending';"
+
 
 def _psql_scalar(sql):
     """Eine einzelne psql-Zeile ("count|images") abholen."""
@@ -1518,12 +1922,32 @@ def get_stats(exact_images):
     return count, images, False
 
 
+def get_pending():
+    """
+    Anzahl der URLs mit status='pending' in der crawl_queue.
+
+    Ist die Tabelle (noch) nicht erreichbar, wird None geliefert - die
+    Messung wird trotzdem gespeichert, nur ohne Queue-Wert.
+    """
+    try:
+        return int(_psql_scalar(PENDING_QUEUE_SQL).strip())
+    except subprocess.CalledProcessError as exc:
+        detail = ((exc.stderr or "").strip().splitlines() or [str(exc)])[-1]
+        print(f"Queue-Statistik nicht verfügbar ({detail}); zähle nur Seiten/Bilder.")
+        return None
+    except (TypeError, ValueError):
+        # Leere/überraschende psql-Ausgabe darf die übrigen Werte nicht kosten.
+        print("Queue-Statistik nicht verfügbar (unerwartete psql-Ausgabe).")
+        return None
+
+
 def collect():
     while True:
         try:
             data = load_data()
             exact = images_exact_due(data)
             count, images, images_exact = get_stats(exact)
+            pending = get_pending()
 
             if images is None:
                 # Stichprobe ohne Treffer (praktisch nur bei sehr großen
@@ -1538,6 +1962,7 @@ def collect():
                 "count": count,
                 "images": images,
                 "images_exact": images_exact,
+                "pending": pending,
             }
 
             data.append(entry)
@@ -1546,7 +1971,8 @@ def collect():
             print(
                 f"[{entry['timestamp']}] "
                 f"Pages: {count} · Images: {images} "
-                f"({'exakt' if images_exact else 'geschätzt'})"
+                f"({'exakt' if images_exact else 'geschätzt'}) "
+                f"· Pending: {pending if pending is not None else 'n/a'}"
             )
 
         except Exception as e:

web/test_stats.py

@@ -117,6 +117,59 @@ class GetStatsTest(unittest.TestCase):
                          "SELECT COUNT(*) FROM pages;")
 
 
+class GetPendingTest(unittest.TestCase):
+
+    @mock.patch.object(stats_app, "_psql_scalar")
+    def test_counts_pending_queue_rows(self, psql):
+        psql.return_value = "1384711"
+        self.assertEqual(stats_app.get_pending(), 1384711)
+        self.assertIn("FROM crawl_queue", stats_app.PENDING_QUEUE_SQL)
+        self.assertIn("status = 'pending'", stats_app.PENDING_QUEUE_SQL)
+
+    @mock.patch.object(stats_app, "_psql_scalar")
+    def test_unavailable_queue_returns_none(self, psql):
+        psql.side_effect = subprocess.CalledProcessError(
+            1, ["psql"], stderr='ERROR: relation "crawl_queue" does not exist'
+        )
+        self.assertIsNone(stats_app.get_pending())
+
+    @mock.patch.object(stats_app, "_psql_scalar")
+    def test_unexpected_output_returns_none(self, psql):
+        psql.return_value = ""
+        self.assertIsNone(stats_app.get_pending())
+
+
+class CollectTest(unittest.TestCase):
+
+    def _run_collect(self, get_stats, get_pending):
+        with mock.patch.object(stats_app, "load_data", return_value=[]), \
+                mock.patch.object(stats_app, "save_data") as save_data, \
+                mock.patch.object(stats_app, "get_stats", get_stats), \
+                mock.patch.object(stats_app, "get_pending", get_pending), \
+                mock.patch.object(stats_app, "time") as time_mock:
+            time_mock.sleep.side_effect = KeyboardInterrupt
+            with self.assertRaises(KeyboardInterrupt):
+                stats_app.collect()
+        return save_data.call_args[0][0][0]
+
+    def test_pending_is_stored_with_every_sample(self):
+        entry = self._run_collect(
+            mock.Mock(return_value=(1000, 2000, True)),
+            mock.Mock(return_value=4711),
+        )
+        self.assertEqual(entry["pending"], 4711)
+        self.assertEqual(entry["count"], 1000)
+        self.assertEqual(entry["images"], 2000)
+
+    def test_missing_queue_does_not_break_the_sample(self):
+        entry = self._run_collect(
+            mock.Mock(return_value=(1000, 2000, False)),
+            mock.Mock(return_value=None),
+        )
+        self.assertIsNone(entry["pending"])
+        self.assertEqual(entry["count"], 1000)
+
+
 class StatsPageTest(unittest.TestCase):
 
     def test_page_contains_image_counter(self):
@@ -159,15 +212,27 @@ class StatsPageTest(unittest.TestCase):
         self.assertIn('id="heatmap-images"', html)
         self.assertIn('id="images-empty"', html)
 
-        # the image series skips entries without an image count (old data)
+        # Image estimates are excluded by default and can be included with the persistent toggle.
-        # and only plots exact measurements - no estimates between the 2 h counts
+        self.assertIn('id="show-estimated-images"', html)
-        self.assertIn("function toImagePoints(points)", html)
+        self.assertIn('type="checkbox" id="show-estimated-images"', html)
+        self.assertIn("Include estimated image counts in graphs", html)
+        self.assertIn("function toImagePoints(points, includeEstimated)", html)
         self.assertIn("p.images !== null && p.images !== undefined", html)
-        self.assertIn(".filter(function (p) { return p.imagesExact; })", html)
+        self.assertIn("includeEstimated || p.imagesExact", html)
-        self.assertIn("only exact measurements are plotted (no estimates)", html)
+        self.assertIn("localStorage.getItem('js-show-estimated-images')", html)
+        self.assertIn("localStorage.setItem('js-show-estimated-images'", html)
+        self.assertIn("toImagePoints(all, showEstimatedImages)", html)
         self.assertIn("Only exact measurements (counted every 2 h) are plotted", html)
+        self.assertIn("Exact and estimated measurements are plotted", html)
         self.assertIn("Only estimated image counts so far", html)
-        self.assertIn("No exact image counts in the selected range", html)
+        self.assertIn("No image counts in the selected range", html)
+        # The headline counter stays the latest collected value; only graphs toggle estimates.
+        self.assertIn("elImages.textContent = fmt(last.images)", html)
+        # Images estimates affect graphs only; other series keep their own points.
+        self.assertIn("renderCharts({\n        prefix: '',\n        points: fpoints", html)
+        self.assertIn("renderCharts({\n        prefix: 'pending',\n        points: pendingFpoints", html)
+        self.assertIn("renderHeatmap(imageAll, 'heatmap-images', 'Images')", html)
+        self.assertIn("renderRecords(imageDeltas, 'records-images', 'Images')", html)
         # existing page chart ids must stay lowercase (mainChart, not MainChart)
         self.assertIn("function chartId(prefix, suffix)", html)
         self.assertIn("chartId(prefix, 'MainChart')", html)
@@ -182,6 +247,79 @@ class StatsPageTest(unittest.TestCase):
         self.assertIn("renderHeatmap(all, 'heatmap', 'Pages')", html)
         self.assertIn("renderHeatmap(imageAll, 'heatmap-images', 'Images')", html)
 
+    def test_page_has_metric_tabs_with_one_panel_each(self):
+        client = stats_app.app.test_client()
+        response = client.get("/")
+        self.assertEqual(response.status_code, 200)
+        html = response.get_data(as_text=True)
+
+        # three equally styled tab buttons (pages / images / queue) with icons
+        self.assertIn('class="metric-btn', html)
+        for metric in ("pages", "images", "queue"):
+            self.assertIn('data-metric="' + metric + '"', html)
+        self.assertIn("metric-icon", html)
+        self.assertIn("📄", html)
+        self.assertIn("🖼️", html)
+        self.assertIn("⏳", html)
+
+        # every metric has its own panel; only the initial one is active
+        self.assertEqual(html.count('class="metric-panel active" data-metric="pages"'), 1)
+        for metric in ("images", "queue"):
+            self.assertIn('class="metric-panel" data-metric="' + metric + '"', html)
+        self.assertIn('id="summary-images"', html)
+        self.assertIn('id="summary-queue"', html)
+
+        # switching tabs hides the other panels and is remembered
+        self.assertIn("function setMetric(next, persist)", html)
+        self.assertIn("panel.classList.toggle('active', panel.dataset.metric === activeMetric)", html)
+        self.assertIn("localStorage.setItem('js-metric', activeMetric)", html)
+        self.assertIn("localStorage.getItem('js-metric')", html)
+
+        # hidden panels must not create Chart.js instances (canvas has 0 px)
+        self.assertIn("function isVisible(el)", html)
+        self.assertIn("if (!target || !isVisible(target)) return;", html)
+
+        # summary cards are generated for the non-page tabs
+        self.assertIn("function renderSummary(containerId, all, points)", html)
+        self.assertIn("renderSummary('summary-images', imageAll, imageFpoints)", html)
+        self.assertIn("renderSummary('summary-queue', pendingAll, pendingFpoints)", html)
+
+        # exports follow the selected tab
+        self.assertIn("function exportSeries()", html)
+        self.assertIn("return { name: 'jonassearch_images'", html)
+        self.assertIn("return { name: 'jonassearch_queue'", html)
+
+    def test_page_contains_pending_queue_counter_and_graphs(self):
+        client = stats_app.app.test_client()
+        response = client.get("/")
+        self.assertEqual(response.status_code, 200)
+        html = response.get_data(as_text=True)
+
+        self.assertIn('id="current-pending"', html)
+        self.assertIn('id="current-pending-meta"', html)
+        self.assertIn("Queue – all graphs", html)
+        for canvas in (
+            "pendingMainChart",
+            "pendingDeltaChart",
+            "pendingCumChart",
+            "pendingHourlyChart",
+            "pendingHodChart",
+        ):
+            self.assertIn('id="' + canvas + '"', html)
+        self.assertIn('id="records-pending"', html)
+        self.assertIn('id="heatmap-pending"', html)
+        self.assertIn('id="pending-empty"', html)
+
+        # pending values are parsed and plotted without sampling/filtering
+        self.assertIn("function toPendingPoints(points)", html)
+        self.assertIn("p.pending !== null && p.pending !== undefined", html)
+        self.assertIn("filterRange(pendingAll, last.x)", html)
+        self.assertIn("unit: 'Pending'", html)
+        self.assertIn("renderRecords(pendingDeltas, 'records-pending', 'Pending')", html)
+        self.assertIn("renderHeatmap(pendingAll, 'heatmap-pending', 'Pending')", html)
+        self.assertIn("No queue counts yet", html)
+        self.assertIn("No queue counts in the selected range.", html)
+
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)