add better analysis

.gitignore

@@ -174,3 +174,7 @@ cython_debug/
 # PyPI configuration file
 .pypirc
 
+# own
+
+
+web/page_counts.json

web/app.py

@@ -9,7 +9,7 @@ from pathlib import Path
 
 from flask import Flask, jsonify, render_template_string
 
-DB_DATA = Path("page_counts.json")
+DB_DATA = Path(__file__).parent / "page_counts.json"
 INTERVAL = 15 * 60  # 15 Minuten
 
 app = Flask(__name__)
@@ -25,112 +25,436 @@ HTML = """
     <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
 
     <style>
+        :root {
+            --bg: #0f1115;
+            --card: #1b1e24;
+            --border: #2c313a;
+            --text: #e8eaed;
+            --muted: #9aa0a6;
+            --accent: #4f8cff;
+            --accent2: #b47aff;
+            --up: #2ecc71;
+            --down: #ff6b6b;
+            --warn: #f5a623;
+        }
+
+        * { box-sizing: border-box; }
+
         body {
             margin: 0;
-            background: #111;
+            background: var(--bg);
-            color: #eee;
+            color: var(--text);
-            font-family: Arial, sans-serif;
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Arial, sans-serif;
         }
 
-        .container {
+        .container { max-width: 1200px; margin: auto; padding: 30px 24px 60px; }
-            max-width: 1200px;
-            margin: auto;
-            padding: 30px;
-        }
 
-        h1 {
+        header { display: flex; flex-wrap: wrap; align-items: baseline; gap: 8px 16px; }
-            margin-bottom: 5px;
+
-        }
+        h1 { margin: 0; font-size: 1.9rem; letter-spacing: 0.3px; }
+
+        .subtitle { color: var(--muted); }
 
         .current {
-            font-size: 2.5rem;
+            font-size: 3.2rem;
-            margin: 20px 0;
+            font-weight: 700;
+            margin: 18px 0 4px;
+            line-height: 1;
         }
 
+        .current .unit {
+            font-size: 1.1rem;
+            font-weight: 400;
+            color: var(--muted);
+            margin-left: 8px;
+        }
+
+        .current-meta { color: var(--muted); margin-bottom: 24px; }
+
+        .badges { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 22px; }
+
+        .badge {
+            display: inline-flex;
+            align-items: center;
+            gap: 6px;
+            background: var(--card);
+            border: 1px solid var(--border);
+            color: var(--muted);
+            font-size: 0.8rem;
+            padding: 5px 10px;
+            border-radius: 999px;
+        }
+
+        .dot { width: 8px; height: 8px; border-radius: 50%; background: var(--warn); display: inline-block; }
+        .dot.ok { background: var(--up); }
+        .dot.stale { background: var(--warn); }
+        .dot.offline { background: var(--down); }
+        .dot.live { animation: pulse 2s ease-in-out infinite; }
+
+        @keyframes pulse {
+            0%, 100% { opacity: 1; }
+            50% { opacity: 0.35; }
+        }
+
+        .grid {
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(165px, 1fr));
+            gap: 14px;
+            margin-bottom: 22px;
+        }
+
+        .card {
+            background: var(--card);
+            border: 1px solid var(--border);
+            border-radius: 15px;
+            padding: 16px 18px;
+            min-width: 0;
+        }
+
+        .card .label {
+            color: var(--muted);
+            font-size: 0.78rem;
+            text-transform: uppercase;
+            letter-spacing: 0.8px;
+            margin-bottom: 8px;
+        }
+
+        .card .value { font-size: 1.45rem; font-weight: 600; }
+
+        .card .hint { color: var(--muted); font-size: 0.78rem; margin-top: 6px; }
+
+        .pos { color: var(--up); }
+        .neg { color: var(--down); }
+
         .chart-container {
-            background: #1b1b1b;
+            background: var(--card);
+            border: 1px solid var(--border);
             padding: 20px;
             border-radius: 15px;
+            margin-bottom: 22px;
+        }
+
+        .chart-box { position: relative; height: 340px; }
+        .chart-box.small { height: 260px; }
+
+        .chart-title { font-weight: 600; margin-bottom: 2px; }
+
+        .chart-sub { color: var(--muted); font-size: 0.82rem; margin-bottom: 14px; }
+
+        .footer { color: var(--muted); font-size: 0.8rem; margin-top: 8px; }
+
+        @media (max-width: 600px) {
+            .current { font-size: 2.4rem; }
+            .container { padding: 20px 14px 40px; }
+            .chart-container { padding: 14px; }
+            .chart-box { height: 260px; }
+            .chart-box.small { height: 200px; }
         }
     </style>
 </head>
 
 <body>
 <div class="container">
-    <h1>JonasSearch</h1>
+    <header>
-    <div>Indexed Pages</div>
+        <h1>JonasSearch</h1>
+        <div class="subtitle">Indexed Pages – Live Statistics</div>
+    </header>
 
     <div class="current">
-        <span id="current">–</span> Pages
+        <span id="current">–</span><span class="unit">Pages</span>
+    </div>
+    <div class="current-meta" id="current-meta">Loading…</div>
+
+    <div class="badges">
+        <span class="badge"><span class="dot" id="status-dot"></span><span id="status-text">Loading…</span></span>
+        <span class="badge" id="badge-interval">Interval: –</span>
+        <span class="badge" id="badge-since">First measurement: –</span>
+        <span class="badge" id="badge-count">Measurements: –</span>
+        <span class="badge" id="badge-updated">Last update: –</span>
+    </div>
+
+    <div class="grid">
+        <div class="card">
+            <div class="label">Growth / hour</div>
+            <div class="value" id="stat-per-hour">–</div>
+            <div class="hint">Average over full history</div>
+        </div>
+        <div class="card">
+            <div class="label">Growth / day</div>
+            <div class="value" id="stat-per-day">–</div>
+            <div class="hint">Average over full history</div>
+        </div>
+        <div class="card">
+            <div class="label">Last 5 min</div>
+            <div class="value" id="stat-delta-5m">–</div>
+            <div class="hint" id="stat-delta-5m-hint">Latest interval change</div>
+        </div>
+        <div class="card">
+            <div class="label">Last hour</div>
+            <div class="value" id="stat-delta-1h">–</div>
+            <div class="hint" id="stat-delta-1h-hint">vs. measurement ~1 h ago</div>
+        </div>
+        <div class="card">
+            <div class="label">Last 24 h</div>
+            <div class="value" id="stat-delta-24h">–</div>
+            <div class="hint" id="stat-delta-24h-hint">vs. measurement ~24 h ago</div>
+        </div>
+        <div class="card">
+            <div class="label">Forecast next hour</div>
+            <div class="value" id="stat-forecast">–</div>
+            <div class="hint" id="stat-forecast-hint">Linear trend extrapolation</div>
+        </div>
     </div>
 
     <div class="chart-container">
-        <canvas id="chart"></canvas>
+        <div class="chart-title">Indexed pages over time</div>
+        <div class="chart-sub">X-axis shows the real measurement timestamps · dashed line = linear-regression trend &amp; 24 h forecast</div>
+        <div class="chart-box"><canvas id="mainChart"></canvas></div>
     </div>
+
+    <div class="chart-container">
+        <div class="chart-title">Change per 5 minutes</div>
+        <div class="chart-sub">Computed from the raw measurements: each change is normalized to a 5-minute interval</div>
+        <div class="chart-box small"><canvas id="deltaChart"></canvas></div>
+    </div>
+
+    <div class="footer" id="footer"></div>
 </div>
 
 <script>
-let chart = null;
+'use strict';
 
-async function update() {
+if (window.Chart && Chart.registerables) {
-    const response = await fetch("/api/data");
+    Chart.register.apply(Chart, Chart.registerables);
-    const data = await response.json();
+}
 
-    document.getElementById("current").textContent =
+Chart.defaults.color = '#9aa0a6';
-        data.values.length
+Chart.defaults.borderColor = '#2c313a';
-            ? data.values[data.values.length - 1].count.toLocaleString("de-DE")
+Chart.defaults.font.family = "-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Arial, sans-serif";
-            : "–";
 
-    const labels = data.values.map(x =>
+const ACCENT = '#4f8cff';
-        new Date(x.timestamp).toLocaleString("de-DE")
+const ACCENT2 = '#b47aff';
-    );
+const UP = '#2ecc71';
+const DOWN = '#ff6b6b';
+const FALLBACK_INTERVAL_MS = 15 * 60 * 1000;
 
-    const values = data.values.map(x => x.count);
+function fmt(n) { return Math.round(n).toLocaleString('de-DE'); }
 
-    if (chart) {
+function fmtSigned(n) {
-        chart.data.labels = labels;
+    const s = Math.round(n).toLocaleString('de-DE');
-        chart.data.datasets[0].data = values;
+    return n > 0 ? '+' + s : s;
-        chart.update();
+}
+
+function fmtDate(ms) {
+    return new Date(ms).toLocaleString('de-DE', {
+        day: '2-digit', month: '2-digit', hour: '2-digit', minute: '2-digit'
+    });
+}
+
+function fmtDateFull(ms) {
+    return new Date(ms).toLocaleString('de-DE');
+}
+
+function fmtDur(ms) {
+    const min = Math.round(ms / 60000);
+    if (min < 1) return '<1 min';
+    if (min < 60) return min + ' min';
+    const h = Math.floor(min / 60);
+    const m = min % 60;
+    if (h < 24) return m ? (h + ' h ' + m + ' min') : (h + ' h');
+    const d = Math.floor(h / 24);
+    return d + ' d ' + (h % 24) + ' h';
+}
+
+function regress(points) {
+    const n = points.length;
+    if (n < 2) return null;
+    const t0 = points[0].x;
+    let sx = 0, sy = 0, sxy = 0, sxx = 0;
+    for (const p of points) {
+        const x = (p.x - t0) / 3600000;
+        sx += x; sy += p.y; sxy += x * p.y; sxx += x * x;
+    }
+    const den = n * sxx - sx * sx;
+    if (den === 0) return null;
+    const b = (n * sxy - sx * sy) / den;
+    const a = (sy - b * sx) / n;
+    let ssTot = 0, ssRes = 0;
+    const mean = sy / n;
+    for (const p of points) {
+        const x = (p.x - t0) / 3600000;
+        const pred = a + b * x;
+        ssTot += Math.pow(p.y - mean, 2);
+        ssRes += Math.pow(p.y - pred, 2);
+    }
+    const r2 = ssTot === 0 ? 1 : Math.max(0, Math.min(1, 1 - ssRes / ssTot));
+    return {
+        slope: b,
+        r2: r2,
+        at: function (ms) { return a + b * ((ms - t0) / 3600000); }
+    };
+}
+
+function median(arr) {
+    if (!arr.length) return null;
+    const s = arr.slice().sort(function (a, b) { return a - b; });
+    const mid = Math.floor(s.length / 2);
+    return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
+}
+
+function closestBefore(points, targetMs, maxLagMs) {
+    let best = null;
+    for (const p of points) {
+        if (p.x <= targetMs && (!best || p.x > best.x)) best = p;
+    }
+    if (best && targetMs - best.x <= maxLagMs) return best;
+    return null;
+}
+
+function setDeltaCard(id, idHint, delta, spanMs, per5) {
+    const el = document.getElementById(id);
+    const hint = document.getElementById(idHint);
+    if (delta === null) {
+        el.textContent = '–';
+        el.classList.remove('pos', 'neg');
+        hint.textContent = 'Not enough data yet';
         return;
     }
+    el.textContent = fmtSigned(delta);
+    el.classList.remove('pos', 'neg');
+    el.classList.add(delta >= 0 ? 'pos' : 'neg');
+    hint.textContent = fmtSigned(spanMs > 0 ? delta * (3600000 / spanMs) : 0) + ' Pages/h · ' + fmtSigned(per5) + ' per 5 min over ' + fmtDur(spanMs);
+}
 
-    chart = new Chart(document.getElementById("chart"), {
+let mainChart = null;
-        type: "line",
+let deltaChart = null;
+let intervalMs = FALLBACK_INTERVAL_MS;
 
+function gradientFill(context) {
+    const chart = context.chart;
+    const ctx = chart.ctx;
+    const area = chart.chartArea;
+    if (!area) return 'rgba(79,140,255,0.15)';
+    const g = ctx.createLinearGradient(0, area.top, 0, area.bottom);
+    g.addColorStop(0, 'rgba(79,140,255,0.28)');
+    g.addColorStop(1, 'rgba(79,140,255,0.02)');
+    return g;
+}
+
+const xAxisConfig = {
+    type: 'linear',
+    ticks: {
+        maxTicksLimit: 8,
+        maxRotation: 0,
+        autoSkip: true,
+        callback: function (v) { return fmtDate(v); }
+    },
+    grid: { color: 'rgba(255,255,255,0.05)' }
+};
+
+function buildCharts(points, trendPoints, deltas) {
+    mainChart = new Chart(document.getElementById('mainChart'), {
+        type: 'line',
         data: {
-            labels: labels,
+            datasets: [
-
+                {
-            datasets: [{
+                    label: 'Pages',
-                label: "Pages",
+                    data: points,
-                data: values,
+                    parsing: false,
-                tension: 0.25,
+                    borderColor: ACCENT,
-                pointRadius: 2,
+                    borderWidth: 2,
-                fill: true
+                    pointRadius: 2.5,
-            }]
+                    pointHoverRadius: 5,
+                    tension: 0.25,
+                    fill: true,
+                    backgroundColor: gradientFill,
+                    spanGaps: false
+                },
+                {
+                    label: 'Trend / Forecast',
+                    data: trendPoints,
+                    parsing: false,
+                    borderColor: ACCENT2,
+                    borderDash: [6, 6],
+                    borderWidth: 2,
+                    pointRadius: 0,
+                    fill: false,
+                    spanGaps: true
+                }
+            ]
         },
-
         options: {
             responsive: true,
-
+            maintainAspectRatio: false,
+            normalized: true,
+            interaction: { mode: 'nearest', axis: 'x', intersect: false },
             scales: {
-                x: {
+                x: xAxisConfig,
-                    ticks: {
-                        color: "#aaa"
-                    }
-                },
-
                 y: {
-                    beginAtZero: false,
+                    grace: '5%',
-                    ticks: {
+                    ticks: { callback: function (v) { return fmt(v); } },
-                        color: "#aaa"
+                    grid: { color: 'rgba(255,255,255,0.05)' }
+                }
+            },
+            plugins: {
+                legend: { labels: { usePointStyle: true, boxWidth: 8 } },
+                tooltip: {
+                    callbacks: {
+                        title: function (items) { return fmtDateFull(items[0].parsed.x); },
+                        label: function (item) {
+                            return item.datasetIndex === 0
+                                ? (fmt(item.parsed.y) + ' Pages')
+                                : ('Trend: ' + fmt(item.parsed.y) + ' Pages');
+                        }
+                    }
+                }
+            }
+        }
+    });
+
+    deltaChart = new Chart(document.getElementById('deltaChart'), {
+        type: 'bar',
+        data: {
+            datasets: [{
+                label: 'Change per 5 min',
+                data: deltas,
+                parsing: false,
+                backgroundColor: function (c) {
+                    return c.raw && c.raw.y >= 0 ? 'rgba(46,204,113,0.75)' : 'rgba(255,107,107,0.75)';
+                },
+                borderColor: function (c) {
+                    return c.raw && c.raw.y >= 0 ? UP : DOWN;
+                },
+                borderWidth: 1,
+                maxBarThickness: 28
+            }]
+        },
+        options: {
+            responsive: true,
+            maintainAspectRatio: false,
+            normalized: true,
+            interaction: { mode: 'nearest', axis: 'x', intersect: false },
+            scales: {
+                x: xAxisConfig,
+                y: {
+                    ticks: { callback: function (v) { return fmtSigned(v); } },
+                    grid: {
+                        color: function (c) { return c.tick.value === 0 ? 'rgba(255,255,255,0.35)' : 'rgba(255,255,255,0.05)'; }
                     }
                 }
             },
-
             plugins: {
-                legend: {
+                legend: { display: false },
-                    labels: {
+                tooltip: {
-                        color: "#eee"
+                    callbacks: {
+                        title: function (items) { return fmtDateFull(items[0].parsed.x); },
+                        label: function (item) {
+                            const raw = item.raw;
+                            return [
+                                fmtSigned(raw.y) + ' Pages per 5 min',
+                                'Actual change: ' + fmtSigned(raw.diff) + ' in ' + fmtDur(raw.dt)
+                            ];
+                        }
                     }
                 }
             }
@@ -138,9 +462,162 @@ async function update() {
     });
 }
 
-update();
+async function update() {
+    let payload;
+    try {
+        const response = await fetch('/api/data');
+        payload = await response.json();
+    } catch (e) {
+        setStatus('offline', 'Fetch error – retrying…');
+        return;
+    }
 
-// Browser alle 30 Sekunden aktualisieren
+    const measurements = (payload && payload.values) ? payload.values.slice() : [];
+    measurements.sort(function (a, b) { return new Date(a.timestamp) - new Date(b.timestamp); });
+
+    const points = measurements.map(function (v) {
+        return { x: new Date(v.timestamp).getTime(), y: v.count };
+    });
+
+    const now = Date.now();
+    const elCurrent = document.getElementById('current');
+    const meta = document.getElementById('current-meta');
+
+    document.getElementById('badge-count').textContent = 'Measurements: ' + points.length;
+
+    if (!points.length) {
+        elCurrent.textContent = '–';
+        meta.textContent = 'No measurements yet – the collector stores one every 15 minutes.';
+        setStatus('offline', 'No data');
+        return;
+    }
+
+    const last = points[points.length - 1];
+    const first = points[0];
+
+    const dts = [];
+    for (let i = 1; i < points.length; i++) dts.push(points[i].x - points[i - 1].x);
+    intervalMs = median(dts) || FALLBACK_INTERVAL_MS;
+    document.getElementById('badge-interval').textContent =
+        'Interval: ~' + fmtDur(intervalMs);
+
+    elCurrent.textContent = fmt(last.y);
+    meta.textContent = 'Last measurement: ' + fmtDateFull(last.x) +
+        ' · next expected in ~' + fmtDur(Math.max(0, last.x + intervalMs - now));
+
+    document.getElementById('badge-since').textContent = 'First measurement: ' + fmtDateFull(first.x);
+    document.getElementById('badge-updated').textContent = 'Last update: ' + fmtDate(last.x);
+
+    const age = now - last.x;
+    if (age <= 3 * intervalMs) {
+        setStatus('ok', 'Live · ' + fmtDur(age) + ' ago');
+    } else if (age <= 24 * 3600000) {
+        setStatus('stale', 'Stale · last update ' + fmtDur(age) + ' ago');
+    } else {
+        setStatus('offline', 'No data for ' + fmtDur(age));
+    }
+
+    const totalHours = (last.x - first.x) / 3600000;
+    const perHour = totalHours > 0 ? (last.y - first.y) / totalHours : null;
+    const elPh = document.getElementById('stat-per-hour');
+    const elPd = document.getElementById('stat-per-day');
+    if (perHour === null) {
+        elPh.textContent = '–';
+        elPd.textContent = '–';
+    } else {
+        elPh.textContent = fmtSigned(perHour);
+        elPh.classList.remove('pos', 'neg');
+        elPh.classList.add(perHour >= 0 ? 'pos' : 'neg');
+        elPd.textContent = fmtSigned(perHour * 24);
+        elPd.classList.remove('pos', 'neg');
+        elPd.classList.add(perHour >= 0 ? 'pos' : 'neg');
+    }
+
+    let deltas = [];
+    for (let i = 1; i < points.length; i++) {
+        const dt = points[i].x - points[i - 1].x;
+        const diff = points[i].y - points[i - 1].y;
+        if (dt <= 0 || dt > 4 * intervalMs) continue;
+        const per5 = dt < 60000 ? diff : diff * (5 * 60000 / dt);
+        deltas.push({ x: points[i].x, y: per5, dt: dt, diff: diff });
+    }
+
+    const lastDelta = deltas.length ? deltas[deltas.length - 1] : null;
+    if (lastDelta) {
+        const el = document.getElementById('stat-delta-5m');
+        el.textContent = fmtSigned(lastDelta.diff);
+        el.classList.remove('pos', 'neg');
+        el.classList.add(lastDelta.diff >= 0 ? 'pos' : 'neg');
+        document.getElementById('stat-delta-5m-hint').textContent =
+            fmtSigned(lastDelta.y) + ' per 5 min · over ' + fmtDur(lastDelta.dt);
+    } else {
+        document.getElementById('stat-delta-5m').textContent = '–';
+        document.getElementById('stat-delta-5m-hint').textContent = 'Needs at least 2 measurements';
+    }
+
+    const span1h = closestBefore(points, last.x - 3600000, Math.max(3 * intervalMs, 30 * 60000));
+    setDeltaCard('stat-delta-1h', 'stat-delta-1h-hint',
+        span1h ? last.y - span1h.y : null,
+        span1h ? last.x - span1h.x : 0,
+        span1h ? (last.y - span1h.y) * (3600000 / Math.max(1, last.x - span1h.x)) * (5 / 60) : 0);
+
+    const span24h = closestBefore(points, last.x - 24 * 3600000, Math.max(6 * intervalMs, 3 * 3600000));
+    setDeltaCard('stat-delta-24h', 'stat-delta-24h-hint',
+        span24h ? last.y - span24h.y : null,
+        span24h ? last.x - span24h.x : 0,
+        span24h ? (last.y - span24h.y) * (3600000 / Math.max(1, last.x - span24h.x)) * (5 / 60) : 0);
+
+    const regWindow = [];
+    for (let i = points.length - 1; i >= 0; i--) {
+        regWindow.unshift(points[i]);
+        if (points.length - i >= 96) break;
+        if (last.x - points[i].x >= 2 * 3600000) break;
+    }
+    const reg = regress(regWindow);
+
+    let trendPoints = [];
+    if (reg) {
+        const horizon = last.x + 24 * 3600000;
+        trendPoints = [
+            { x: regWindow[0].x, y: reg.at(regWindow[0].x) },
+            { x: horizon, y: reg.at(horizon) }
+        ];
+        const elF = document.getElementById('stat-forecast');
+        const fc = last.y + reg.slope;
+        elF.textContent = fmt(fc);
+        elF.classList.remove('pos', 'neg');
+        elF.classList.add(fc >= last.y ? 'pos' : 'neg');
+        document.getElementById('stat-forecast-hint').textContent =
+            'Trend ' + fmtSigned(reg.slope) + '/h · R² = ' + reg.r2.toFixed(3);
+        document.getElementById('footer').textContent =
+            'Forecast: linear least-squares regression over the last ' + regWindow.length +
+            ' measurements (' + fmtDur(last.x - regWindow[0].x) + '), extrapolated +24 h · R² = ' +
+            reg.r2.toFixed(3) + ' · All statistics are computed in the browser.';
+    } else {
+        document.getElementById('stat-forecast').textContent = '–';
+        document.getElementById('stat-forecast-hint').textContent = 'Needs at least 2 measurements';
+        document.getElementById('footer').textContent =
+            'Forecast available once there are at least 2 measurements.';
+    }
+
+    if (!mainChart) {
+        buildCharts(points, trendPoints, deltas);
+    } else {
+        mainChart.data.datasets[0].data = points;
+        mainChart.data.datasets[1].data = trendPoints;
+        mainChart.update('none');
+        deltaChart.data.datasets[0].data = deltas;
+        deltaChart.update('none');
+    }
+}
+
+function setStatus(kind, text) {
+    const dot = document.getElementById('status-dot');
+    dot.className = 'dot ' + kind + (kind === 'ok' ? ' live' : '');
+    document.getElementById('status-text').textContent = text;
+}
+
+update();
 setInterval(update, 30000);
 </script>