Files
search/web/app.py
T
2026-10-18 11:21:25 +02:00

1590 lines
56 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
import subprocess
import json
import time
import threading
from datetime import datetime, timedelta
from pathlib import Path
from flask import Flask, jsonify, render_template_string, Response
DB_DATA = Path(__file__).parent / "page_counts.json"
INTERVAL = 30 * 60 # 30 Minuten
# Bilder werden nur alle 2 Stunden exakt gezählt (SUM über die ~10-GB-
# Tabelle). Dazwischen wird eine 1-%-Stichprobe (TABLESAMPLE SYSTEM)
# hochgerechnet - das kostet nur einen Bruchteil der IO.
EXACT_IMAGE_INTERVAL = 2 * 60 * 60 # Sekunden
IMAGE_SAMPLE_PERCENT = 1
# Sehr kleine Tabellen: Stichprobe kann leer sein; dort ist der exakte
# Count ohnehin billig.
SAMPLE_EXACT_FALLBACK_ROWS = 100_000
app = Flask(__name__)
# r""" raw string: preserves \n etc. for the embedded JavaScript
# (a normal string would eat the backslashes and break the JS)
HTML = r"""
<!DOCTYPE html>
<html lang="de">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>JonasSearch – Pages</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<style>
:root {
--bg: #0f1115;
--card: #1b1e24;
--border: #2c313a;
--text: #e8eaed;
--muted: #9aa0a6;
--accent: #4f8cff;
--accent2: #b47aff;
--up: #2ecc71;
--down: #ff6b6b;
--warn: #f5a623;
}
html[data-theme="light"] {
--bg: #f4f6fa;
--card: #ffffff;
--border: #d8dde5;
--text: #1a1d23;
--muted: #6b7280;
--accent: #2f6fe4;
--accent2: #8a4fe0;
}
* { box-sizing: border-box; }
body {
margin: 0;
background: var(--bg);
color: var(--text);
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Arial, sans-serif;
}
.container { max-width: 1200px; margin: auto; padding: 30px 24px 60px; }
header {
display: flex;
flex-wrap: wrap;
align-items: baseline;
gap: 8px 16px;
position: sticky;
top: 0;
z-index: 10;
background: var(--bg);
padding: 10px 0 14px;
border-bottom: 1px solid var(--border);
}
h1 { margin: 0; font-size: 1.9rem; letter-spacing: 0.3px; }
.subtitle { color: var(--muted); }
.header-right { margin-left: auto; display: flex; gap: 8px; align-items: center; }
.btn {
background: var(--card);
border: 1px solid var(--border);
color: var(--text);
font-size: 0.8rem;
padding: 6px 12px;
border-radius: 999px;
cursor: pointer;
text-decoration: none;
}
.btn:hover { border-color: var(--accent); }
.current {
font-size: 3.2rem;
font-weight: 700;
margin: 18px 0 4px;
line-height: 1;
}
.current .unit {
font-size: 1.1rem;
font-weight: 400;
color: var(--muted);
margin-left: 8px;
}
.current-meta { color: var(--muted); margin-bottom: 24px; }
.current-row { display: flex; flex-wrap: wrap; gap: 0 56px; }
.badges { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 14px; }
.badge {
display: inline-flex;
align-items: center;
gap: 6px;
background: var(--card);
border: 1px solid var(--border);
color: var(--muted);
font-size: 0.8rem;
padding: 5px 10px;
border-radius: 999px;
}
.dot { width: 8px; height: 8px; border-radius: 50%; background: var(--warn); display: inline-block; }
.dot.ok { background: var(--up); }
.dot.stale { background: var(--warn); }
.dot.offline { background: var(--down); }
.dot.live { animation: pulse 2s ease-in-out infinite; }
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.35; }
}
.ranges { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 18px; }
.range-btn {
background: var(--card);
border: 1px solid var(--border);
color: var(--muted);
font-size: 0.82rem;
padding: 6px 14px;
border-radius: 999px;
cursor: pointer;
}
.range-btn.active {
color: #fff;
background: var(--accent);
border-color: var(--accent);
}
.grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(165px, 1fr));
gap: 14px;
margin-bottom: 22px;
}
.card {
background: var(--card);
border: 1px solid var(--border);
border-radius: 15px;
padding: 16px 18px;
min-width: 0;
}
.card .label {
color: var(--muted);
font-size: 0.78rem;
text-transform: uppercase;
letter-spacing: 0.8px;
margin-bottom: 8px;
}
.card .value { font-size: 1.45rem; font-weight: 600; }
.card .hint { color: var(--muted); font-size: 0.78rem; margin-top: 6px; }
.pos { color: var(--up); }
.neg { color: var(--down); }
.chart-container {
background: var(--card);
border: 1px solid var(--border);
padding: 20px;
border-radius: 15px;
margin-bottom: 22px;
}
.chart-row { display: grid; grid-template-columns: repeat(auto-fit, minmax(420px, 1fr)); gap: 22px; }
.chart-row .chart-container { margin-bottom: 0; }
@media (max-width: 950px) {
.chart-row { grid-template-columns: 1fr; }
.chart-row .chart-container { margin-bottom: 22px; }
}
.chart-box { position: relative; height: 340px; }
.chart-box.small { height: 260px; }
.chart-title { font-weight: 600; margin-bottom: 2px; }
.chart-sub { color: var(--muted); font-size: 0.82rem; margin-bottom: 14px; }
.heatmap { overflow-x: auto; }
.heatmap table { border-collapse: collapse; width: 100%; min-width: 720px; }
.heatmap td, .heatmap th {
font-size: 0.68rem;
color: var(--muted);
padding: 2px;
text-align: center;
}
.heatmap td.day { text-align: right; padding-right: 8px; white-space: nowrap; }
.heatmap .cell {
width: 100%;
min-width: 22px;
height: 18px;
border-radius: 4px;
background: rgba(128,128,128,0.12);
}
.footer { color: var(--muted); font-size: 0.8rem; margin-top: 8px; }
table.records { width: 100%; border-collapse: collapse; font-size: 0.85rem; }
table.records th, table.records td {
text-align: left;
padding: 7px 8px;
border-bottom: 1px solid var(--border);
}
table.records th { color: var(--muted); font-weight: 500; font-size: 0.75rem; text-transform: uppercase; letter-spacing: 0.6px; }
table.records td.num { text-align: right; font-variant-numeric: tabular-nums; }
.section-title { font-weight: 600; margin: 30px 0 12px; font-size: 1.05rem; }
@media (max-width: 600px) {
.current { font-size: 2.4rem; }
.container { padding: 20px 14px 40px; }
.chart-container { padding: 14px; }
.chart-box { height: 260px; }
.chart-box.small { height: 200px; }
}
</style>
</head>
<body>
<div class="container">
<header>
<h1>JonasSearch</h1>
<div class="subtitle">Indexed Pages – Live Statistics</div>
<div class="header-right">
<a class="btn" id="btn-csv" href="#">Export CSV</a>
<a class="btn" id="btn-json" href="#">Export JSON</a>
<button class="btn" id="btn-theme" title="Toggle light/dark">☀️ Light</button>
</div>
</header>
<div class="current-row">
<div>
<div class="current">
<span id="current">–</span><span class="unit">Pages</span>
</div>
<div class="current-meta" id="current-meta">Loading…</div>
</div>
<div>
<div class="current">
<span id="current-images">–</span><span class="unit">Images</span>
</div>
<div class="current-meta" id="current-images-meta">Loading…</div>
</div>
</div>
<div class="badges">
<span class="badge"><span class="dot" id="status-dot"></span><span id="status-text">Loading…</span></span>
<span class="badge" id="badge-interval">Interval: –</span>
<span class="badge" id="badge-since">First measurement: –</span>
<span class="badge" id="badge-count">Measurements: –</span>
<span class="badge" id="badge-updated">Last update: –</span>
<span class="badge" id="badge-refresh">Next refresh: –</span>
</div>
<div class="ranges">
<button class="range-btn active" data-range="all">All</button>
<button class="range-btn" data-range="7d">Last 7 days</button>
<button class="range-btn" data-range="24h">Last 24 h</button>
<button class="range-btn" data-range="6h">Last 6 h</button>
</div>
<div class="grid">
<div class="card">
<div class="label">Growth / hour</div>
<div class="value" id="stat-per-hour">–</div>
<div class="hint">Average over full history</div>
</div>
<div class="card">
<div class="label">Growth / day</div>
<div class="value" id="stat-per-day">–</div>
<div class="hint">Average over full history</div>
</div>
<div class="card">
<div class="label">Growth 6 h</div>
<div class="value" id="stat-delta-6h">–</div>
<div class="hint" id="stat-delta-6h-hint">vs. measurement ~6 h ago</div>
</div>
<div class="card">
<div class="label">Growth 24 h</div>
<div class="value" id="stat-delta-24h">–</div>
<div class="hint" id="stat-delta-24h-hint">vs. measurement ~24 h ago</div>
</div>
<div class="card">
<div class="label">Growth 7 days</div>
<div class="value" id="stat-delta-7d">–</div>
<div class="hint" id="stat-delta-7d-hint">vs. measurement ~7 d ago</div>
</div>
<div class="card">
<div class="label">Avg / h (24 h)</div>
<div class="value" id="stat-avg-hour">–</div>
<div class="hint">Mean of last 24 h changes</div>
</div>
<div class="card">
<div class="label">Best hour</div>
<div class="value" id="stat-best-hour">–</div>
<div class="hint" id="stat-best-hour-hint">Largest gain within one hour</div>
</div>
<div class="card">
<div class="label">Worst hour</div>
<div class="value" id="stat-worst-hour">–</div>
<div class="hint" id="stat-worst-hour-hint">Largest loss within one hour</div>
</div>
<div class="card">
<div class="label">Peak</div>
<div class="value" id="stat-peak">–</div>
<div class="hint" id="stat-peak-hint">Highest measurement ever</div>
</div>
<div class="card">
<div class="label">Lowest</div>
<div class="value" id="stat-min">–</div>
<div class="hint" id="stat-min-hint">Lowest measurement ever</div>
</div>
<div class="card">
<div class="label">Forecast next hour</div>
<div class="value" id="stat-forecast">–</div>
<div class="hint" id="stat-forecast-hint">Linear trend extrapolation</div>
</div>
</div>
<div class="chart-container">
<div class="chart-title">Indexed pages over time</div>
<div class="chart-sub">Real measurement timestamps · dashed line = linear-regression trend &amp; 24 h forecast</div>
<div class="chart-box"><canvas id="mainChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Change per 5 minutes</div>
<div class="chart-sub">Normalized to a 5-minute interval · grey = raw · colored = smoothed (moving average over 5 measurements)</div>
<div class="chart-box small"><canvas id="deltaChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Cumulative net growth</div>
<div class="chart-sub">Total pages gained/lost since the start of the selected range · linear time axis</div>
<div class="chart-box small"><canvas id="cumChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Growth per hour (last 48 h)</div>
<div class="chart-sub">Sum of changes per hour bucket · green = growth, red = loss</div>
<div class="chart-box small"><canvas id="hourlyChart"></canvas></div>
</div>
<div class="chart-row">
<div class="chart-container">
<div class="chart-title">Growth by hour of day</div>
<div class="chart-sub">Average gain per 5 min, grouped by hour of day (whole history) · shows when the crawler is most active</div>
<div class="chart-box small"><canvas id="hodChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Records – biggest changes</div>
<div class="chart-sub">Largest gains and losses between two consecutive measurements (selected range)</div>
<div id="records"></div>
</div>
</div>
<div class="section-title">Activity heatmap – gain per hour of day</div>
<div class="chart-container heatmap">
<div class="chart-sub">Rows = days · columns = hours (0–23) · green = growth, red = loss, grey = no data</div>
<div id="heatmap"></div>
</div>
<div class="section-title">Images – all graphs</div>
<div class="chart-sub">Only exact measurements (counted every 2 h) are plotted – the estimates in between are excluded.</div>
<div class="chart-sub" id="images-empty" style="display:none">
No image counts yet – they appear after the next collector run with the image migration in place.
</div>
<div class="chart-container">
<div class="chart-title">Images over time</div>
<div class="chart-sub">Image URLs found while crawling · exact count every 2 h · only exact measurements are plotted (no estimates) · dashed line = trend &amp; 24 h forecast</div>
<div class="chart-box"><canvas id="imagesMainChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Change per 5 minutes</div>
<div class="chart-sub">Normalized to a 5-minute interval · spans between two exact counts (2 h) · grey = raw · colored = smoothed (moving average over 5 measurements)</div>
<div class="chart-box small"><canvas id="imagesDeltaChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Cumulative net growth</div>
<div class="chart-sub">Total images gained/lost since the start of the selected range · linear time axis</div>
<div class="chart-box small"><canvas id="imagesCumChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Images per hour (last 48 h)</div>
<div class="chart-sub">Sum of changes per hour bucket · green = growth, red = loss</div>
<div class="chart-box small"><canvas id="imagesHourlyChart"></canvas></div>
</div>
<div class="chart-row">
<div class="chart-container">
<div class="chart-title">Image growth by hour of day</div>
<div class="chart-sub">Average gain per 5 min, grouped by hour of day (whole history)</div>
<div class="chart-box small"><canvas id="imagesHodChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Records – biggest image changes</div>
<div class="chart-sub">Largest gains and losses between two consecutive measurements (selected range)</div>
<div id="records-images"></div>
</div>
</div>
<div class="section-title">Activity heatmap – image gain per hour of day</div>
<div class="chart-container heatmap">
<div class="chart-sub">Rows = days · columns = hours (0–23) · green = growth, red = loss, grey = no data</div>
<div id="heatmap-images"></div>
</div>
<div class="footer" id="footer"></div>
</div>
<script>
'use strict';
if (window.Chart && Chart.registerables) {
Chart.register.apply(Chart, Chart.registerables);
}
const ACCENT = '#4f8cff';
const ACCENT2 = '#b47aff';
const UP = '#2ecc71';
const DOWN = '#ff6b6b';
const FALLBACK_INTERVAL_MS = 15 * 60 * 1000;
const DELTA_SMOOTH_WINDOW = 5;
const REFRESH_S = 30;
let rawValues = [];
let intervalMs = FALLBACK_INTERVAL_MS;
let range = 'all';
let charts = {};
let refreshLeft = REFRESH_S;
function cssVar(name) {
return getComputedStyle(document.documentElement).getPropertyValue(name).trim();
}
function borderColor() { return cssVar('--border') || '#2c313a'; }
function mutedColor() { return cssVar('--muted') || '#9aa0a6'; }
function fmt(n) { return Math.round(n).toLocaleString('de-DE'); }
function fmtSigned(n) {
const s = Math.round(n).toLocaleString('de-DE');
return n > 0 ? '+' + s : s;
}
function fmtDate(ms) {
return new Date(ms).toLocaleString('de-DE', {
day: '2-digit', month: '2-digit', hour: '2-digit', minute: '2-digit'
});
}
function fmtDateFull(ms) {
return new Date(ms).toLocaleString('de-DE');
}
function fmtDay(ms) {
return new Date(ms).toLocaleDateString('de-DE', { day: '2-digit', month: '2-digit' });
}
function fmtDur(ms) {
const min = Math.round(ms / 60000);
if (min < 1) return '<1 min';
if (min < 60) return min + ' min';
const h = Math.floor(min / 60);
const m = min % 60;
if (h < 24) return m ? (h + ' h ' + m + ' min') : (h + ' h');
const d = Math.floor(h / 24);
return d + ' d ' + (h % 24) + ' h';
}
function regress(points) {
const n = points.length;
if (n < 2) return null;
const t0 = points[0].x;
let sx = 0, sy = 0, sxy = 0, sxx = 0;
for (const p of points) {
const x = (p.x - t0) / 3600000;
sx += x; sy += p.y; sxy += x * p.y; sxx += x * x;
}
const den = n * sxx - sx * sx;
if (den === 0) return null;
const b = (n * sxy - sx * sy) / den;
const a = (sy - b * sx) / n;
let ssTot = 0, ssRes = 0;
const mean = sy / n;
for (const p of points) {
const x = (p.x - t0) / 3600000;
const pred = a + b * x;
ssTot += Math.pow(p.y - mean, 2);
ssRes += Math.pow(p.y - pred, 2);
}
const r2 = ssTot === 0 ? 1 : Math.max(0, Math.min(1, 1 - ssRes / ssTot));
return {
slope: b,
r2: r2,
at: function (ms) { return a + b * ((ms - t0) / 3600000); }
};
}
function movingAverage(values, window) {
const half = Math.floor(window / 2);
return values.map(function (_, i) {
let sum = 0, count = 0;
for (let j = i - half; j <= i + half; j++) {
if (j >= 0 && j < values.length) { sum += values[j]; count++; }
}
return sum / count;
});
}
function median(arr) {
if (!arr.length) return null;
const s = arr.slice().sort(function (a, b) { return a - b; });
const mid = Math.floor(s.length / 2);
return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
}
function closestBefore(points, targetMs, maxLagMs) {
let best = null;
for (const p of points) {
if (p.x <= targetMs && (!best || p.x > best.x)) best = p;
}
if (best && targetMs - best.x <= maxLagMs) return best;
return null;
}
function setDeltaCard(id, idHint, delta, spanMs, per5) {
const el = document.getElementById(id);
const hint = document.getElementById(idHint);
if (delta === null) {
el.textContent = '–';
el.classList.remove('pos', 'neg');
hint.textContent = 'Not enough data yet';
return;
}
el.textContent = fmtSigned(delta);
el.classList.remove('pos', 'neg');
el.classList.add(delta >= 0 ? 'pos' : 'neg');
hint.textContent = fmtSigned(spanMs > 0 ? delta * (3600000 / spanMs) : 0) + ' Pages/h · ' + fmtSigned(per5) + ' per 5 min over ' + fmtDur(spanMs);
}
function setSigned(id, value) {
const el = document.getElementById(id);
el.textContent = fmtSigned(value);
el.classList.remove('pos', 'neg');
el.classList.add(value >= 0 ? 'pos' : 'neg');
}
/* ---------- data helpers ---------- */
function parsePoints(values) {
return (values || [])
.map(function (v) {
const images = (v.images === null || v.images === undefined)
? null : Number(v.images);
return {
x: new Date(v.timestamp).getTime(),
y: v.count,
images: Number.isFinite(images) ? images : null,
imagesExact: v.images_exact === true
};
})
.filter(function (p) { return !isNaN(p.x); })
.sort(function (a, b) { return a.x - b.x; });
}
// Nur exakte Bildzählungen (alle 2 h). Die dazwischen geschätzten Werte
// werden in keinem Diagramm dargestellt, damit alle Graphen ausschließlich
// echte Messpunkte zeigen.
function toImagePoints(points) {
return (points || [])
.filter(function (p) { return p.images !== null && p.images !== undefined; })
.filter(function (p) { return p.imagesExact; })
.map(function (p) { return { x: p.x, y: p.images }; });
}
const RANGE_SPANS = {
'6h': 6 * 3600000,
'24h': 24 * 3600000,
'7d': 7 * 24 * 3600000
};
function filterRange(points, anchorX) {
if (range === 'all' || !points.length) return points;
const last = anchorX === undefined ? points[points.length - 1].x : anchorX;
const span = RANGE_SPANS[range] || RANGE_SPANS['24h'];
return points.filter(function (p) { return p.x >= last - span; });
}
function computeDeltas(points) {
const dts = [];
for (let i = 1; i < points.length; i++) dts.push(points[i].x - points[i - 1].x);
const iv = median(dts) || FALLBACK_INTERVAL_MS;
const deltas = [];
for (let i = 1; i < points.length; i++) {
const dt = points[i].x - points[i - 1].x;
const diff = points[i].y - points[i - 1].y;
if (dt <= 0 || dt > 4 * iv) continue;
const per5 = dt < 60000 ? diff : diff * (5 * 60000 / dt);
deltas.push({ x: points[i].x, y: per5, dt: dt, diff: diff });
}
return { deltas: deltas, interval: iv };
}
/* ---------- status ---------- */
function setStatus(kind, text) {
const dot = document.getElementById('status-dot');
dot.className = 'dot ' + kind + (kind === 'ok' ? ' live' : '');
document.getElementById('status-text').textContent = text;
}
/* ---------- theme ---------- */
function applyTheme(theme) {
document.documentElement.dataset.theme = theme;
document.getElementById('btn-theme').textContent = theme === 'light' ? '🌙 Dark' : '☀️ Light';
try { localStorage.setItem('js-theme', theme); } catch (e) {}
if (rawValues.length) renderAll(); // re-render charts with new palette
}
document.getElementById('btn-theme').addEventListener('click', function () {
applyTheme(document.documentElement.dataset.theme === 'light' ? 'dark' : 'light');
});
(function () {
let saved = null;
try { saved = localStorage.getItem('js-theme'); } catch (e) {}
if (saved) document.documentElement.dataset.theme = saved;
document.getElementById('btn-theme').textContent = saved === 'light' ? '🌙 Dark' : '☀️ Light';
})();
/* ---------- range buttons ---------- */
document.querySelectorAll('.range-btn').forEach(function (btn) {
btn.addEventListener('click', function () {
document.querySelectorAll('.range-btn').forEach(function (b) { b.classList.remove('active'); });
btn.classList.add('active');
range = btn.dataset.range;
update();
});
});
/* ---------- export ---------- */
function download(name, text, mime) {
const blob = new Blob([text], { type: mime });
const a = document.createElement('a');
a.href = URL.createObjectURL(blob);
a.download = name;
a.click();
URL.revokeObjectURL(a.href);
}
document.getElementById('btn-csv').addEventListener('click', function (e) {
e.preventDefault();
const lines = ['timestamp;count'];
rawValues.forEach(function (v) { lines.push(v.timestamp + ';' + v.count); });
download('jonassearch_pages.csv', lines.join('\n'), 'text/csv');
});
document.getElementById('btn-json').addEventListener('click', function (e) {
e.preventDefault();
download('jonassearch_pages.json', JSON.stringify({ values: rawValues }, null, 2), 'application/json');
});
/* ---------- chart helpers ---------- */
function destroyCharts() {
Object.keys(charts).forEach(function (k) {
if (charts[k]) charts[k].destroy();
});
charts = {};
}
function baseScales(yFmt) {
return {
x: {
type: 'linear',
ticks: {
maxTicksLimit: 8,
maxRotation: 0,
autoSkip: true,
color: mutedColor(),
callback: function (v) { return fmtDate(v); }
},
grid: { color: 'rgba(128,128,128,0.15)' }
},
y: {
grace: '5%',
ticks: { color: mutedColor(), callback: yFmt },
grid: { color: 'rgba(128,128,128,0.15)' }
}
};
}
function tooltipTitle(items) { return fmtDateFull(items[0].parsed.x); }
// Pages keep the existing camelCase ids (mainChart); other metrics get a prefix
// (imagesMainChart).
function chartId(prefix, suffix) {
return prefix ? prefix + suffix : suffix.charAt(0).toLowerCase() + suffix.slice(1);
}
function gradientFill(colorTop, colorBottom) {
return function (context) {
const chart = context.chart;
const area = chart.chartArea;
if (!area) return colorBottom;
const g = chart.ctx.createLinearGradient(0, area.top, 0, area.bottom);
g.addColorStop(0, colorTop);
g.addColorStop(1, colorBottom);
return g;
};
}
/* ---------- charts ---------- */
function renderCharts(cfg) {
const prefix = cfg.prefix;
const points = cfg.points;
const deltas = cfg.deltas;
const smoothedDeltas = cfg.smoothedDeltas;
const unit = cfg.unit;
const accent = cfg.accent;
const accentBar = cfg.accentBar;
const grid = 'rgba(128,128,128,0.15)';
/* 1. main chart + trend */
const regWindow = [];
for (let i = points.length - 1; i >= 0; i--) {
regWindow.unshift(points[i]);
if (points.length - i >= 96) break;
if (points[points.length - 1].x - points[i].x >= 2 * 3600000) break;
}
const reg = regress(regWindow);
let trendPoints = [];
if (reg) {
const horizon = points[points.length - 1].x + 24 * 3600000;
trendPoints = [
{ x: regWindow[0].x, y: reg.at(regWindow[0].x) },
{ x: horizon, y: reg.at(horizon) }
];
}
charts[prefix + 'main'] = new Chart(document.getElementById(chartId(prefix, 'MainChart')), {
type: 'line',
data: {
datasets: [
{
label: unit,
data: points,
parsing: false,
borderColor: accent,
borderWidth: 2,
pointRadius: 2.5,
pointHoverRadius: 5,
tension: 0.25,
fill: true,
backgroundColor: gradientFill(cfg.gradTop, cfg.gradBottom),
spanGaps: false
},
{
label: 'Trend / Forecast',
data: trendPoints,
parsing: false,
borderColor: ACCENT2,
borderDash: [6, 6],
borderWidth: 2,
pointRadius: 0,
fill: false,
spanGaps: true
}
]
},
options: {
responsive: true,
maintainAspectRatio: false,
normalized: true,
interaction: { mode: 'nearest', axis: 'x', intersect: false },
scales: baseScales(function (v) { return fmt(v); }),
plugins: {
legend: { labels: { usePointStyle: true, boxWidth: 8, color: mutedColor() } },
tooltip: {
callbacks: {
title: tooltipTitle,
label: function (item) {
return item.datasetIndex === 0
? (fmt(item.parsed.y) + ' ' + unit)
: ('Trend: ' + fmt(item.parsed.y) + ' ' + unit);
}
}
}
}
}
});
/* 2. delta chart */
charts[prefix + 'delta'] = new Chart(document.getElementById(chartId(prefix, 'DeltaChart')), {
type: 'line',
data: {
datasets: [
{
label: 'Raw',
data: deltas,
parsing: false,
borderColor: 'rgba(128,128,128,0.30)',
borderWidth: 1,
pointRadius: 0,
tension: 0,
fill: false
},
{
label: 'Change per 5 min',
data: smoothedDeltas,
parsing: false,
borderColor: accent,
borderWidth: 2.5,
pointRadius: 0,
pointHoverRadius: 4,
tension: 0.35,
fill: false,
segment: {
borderColor: function (c) { return c.p1.parsed.y >= 0 ? UP : DOWN; }
}
}
]
},
options: {
responsive: true,
maintainAspectRatio: false,
normalized: true,
interaction: { mode: 'nearest', axis: 'x', intersect: false },
scales: {
x: baseScales(function () { return ''; }).x,
y: {
ticks: { color: mutedColor(), callback: function (v) { return fmtSigned(v); } },
grid: {
color: function (c) { return c.tick.value === 0 ? 'rgba(128,128,128,0.5)' : grid; }
}
}
},
plugins: {
legend: {
display: true,
labels: {
usePointStyle: true,
boxWidth: 8,
color: mutedColor(),
filter: function (item) { return item.text !== 'Raw'; }
}
},
tooltip: {
filter: function (item) { return item.datasetIndex === 1; },
callbacks: {
title: tooltipTitle,
label: function (item) {
const raw = item.raw;
return [
'Smoothed: ' + fmtSigned(raw.y) + ' ' + unit + ' per 5 min',
'Raw: ' + fmtSigned(raw.raw) + ' per 5 min (change ' + fmtSigned(raw.diff) + ' in ' + fmtDur(raw.dt) + ')'
];
}
}
}
}
}
});
/* 3. cumulative net growth */
const cum = points.map(function (p, i) {
return { x: p.x, y: p.y - points[0].y };
});
charts[prefix + 'cum'] = new Chart(document.getElementById(chartId(prefix, 'CumChart')), {
type: 'line',
data: {
datasets: [{
label: 'Net growth since range start',
data: cum,
parsing: false,
borderColor: UP,
borderWidth: 2,
pointRadius: 0,
pointHoverRadius: 4,
tension: 0.25,
fill: true,
backgroundColor: gradientFill('rgba(46,204,113,0.22)', 'rgba(46,204,113,0.02)')
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
normalized: true,
interaction: { mode: 'nearest', axis: 'x', intersect: false },
scales: {
x: baseScales(function () { return ''; }).x,
y: {
ticks: { color: mutedColor(), callback: function (v) { return fmtSigned(v); } },
grid: {
color: function (c) { return c.tick.value === 0 ? 'rgba(128,128,128,0.5)' : grid; }
}
}
},
plugins: {
legend: { labels: { usePointStyle: true, boxWidth: 8, color: mutedColor() } },
tooltip: {
callbacks: {
title: tooltipTitle,
label: function (item) { return fmtSigned(item.parsed.y) + ' ' + unit + ' since range start'; }
}
}
}
}
});
/* 4. hourly growth, last 48 h */
const buckets = {};
deltas.forEach(function (d) {
const h = Math.floor(d.x / 3600000) * 3600000;
buckets[h] = (buckets[h] || 0) + d.y;
});
const hours = Object.keys(buckets).map(Number).sort(function (a, b) { return a - b; }).slice(-48);
const hourlyLabels = hours.map(function (h) { return fmtDate(h); });
const hourlyValues = hours.map(function (h) { return Math.round(buckets[h]); });
charts[prefix + 'hourly'] = new Chart(document.getElementById(chartId(prefix, 'HourlyChart')), {
type: 'bar',
data: {
labels: hourlyLabels,
datasets: [{
label: 'Gain per hour',
data: hourlyValues,
backgroundColor: hourlyValues.map(function (v) { return v >= 0 ? 'rgba(46,204,113,0.65)' : 'rgba(255,107,107,0.65)'; }),
borderRadius: 4
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
interaction: { mode: 'nearest', axis: 'x', intersect: false },
scales: {
x: { ticks: { color: mutedColor(), maxRotation: 45, autoSkip: true, maxTicksLimit: 12 }, grid: { display: false } },
y: {
ticks: { color: mutedColor(), callback: function (v) { return fmtSigned(v); } },
grid: {
color: function (c) { return c.tick.value === 0 ? 'rgba(128,128,128,0.5)' : grid; }
}
}
},
plugins: {
legend: { labels: { usePointStyle: true, boxWidth: 8, color: mutedColor() } },
tooltip: { callbacks: { label: function (item) { return fmtSigned(item.parsed.y) + ' ' + unit + ' in this hour'; } } }
}
}
});
/* 5. growth by hour of day (whole history) */
const hodSum = new Array(24).fill(0);
const hodCount = new Array(24).fill(0);
deltas.forEach(function (d) {
const h = new Date(d.x).getHours();
hodSum[h] += d.y;
hodCount[h] += 1;
});
const hodValues = hodSum.map(function (s, i) {
return hodCount[i] ? Math.round(s / hodCount[i] * 10) / 10 : 0;
});
charts[prefix + 'hod'] = new Chart(document.getElementById(chartId(prefix, 'HodChart')), {
type: 'bar',
data: {
labels: hodValues.map(function (_, i) { return (i < 10 ? '0' : '') + i + ':00'; }),
datasets: [{
label: 'Avg gain per 5 min',
data: hodValues,
backgroundColor: hodValues.map(function (v) { return v >= 0 ? accentBar : 'rgba(255,107,107,0.65)'; }),
borderRadius: 4
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
scales: {
x: { ticks: { color: mutedColor(), maxRotation: 0, autoSkip: true }, grid: { display: false } },
y: {
ticks: { color: mutedColor(), callback: function (v) { return fmtSigned(v); } },
grid: {
color: function (c) { return c.tick.value === 0 ? 'rgba(128,128,128,0.5)' : grid; }
}
}
},
plugins: {
legend: { labels: { usePointStyle: true, boxWidth: 8, color: mutedColor() } },
tooltip: { callbacks: { label: function (item) { return fmtSigned(item.parsed.y) + ' ' + unit + ' per 5 min (avg of ' + hodCount[item.dataIndex] + ' measurements)'; } } }
}
}
});
}
/* ---------- records table ---------- */
function renderRecords(deltas, id, unit) {
const el = document.getElementById(id);
if (deltas.length < 2) {
el.innerHTML = '<div class="chart-sub">Not enough data yet.</div>';
return;
}
const sorted = deltas.slice().sort(function (a, b) { return b.diff - a.diff; });
const gains = sorted.slice(0, 5).filter(function (d) { return d.diff > 0; });
const losses = sorted.slice().reverse().slice(0, 5).filter(function (d) { return d.diff < 0; }).reverse();
function rows(items, cls) {
return items.map(function (d) {
return '<tr><td>' + fmtDate(d.x) + '</td><td class="num ' + cls + '">' + fmtSigned(d.diff) + '</td><td class="num">' + fmtDur(d.dt) + '</td></tr>';
}).join('');
}
el.innerHTML =
'<table class="records">' +
'<tr><th>Time</th><th style="text-align:right">Change (' + unit + ')</th><th style="text-align:right">Span</th></tr>' +
rows(gains, 'pos') +
rows(losses, 'neg') +
'</table>';
}
/* ---------- heatmap ---------- */
function renderHeatmap(points, id, unit) {
const el = document.getElementById(id);
if (!points.length) { el.innerHTML = ''; return; }
function localDayKey(d) {
return d.getFullYear() + '-' +
('0' + (d.getMonth() + 1)).slice(-2) + '-' +
('0' + d.getDate()).slice(-2);
}
const map = {};
points.forEach(function (p) {
const d = new Date(p.x);
map[localDayKey(d) + '|' + d.getHours()] = p.y;
});
const days = {};
Object.keys(map).forEach(function (k) { days[k.split('|')[0]] = true; });
const dayList = Object.keys(days).sort();
// deltas per day+hour for coloring
const cellDeltas = {};
let maxAbs = 0;
const sorted = points.slice().sort(function (a, b) { return a.x - b.x; });
const byDayHour = {};
sorted.forEach(function (p) {
const d = new Date(p.x);
const key = localDayKey(d) + '|' + d.getHours();
if (byDayHour[key] !== undefined) {
cellDeltas[key] = p.y - byDayHour[key];
maxAbs = Math.max(maxAbs, Math.abs(cellDeltas[key]));
}
byDayHour[key] = p.y;
});
let html = '<table><tr><th></th>';
for (let h = 0; h < 24; h++) html += '<th>' + h + '</th>';
html += '</tr>';
dayList.forEach(function (day) {
html += '<tr><td class="day">' + fmtDay(new Date(day + 'T12:00:00').getTime()) + '</td>';
for (let h = 0; h < 24; h++) {
const key = day + '|' + h;
const val = map[key];
let style = '';
let title = fmtDay(new Date(day + 'T12:00:00').getTime()) + ' ' + h + ':00 – no data';
if (val !== undefined && cellDeltas[key] !== undefined) {
const dv = cellDeltas[key];
const op = maxAbs ? Math.min(0.85, 0.15 + 0.7 * Math.abs(dv) / maxAbs) : 0.3;
const color = dv >= 0 ? '46,204,113' : '255,107,107';
style = ' style="background:rgba(' + color + ',' + op.toFixed(2) + ')"';
title = fmtDay(new Date(day + 'T12:00:00').getTime()) + ' ' + h + ':00 – ' + fmtSigned(dv) + ' ' + unit + ' (' + fmt(val) + ' total)';
}
html += '<td><div class="cell"' + style + ' title="' + title + '"></div></td>';
}
html += '</tr>';
});
html += '</table>';
el.innerHTML = html;
}
/* ---------- main update ---------- */
function renderAll() {
const all = parsePoints(rawValues);
const points = filterRange(all);
const now = Date.now();
const elCurrent = document.getElementById('current');
const meta = document.getElementById('current-meta');
const elImages = document.getElementById('current-images');
const imagesMeta = document.getElementById('current-images-meta');
document.getElementById('badge-count').textContent = 'Measurements: ' + all.length;
if (!all.length) {
elCurrent.textContent = '–';
meta.textContent = 'No measurements yet – the collector stores one every 30 minutes.';
elImages.textContent = '–';
imagesMeta.textContent = 'No image count yet';
setStatus('offline', 'No data');
return;
}
const last = all[all.length - 1];
const first = all[0];
const dts = [];
for (let i = 1; i < all.length; i++) dts.push(all[i].x - all[i - 1].x);
intervalMs = median(dts) || FALLBACK_INTERVAL_MS;
document.getElementById('badge-interval').textContent = 'Interval: ~' + fmtDur(intervalMs);
elCurrent.textContent = fmt(last.y);
meta.textContent = 'Last measurement: ' + fmtDateFull(last.x) +
' · next expected in ~' + fmtDur(Math.max(0, last.x + intervalMs - now));
if (last.images === null) {
elImages.textContent = '–';
imagesMeta.textContent = 'No image count yet';
} else {
elImages.textContent = fmt(last.images);
imagesMeta.textContent = (last.imagesExact
? 'Exact count' : 'Estimated · exact every 2 h') +
' · ' + fmtDateFull(last.x);
}
document.getElementById('badge-since').textContent = 'First measurement: ' + fmtDateFull(first.x);
document.getElementById('badge-updated').textContent = 'Last update: ' + fmtDate(last.x);
const age = now - last.x;
if (age <= 3 * intervalMs) {
setStatus('ok', 'Live · ' + fmtDur(age) + ' ago');
} else if (age <= 24 * 3600000) {
setStatus('stale', 'Stale · last update ' + fmtDur(age) + ' ago');
} else {
setStatus('offline', 'No data for ' + fmtDur(age));
}
/* fixed-window cards (full history) */
const totalHours = (last.x - first.x) / 3600000;
const perHour = totalHours > 0 ? (last.y - first.y) / totalHours : null;
const elPh = document.getElementById('stat-per-hour');
const elPd = document.getElementById('stat-per-day');
if (perHour === null) {
elPh.textContent = '–';
elPd.textContent = '–';
} else {
setSigned('stat-per-hour', perHour);
setSigned('stat-per-day', perHour * 24);
}
const span6h = closestBefore(all, last.x - 6 * 3600000, Math.max(3 * intervalMs, 3600000));
setDeltaCard('stat-delta-6h', 'stat-delta-6h-hint',
span6h ? last.y - span6h.y : null,
span6h ? last.x - span6h.x : 0,
span6h ? (last.y - span6h.y) * (3600000 / Math.max(1, last.x - span6h.x)) * (5 / 60) : 0);
const span24h = closestBefore(all, last.x - 24 * 3600000, Math.max(6 * intervalMs, 3 * 3600000));
setDeltaCard('stat-delta-24h', 'stat-delta-24h-hint',
span24h ? last.y - span24h.y : null,
span24h ? last.x - span24h.x : 0,
span24h ? (last.y - span24h.y) * (3600000 / Math.max(1, last.x - span24h.x)) * (5 / 60) : 0);
const span7d = closestBefore(all, last.x - 7 * 24 * 3600000, Math.max(12 * intervalMs, 6 * 3600000));
setDeltaCard('stat-delta-7d', 'stat-delta-7d-hint',
span7d ? last.y - span7d.y : null,
span7d ? last.x - span7d.x : 0,
span7d ? (last.y - span7d.y) * (3600000 / Math.max(1, last.x - span7d.x)) * (5 / 60) : 0);
/* range-dependent cards + charts */
const fpoints = points.length >= 2 ? points : all;
const dd = computeDeltas(fpoints);
const deltas = dd.deltas;
const smoothY = movingAverage(deltas.map(function (d) { return d.y; }), DELTA_SMOOTH_WINDOW);
const smoothedDeltas = deltas.map(function (d, i) {
return { x: d.x, y: smoothY[i], raw: d.y, dt: d.dt, diff: d.diff };
});
/* avg/h over last 24 h: sum of all changes within 24 h ÷ hours actually covered */
let sum24 = 0;
let windowStart = Math.max(fpoints[0].x, last.x - 24 * 3600000);
let coveredH = (last.x - windowStart) / 3600000;
deltas.forEach(function (d) {
if (d.x > last.x - 24 * 3600000) sum24 += d.diff;
});
if (coveredH >= 0.5) {
setSigned('stat-avg-hour', sum24 / coveredH);
} else {
document.getElementById('stat-avg-hour').textContent = '–';
}
/* best / worst hour: best total gain within a sliding ~1 h window */
let best = null, worst = null;
for (let i = 0; i < deltas.length; i++) {
let gain = 0, endMs = deltas[i].x;
for (let j = i; j < deltas.length; j++) {
if (deltas[j].x - deltas[i].x > 3600000) break;
gain += deltas[j].diff;
endMs = deltas[j].x;
}
if (!best || gain > best.gain) best = { gain: gain, x: endMs };
if (!worst || gain < worst.gain) worst = { gain: gain, x: endMs };
}
if (best && best.gain !== 0) {
setSigned('stat-best-hour', best.gain);
document.getElementById('stat-best-hour-hint').textContent = 'Around ' + fmtDate(best.x);
}
if (worst && worst.gain !== 0) {
setSigned('stat-worst-hour', worst.gain);
document.getElementById('stat-worst-hour-hint').textContent = 'Around ' + fmtDate(worst.x);
}
/* peak / lowest (whole history) */
let peak = all[0], low = all[0];
all.forEach(function (p) {
if (p.y > peak.y) peak = p;
if (p.y < low.y) low = p;
});
document.getElementById('stat-peak').textContent = fmt(peak.y);
document.getElementById('stat-peak-hint').textContent = 'On ' + fmtDateFull(peak.x);
document.getElementById('stat-min').textContent = fmt(low.y);
document.getElementById('stat-min-hint').textContent = 'On ' + fmtDateFull(low.x);
/* forecast */
const regWindow = [];
for (let i = fpoints.length - 1; i >= 0; i--) {
regWindow.unshift(fpoints[i]);
if (fpoints.length - i >= 96) break;
if (fpoints[fpoints.length - 1].x - fpoints[i].x >= 2 * 3600000) break;
}
const reg = regress(regWindow);
if (reg) {
const elF = document.getElementById('stat-forecast');
const fc = fpoints[fpoints.length - 1].y + reg.slope;
elF.textContent = fmt(fc);
elF.classList.remove('pos', 'neg');
elF.classList.add(fc >= fpoints[fpoints.length - 1].y ? 'pos' : 'neg');
document.getElementById('stat-forecast-hint').textContent =
'Trend ' + fmtSigned(reg.slope) + '/h · R² = ' + reg.r2.toFixed(3);
document.getElementById('footer').textContent =
'Forecast: linear least-squares regression over the last ' + regWindow.length +
' measurements (' + fmtDur(fpoints[fpoints.length - 1].x - regWindow[0].x) +
'), extrapolated +24 h · R² = ' + reg.r2.toFixed(3) +
' · Range: ' + (range === 'all' ? 'all' : range) + ' · All statistics are computed in the browser.';
} else {
document.getElementById('stat-forecast').textContent = '–';
document.getElementById('stat-forecast-hint').textContent = 'Needs at least 2 measurements';
document.getElementById('footer').textContent = 'Forecast available once there are at least 2 measurements.';
}
/* images: same graphs, but only exact measurements (every 2 h) */
const imageAll = toImagePoints(all);
const imageFpoints = filterRange(imageAll, last.x);
const imageDeltas = computeDeltas(imageFpoints).deltas;
const imageSmoothY = movingAverage(imageDeltas.map(function (d) { return d.y; }), DELTA_SMOOTH_WINDOW);
const imageSmoothedDeltas = imageDeltas.map(function (d, i) {
return { x: d.x, y: imageSmoothY[i], raw: d.y, dt: d.dt, diff: d.diff };
});
const imagesEmpty = document.getElementById('images-empty');
const hasImageValues = all.some(function (p) { return p.images !== null; });
if (!imageAll.length) {
imagesEmpty.textContent = hasImageValues
? 'Only estimated image counts so far – the graphs show exact counts only (every 2 h).'
: 'No image counts yet – they appear after the next collector run with the image migration in place.';
imagesEmpty.style.display = '';
} else if (!imageFpoints.length) {
imagesEmpty.textContent = 'No exact image counts in the selected range (counted every 2 h).';
imagesEmpty.style.display = '';
} else {
imagesEmpty.style.display = 'none';
}
destroyCharts();
renderCharts({
prefix: '',
points: fpoints,
deltas: deltas,
smoothedDeltas: smoothedDeltas,
unit: 'Pages',
accent: ACCENT,
accentBar: 'rgba(79,140,255,0.65)',
gradTop: 'rgba(79,140,255,0.28)',
gradBottom: 'rgba(79,140,255,0.02)'
});
renderCharts({
prefix: 'images',
points: imageFpoints,
deltas: imageDeltas,
smoothedDeltas: imageSmoothedDeltas,
unit: 'Images',
accent: ACCENT2,
accentBar: 'rgba(180,122,255,0.65)',
gradTop: 'rgba(180,122,255,0.28)',
gradBottom: 'rgba(180,122,255,0.02)'
});
renderRecords(deltas, 'records', 'Pages');
renderRecords(imageDeltas, 'records-images', 'Images');
renderHeatmap(all, 'heatmap', 'Pages');
renderHeatmap(imageAll, 'heatmap-images', 'Images');
}
async function update() {
refreshLeft = REFRESH_S;
try {
const response = await fetch('/api/data');
rawValues = (await response.json()).values || [];
} catch (e) {
setStatus('offline', 'Fetch error – retrying…');
return;
}
renderAll();
}
setInterval(function () {
refreshLeft -= 1;
if (refreshLeft <= 0) {
document.getElementById('badge-refresh').textContent = 'Refreshing…';
} else {
document.getElementById('badge-refresh').textContent = 'Next refresh: ' + refreshLeft + ' s';
}
}, 1000);
update();
setInterval(update, REFRESH_S * 1000);
</script>
</body>
</html>
"""
def load_data():
if not DB_DATA.exists():
return []
try:
with DB_DATA.open("r", encoding="utf-8") as f:
return json.load(f)
except Exception:
return []
def save_data(data):
tmp = DB_DATA.with_suffix(".tmp")
with tmp.open("w", encoding="utf-8") as f:
json.dump(data, f, indent=2)
tmp.replace(DB_DATA)
# Exakt: ein Scan, der COUNT(*) und die Bilder in einem Durchgang
# ermittelt. Wird nur alle EXACT_IMAGE_INTERVAL Sekunden ausgeführt.
EXACT_IMAGE_SQL = (
"SELECT COUNT(*), "
"COALESCE(SUM(cardinality(pictures)), 0)::bigint "
"FROM pages;"
)
# Schätzung: COUNT(*) wie bisher, Bilder aus 1 % der Tabellenblöcke
# (TABLESAMPLE SYSTEM), hochgerechnet. Die zweite Spalte ist NULL, wenn
# kein Block ausgewählt wurde (sehr kleine Tabelle).
ESTIMATE_IMAGE_SQL = (
"SELECT (SELECT COUNT(*) FROM pages), "
"(SELECT (SUM(cardinality(pictures)) * {scale})::bigint "
"FROM pages TABLESAMPLE SYSTEM ({percent}));"
).format(
scale=100 // IMAGE_SAMPLE_PERCENT,
percent=IMAGE_SAMPLE_PERCENT,
)
def _psql_scalar(sql):
"""Eine einzelne psql-Zeile ("count|images") abholen."""
command = [
"sudo",
"-u",
"postgres",
"psql",
"-p",
"25432",
"-d",
"jonassearch",
"-t",
"-A",
"-F",
"|",
"-c",
sql,
]
result = subprocess.run(
command,
capture_output=True,
text=True,
check=True
)
return result.stdout.strip()
def _parse_stats_row(output):
""""count|images" -> (count, images|None)."""
count_raw, _sep, images_raw = (output or "").strip().partition("|")
count = int(count_raw.strip())
images = int(images_raw.strip()) if images_raw.strip() else None
return count, images
def images_exact_due(data, now=None):
"""
True, wenn der letzte exakte Bild-Count fehlt oder mindestens
EXACT_IMAGE_INTERVAL alt ist. Einträge aus der Zeit vor dieser
Funktion (ohne images_exact) zählen nicht als exakt.
"""
now = now or datetime.now()
for entry in reversed(data or []):
if entry.get("images_exact") is not True:
continue
try:
timestamp = datetime.fromisoformat(str(entry.get("timestamp", "")))
except (TypeError, ValueError):
continue
return now - timestamp >= timedelta(seconds=EXACT_IMAGE_INTERVAL)
return True
def get_stats(exact_images):
"""
Seitenzahl + Bildanzahl holen.
exact_images=True: ein Scan, exakter Bild-Count.
exact_images=False: Seitenzahl wie bisher, Bilder aus einer
1-%-Stichprobe hochgerechnet. Liefert die Stichprobe nichts (kleine
Tabelle), wird exakt gezählt - das ist dort billig.
Rückgabe: (pages, images, images_exact). images kann None sein,
wenn eine große Tabelle wider Erwarten keinen Block gesampelt hat.
"""
try:
if exact_images:
count, images = _parse_stats_row(_psql_scalar(EXACT_IMAGE_SQL))
return count, images, True
count, images = _parse_stats_row(_psql_scalar(ESTIMATE_IMAGE_SQL))
except subprocess.CalledProcessError as exc:
# Z. B. noch keine Migration (pages.pictures fehlt): die
# Seitenzahl soll trotzdem weiter erfasst werden.
detail = ((exc.stderr or "").strip().splitlines() or [str(exc)])[-1]
print(f"Bild-Statistik nicht verfügbar ({detail}); zähle nur Seiten.")
return int(_psql_scalar("SELECT COUNT(*) FROM pages;").strip()), None, False
if images is None:
if count < SAMPLE_EXACT_FALLBACK_ROWS:
return get_stats(True)
return count, None, False
return count, images, False
def collect():
while True:
try:
data = load_data()
exact = images_exact_due(data)
count, images, images_exact = get_stats(exact)
if images is None:
# Stichprobe ohne Treffer (praktisch nur bei sehr großen
# Tabellen): letzten bekannten Wert fortschreiben.
for previous in reversed(data):
if previous.get("images") is not None:
images = previous["images"]
break
entry = {
"timestamp": datetime.now().isoformat(),
"count": count,
"images": images,
"images_exact": images_exact,
}
data.append(entry)
save_data(data)
print(
f"[{entry['timestamp']}] "
f"Pages: {count} · Images: {images} "
f"({'exakt' if images_exact else 'geschätzt'})"
)
except Exception as e:
print(f"Fehler beim Abrufen der Statistik: {e}")
time.sleep(INTERVAL)
@app.route("/")
def index():
return render_template_string(HTML)
@app.route("/api/data")
def api_data():
return jsonify({
"values": load_data()
})
@app.route("/api/export")
def api_export():
lines = ["timestamp;count"]
for entry in load_data():
lines.append(f"{entry.get('timestamp', '')};{entry.get('count', '')}")
return Response("\n".join(lines), mimetype="text/csv")
if __name__ == "__main__":
# Collector im Hintergrund starten
thread = threading.Thread(target=collect, daemon=True)
thread.start()
# Webserver
app.run(
host="0.0.0.0",
port=8081,
threaded=True,
load_dotenv=False # sonst liest Flask beim Start .env im Projektordner -> PermissionError
)