Files
search/web/app.py
T
2026-10-04 01:13:13 +02:00

1315 lines
44 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
import subprocess
import json
import time
import threading
from datetime import datetime
from pathlib import Path
from flask import Flask, jsonify, render_template_string, Response
DB_DATA = Path(__file__).parent / "page_counts.json"
INTERVAL = 30 * 60 # 30 Minuten
app = Flask(__name__)
# r""" raw string: preserves \n etc. for the embedded JavaScript
# (a normal string would eat the backslashes and break the JS)
HTML = r"""
<!DOCTYPE html>
<html lang="de">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>JonasSearch – Pages</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<style>
:root {
--bg: #0f1115;
--card: #1b1e24;
--border: #2c313a;
--text: #e8eaed;
--muted: #9aa0a6;
--accent: #4f8cff;
--accent2: #b47aff;
--up: #2ecc71;
--down: #ff6b6b;
--warn: #f5a623;
}
html[data-theme="light"] {
--bg: #f4f6fa;
--card: #ffffff;
--border: #d8dde5;
--text: #1a1d23;
--muted: #6b7280;
--accent: #2f6fe4;
--accent2: #8a4fe0;
}
* { box-sizing: border-box; }
body {
margin: 0;
background: var(--bg);
color: var(--text);
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Arial, sans-serif;
}
.container { max-width: 1200px; margin: auto; padding: 30px 24px 60px; }
header {
display: flex;
flex-wrap: wrap;
align-items: baseline;
gap: 8px 16px;
position: sticky;
top: 0;
z-index: 10;
background: var(--bg);
padding: 10px 0 14px;
border-bottom: 1px solid var(--border);
}
h1 { margin: 0; font-size: 1.9rem; letter-spacing: 0.3px; }
.subtitle { color: var(--muted); }
.header-right { margin-left: auto; display: flex; gap: 8px; align-items: center; }
.btn {
background: var(--card);
border: 1px solid var(--border);
color: var(--text);
font-size: 0.8rem;
padding: 6px 12px;
border-radius: 999px;
cursor: pointer;
text-decoration: none;
}
.btn:hover { border-color: var(--accent); }
.current {
font-size: 3.2rem;
font-weight: 700;
margin: 18px 0 4px;
line-height: 1;
}
.current .unit {
font-size: 1.1rem;
font-weight: 400;
color: var(--muted);
margin-left: 8px;
}
.current-meta { color: var(--muted); margin-bottom: 24px; }
.badges { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 14px; }
.badge {
display: inline-flex;
align-items: center;
gap: 6px;
background: var(--card);
border: 1px solid var(--border);
color: var(--muted);
font-size: 0.8rem;
padding: 5px 10px;
border-radius: 999px;
}
.dot { width: 8px; height: 8px; border-radius: 50%; background: var(--warn); display: inline-block; }
.dot.ok { background: var(--up); }
.dot.stale { background: var(--warn); }
.dot.offline { background: var(--down); }
.dot.live { animation: pulse 2s ease-in-out infinite; }
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.35; }
}
.ranges { display: flex; gap: 8px; margin-bottom: 18px; }
.range-btn {
background: var(--card);
border: 1px solid var(--border);
color: var(--muted);
font-size: 0.82rem;
padding: 6px 14px;
border-radius: 999px;
cursor: pointer;
}
.range-btn.active {
color: #fff;
background: var(--accent);
border-color: var(--accent);
}
.grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(165px, 1fr));
gap: 14px;
margin-bottom: 22px;
}
.card {
background: var(--card);
border: 1px solid var(--border);
border-radius: 15px;
padding: 16px 18px;
min-width: 0;
}
.card .label {
color: var(--muted);
font-size: 0.78rem;
text-transform: uppercase;
letter-spacing: 0.8px;
margin-bottom: 8px;
}
.card .value { font-size: 1.45rem; font-weight: 600; }
.card .hint { color: var(--muted); font-size: 0.78rem; margin-top: 6px; }
.pos { color: var(--up); }
.neg { color: var(--down); }
.chart-container {
background: var(--card);
border: 1px solid var(--border);
padding: 20px;
border-radius: 15px;
margin-bottom: 22px;
}
.chart-row { display: grid; grid-template-columns: repeat(auto-fit, minmax(420px, 1fr)); gap: 22px; }
.chart-row .chart-container { margin-bottom: 0; }
@media (max-width: 950px) {
.chart-row { grid-template-columns: 1fr; }
.chart-row .chart-container { margin-bottom: 22px; }
}
.chart-box { position: relative; height: 340px; }
.chart-box.small { height: 260px; }
.chart-title { font-weight: 600; margin-bottom: 2px; }
.chart-sub { color: var(--muted); font-size: 0.82rem; margin-bottom: 14px; }
.heatmap { overflow-x: auto; }
.heatmap table { border-collapse: collapse; width: 100%; min-width: 720px; }
.heatmap td, .heatmap th {
font-size: 0.68rem;
color: var(--muted);
padding: 2px;
text-align: center;
}
.heatmap td.day { text-align: right; padding-right: 8px; white-space: nowrap; }
.heatmap .cell {
width: 100%;
min-width: 22px;
height: 18px;
border-radius: 4px;
background: rgba(128,128,128,0.12);
}
.footer { color: var(--muted); font-size: 0.8rem; margin-top: 8px; }
table.records { width: 100%; border-collapse: collapse; font-size: 0.85rem; }
table.records th, table.records td {
text-align: left;
padding: 7px 8px;
border-bottom: 1px solid var(--border);
}
table.records th { color: var(--muted); font-weight: 500; font-size: 0.75rem; text-transform: uppercase; letter-spacing: 0.6px; }
table.records td.num { text-align: right; font-variant-numeric: tabular-nums; }
.section-title { font-weight: 600; margin: 30px 0 12px; font-size: 1.05rem; }
@media (max-width: 600px) {
.current { font-size: 2.4rem; }
.container { padding: 20px 14px 40px; }
.chart-container { padding: 14px; }
.chart-box { height: 260px; }
.chart-box.small { height: 200px; }
}
</style>
</head>
<body>
<div class="container">
<header>
<h1>JonasSearch</h1>
<div class="subtitle">Indexed Pages – Live Statistics</div>
<div class="header-right">
<a class="btn" id="btn-csv" href="#">Export CSV</a>
<a class="btn" id="btn-json" href="#">Export JSON</a>
<button class="btn" id="btn-theme" title="Toggle light/dark">☀️ Light</button>
</div>
</header>
<div class="current">
<span id="current">–</span><span class="unit">Pages</span>
</div>
<div class="current-meta" id="current-meta">Loading…</div>
<div class="badges">
<span class="badge"><span class="dot" id="status-dot"></span><span id="status-text">Loading…</span></span>
<span class="badge" id="badge-interval">Interval: –</span>
<span class="badge" id="badge-since">First measurement: –</span>
<span class="badge" id="badge-count">Measurements: –</span>
<span class="badge" id="badge-updated">Last update: –</span>
<span class="badge" id="badge-refresh">Next refresh: –</span>
</div>
<div class="ranges">
<button class="range-btn active" data-range="all">All</button>
<button class="range-btn" data-range="7d">Last 7 days</button>
<button class="range-btn" data-range="24h">Last 24 h</button>
</div>
<div class="grid">
<div class="card">
<div class="label">Growth / hour</div>
<div class="value" id="stat-per-hour">–</div>
<div class="hint">Average over full history</div>
</div>
<div class="card">
<div class="label">Growth / day</div>
<div class="value" id="stat-per-day">–</div>
<div class="hint">Average over full history</div>
</div>
<div class="card">
<div class="label">Growth 24 h</div>
<div class="value" id="stat-delta-24h">–</div>
<div class="hint" id="stat-delta-24h-hint">vs. measurement ~24 h ago</div>
</div>
<div class="card">
<div class="label">Growth 7 days</div>
<div class="value" id="stat-delta-7d">–</div>
<div class="hint" id="stat-delta-7d-hint">vs. measurement ~7 d ago</div>
</div>
<div class="card">
<div class="label">Avg / h (24 h)</div>
<div class="value" id="stat-avg-hour">–</div>
<div class="hint">Mean of last 24 h changes</div>
</div>
<div class="card">
<div class="label">Best hour</div>
<div class="value" id="stat-best-hour">–</div>
<div class="hint" id="stat-best-hour-hint">Largest gain within one hour</div>
</div>
<div class="card">
<div class="label">Worst hour</div>
<div class="value" id="stat-worst-hour">–</div>
<div class="hint" id="stat-worst-hour-hint">Largest loss within one hour</div>
</div>
<div class="card">
<div class="label">Peak</div>
<div class="value" id="stat-peak">–</div>
<div class="hint" id="stat-peak-hint">Highest measurement ever</div>
</div>
<div class="card">
<div class="label">Lowest</div>
<div class="value" id="stat-min">–</div>
<div class="hint" id="stat-min-hint">Lowest measurement ever</div>
</div>
<div class="card">
<div class="label">Forecast next hour</div>
<div class="value" id="stat-forecast">–</div>
<div class="hint" id="stat-forecast-hint">Linear trend extrapolation</div>
</div>
</div>
<div class="chart-container">
<div class="chart-title">Indexed pages over time</div>
<div class="chart-sub">Real measurement timestamps · dashed line = linear-regression trend &amp; 24 h forecast</div>
<div class="chart-box"><canvas id="mainChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Change per 5 minutes</div>
<div class="chart-sub">Normalized to a 5-minute interval · grey = raw · colored = smoothed (moving average over 5 measurements)</div>
<div class="chart-box small"><canvas id="deltaChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Cumulative net growth</div>
<div class="chart-sub">Total pages gained/lost since the start of the selected range · linear time axis</div>
<div class="chart-box small"><canvas id="cumChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Growth per hour (last 48 h)</div>
<div class="chart-sub">Sum of changes per hour bucket · green = growth, red = loss</div>
<div class="chart-box small"><canvas id="hourlyChart"></canvas></div>
</div>
<div class="chart-row">
<div class="chart-container">
<div class="chart-title">Growth by hour of day</div>
<div class="chart-sub">Average gain per 5 min, grouped by hour of day (whole history) · shows when the crawler is most active</div>
<div class="chart-box small"><canvas id="hodChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Records – biggest changes</div>
<div class="chart-sub">Largest gains and losses between two consecutive measurements (selected range)</div>
<div id="records"></div>
</div>
</div>
<div class="section-title">Activity heatmap – gain per hour of day</div>
<div class="chart-container heatmap">
<div class="chart-sub">Rows = days · columns = hours (0–23) · green = growth, red = loss, grey = no data</div>
<div id="heatmap"></div>
</div>
<div class="footer" id="footer"></div>
</div>
<script>
'use strict';
if (window.Chart && Chart.registerables) {
Chart.register.apply(Chart, Chart.registerables);
}
const ACCENT = '#4f8cff';
const ACCENT2 = '#b47aff';
const UP = '#2ecc71';
const DOWN = '#ff6b6b';
const FALLBACK_INTERVAL_MS = 15 * 60 * 1000;
const DELTA_SMOOTH_WINDOW = 5;
const REFRESH_S = 30;
let rawValues = [];
let intervalMs = FALLBACK_INTERVAL_MS;
let range = 'all';
let charts = {};
let refreshLeft = REFRESH_S;
function cssVar(name) {
return getComputedStyle(document.documentElement).getPropertyValue(name).trim();
}
function borderColor() { return cssVar('--border') || '#2c313a'; }
function mutedColor() { return cssVar('--muted') || '#9aa0a6'; }
function fmt(n) { return Math.round(n).toLocaleString('de-DE'); }
function fmtSigned(n) {
const s = Math.round(n).toLocaleString('de-DE');
return n > 0 ? '+' + s : s;
}
function fmtDate(ms) {
return new Date(ms).toLocaleString('de-DE', {
day: '2-digit', month: '2-digit', hour: '2-digit', minute: '2-digit'
});
}
function fmtDateFull(ms) {
return new Date(ms).toLocaleString('de-DE');
}
function fmtDay(ms) {
return new Date(ms).toLocaleDateString('de-DE', { day: '2-digit', month: '2-digit' });
}
function fmtDur(ms) {
const min = Math.round(ms / 60000);
if (min < 1) return '<1 min';
if (min < 60) return min + ' min';
const h = Math.floor(min / 60);
const m = min % 60;
if (h < 24) return m ? (h + ' h ' + m + ' min') : (h + ' h');
const d = Math.floor(h / 24);
return d + ' d ' + (h % 24) + ' h';
}
function regress(points) {
const n = points.length;
if (n < 2) return null;
const t0 = points[0].x;
let sx = 0, sy = 0, sxy = 0, sxx = 0;
for (const p of points) {
const x = (p.x - t0) / 3600000;
sx += x; sy += p.y; sxy += x * p.y; sxx += x * x;
}
const den = n * sxx - sx * sx;
if (den === 0) return null;
const b = (n * sxy - sx * sy) / den;
const a = (sy - b * sx) / n;
let ssTot = 0, ssRes = 0;
const mean = sy / n;
for (const p of points) {
const x = (p.x - t0) / 3600000;
const pred = a + b * x;
ssTot += Math.pow(p.y - mean, 2);
ssRes += Math.pow(p.y - pred, 2);
}
const r2 = ssTot === 0 ? 1 : Math.max(0, Math.min(1, 1 - ssRes / ssTot));
return {
slope: b,
r2: r2,
at: function (ms) { return a + b * ((ms - t0) / 3600000); }
};
}
function movingAverage(values, window) {
const half = Math.floor(window / 2);
return values.map(function (_, i) {
let sum = 0, count = 0;
for (let j = i - half; j <= i + half; j++) {
if (j >= 0 && j < values.length) { sum += values[j]; count++; }
}
return sum / count;
});
}
function median(arr) {
if (!arr.length) return null;
const s = arr.slice().sort(function (a, b) { return a - b; });
const mid = Math.floor(s.length / 2);
return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
}
function closestBefore(points, targetMs, maxLagMs) {
let best = null;
for (const p of points) {
if (p.x <= targetMs && (!best || p.x > best.x)) best = p;
}
if (best && targetMs - best.x <= maxLagMs) return best;
return null;
}
function setDeltaCard(id, idHint, delta, spanMs, per5) {
const el = document.getElementById(id);
const hint = document.getElementById(idHint);
if (delta === null) {
el.textContent = '–';
el.classList.remove('pos', 'neg');
hint.textContent = 'Not enough data yet';
return;
}
el.textContent = fmtSigned(delta);
el.classList.remove('pos', 'neg');
el.classList.add(delta >= 0 ? 'pos' : 'neg');
hint.textContent = fmtSigned(spanMs > 0 ? delta * (3600000 / spanMs) : 0) + ' Pages/h · ' + fmtSigned(per5) + ' per 5 min over ' + fmtDur(spanMs);
}
function setSigned(id, value) {
const el = document.getElementById(id);
el.textContent = fmtSigned(value);
el.classList.remove('pos', 'neg');
el.classList.add(value >= 0 ? 'pos' : 'neg');
}
/* ---------- data helpers ---------- */
function parsePoints(values) {
return (values || [])
.map(function (v) { return { x: new Date(v.timestamp).getTime(), y: v.count }; })
.filter(function (p) { return !isNaN(p.x); })
.sort(function (a, b) { return a.x - b.x; });
}
function filterRange(points) {
if (range === 'all' || !points.length) return points;
const last = points[points.length - 1].x;
const span = range === '24h' ? 24 * 3600000 : 7 * 24 * 3600000;
return points.filter(function (p) { return p.x >= last - span; });
}
function computeDeltas(points) {
const dts = [];
for (let i = 1; i < points.length; i++) dts.push(points[i].x - points[i - 1].x);
const iv = median(dts) || FALLBACK_INTERVAL_MS;
const deltas = [];
for (let i = 1; i < points.length; i++) {
const dt = points[i].x - points[i - 1].x;
const diff = points[i].y - points[i - 1].y;
if (dt <= 0 || dt > 4 * iv) continue;
const per5 = dt < 60000 ? diff : diff * (5 * 60000 / dt);
deltas.push({ x: points[i].x, y: per5, dt: dt, diff: diff });
}
return { deltas: deltas, interval: iv };
}
/* ---------- status ---------- */
function setStatus(kind, text) {
const dot = document.getElementById('status-dot');
dot.className = 'dot ' + kind + (kind === 'ok' ? ' live' : '');
document.getElementById('status-text').textContent = text;
}
/* ---------- theme ---------- */
function applyTheme(theme) {
document.documentElement.dataset.theme = theme;
document.getElementById('btn-theme').textContent = theme === 'light' ? '🌙 Dark' : '☀️ Light';
try { localStorage.setItem('js-theme', theme); } catch (e) {}
if (rawValues.length) renderAll(); // re-render charts with new palette
}
document.getElementById('btn-theme').addEventListener('click', function () {
applyTheme(document.documentElement.dataset.theme === 'light' ? 'dark' : 'light');
});
(function () {
let saved = null;
try { saved = localStorage.getItem('js-theme'); } catch (e) {}
if (saved) document.documentElement.dataset.theme = saved;
document.getElementById('btn-theme').textContent = saved === 'light' ? '🌙 Dark' : '☀️ Light';
})();
/* ---------- range buttons ---------- */
document.querySelectorAll('.range-btn').forEach(function (btn) {
btn.addEventListener('click', function () {
document.querySelectorAll('.range-btn').forEach(function (b) { b.classList.remove('active'); });
btn.classList.add('active');
range = btn.dataset.range;
update();
});
});
/* ---------- export ---------- */
function download(name, text, mime) {
const blob = new Blob([text], { type: mime });
const a = document.createElement('a');
a.href = URL.createObjectURL(blob);
a.download = name;
a.click();
URL.revokeObjectURL(a.href);
}
document.getElementById('btn-csv').addEventListener('click', function (e) {
e.preventDefault();
const lines = ['timestamp;count'];
rawValues.forEach(function (v) { lines.push(v.timestamp + ';' + v.count); });
download('jonassearch_pages.csv', lines.join('\n'), 'text/csv');
});
document.getElementById('btn-json').addEventListener('click', function (e) {
e.preventDefault();
download('jonassearch_pages.json', JSON.stringify({ values: rawValues }, null, 2), 'application/json');
});
/* ---------- chart helpers ---------- */
function destroyCharts() {
Object.keys(charts).forEach(function (k) {
if (charts[k]) charts[k].destroy();
});
charts = {};
}
function baseScales(yFmt) {
return {
x: {
type: 'linear',
ticks: {
maxTicksLimit: 8,
maxRotation: 0,
autoSkip: true,
color: mutedColor(),
callback: function (v) { return fmtDate(v); }
},
grid: { color: 'rgba(128,128,128,0.15)' }
},
y: {
grace: '5%',
ticks: { color: mutedColor(), callback: yFmt },
grid: { color: 'rgba(128,128,128,0.15)' }
}
};
}
function tooltipTitle(items) { return fmtDateFull(items[0].parsed.x); }
function gradientFill(colorTop, colorBottom) {
return function (context) {
const chart = context.chart;
const area = chart.chartArea;
if (!area) return colorBottom;
const g = chart.ctx.createLinearGradient(0, area.top, 0, area.bottom);
g.addColorStop(0, colorTop);
g.addColorStop(1, colorBottom);
return g;
};
}
/* ---------- charts ---------- */
function renderCharts(points, deltas, smoothedDeltas) {
destroyCharts();
const grid = 'rgba(128,128,128,0.15)';
/* 1. main chart + trend */
const regWindow = [];
for (let i = points.length - 1; i >= 0; i--) {
regWindow.unshift(points[i]);
if (points.length - i >= 96) break;
if (points[points.length - 1].x - points[i].x >= 2 * 3600000) break;
}
const reg = regress(regWindow);
let trendPoints = [];
if (reg) {
const horizon = points[points.length - 1].x + 24 * 3600000;
trendPoints = [
{ x: regWindow[0].x, y: reg.at(regWindow[0].x) },
{ x: horizon, y: reg.at(horizon) }
];
}
charts.main = new Chart(document.getElementById('mainChart'), {
type: 'line',
data: {
datasets: [
{
label: 'Pages',
data: points,
parsing: false,
borderColor: ACCENT,
borderWidth: 2,
pointRadius: 2.5,
pointHoverRadius: 5,
tension: 0.25,
fill: true,
backgroundColor: gradientFill('rgba(79,140,255,0.28)', 'rgba(79,140,255,0.02)'),
spanGaps: false
},
{
label: 'Trend / Forecast',
data: trendPoints,
parsing: false,
borderColor: ACCENT2,
borderDash: [6, 6],
borderWidth: 2,
pointRadius: 0,
fill: false,
spanGaps: true
}
]
},
options: {
responsive: true,
maintainAspectRatio: false,
normalized: true,
interaction: { mode: 'nearest', axis: 'x', intersect: false },
scales: baseScales(function (v) { return fmt(v); }),
plugins: {
legend: { labels: { usePointStyle: true, boxWidth: 8, color: mutedColor() } },
tooltip: {
callbacks: {
title: tooltipTitle,
label: function (item) {
return item.datasetIndex === 0
? (fmt(item.parsed.y) + ' Pages')
: ('Trend: ' + fmt(item.parsed.y) + ' Pages');
}
}
}
}
}
});
/* 2. delta chart */
charts.delta = new Chart(document.getElementById('deltaChart'), {
type: 'line',
data: {
datasets: [
{
label: 'Raw',
data: deltas,
parsing: false,
borderColor: 'rgba(128,128,128,0.30)',
borderWidth: 1,
pointRadius: 0,
tension: 0,
fill: false
},
{
label: 'Change per 5 min',
data: smoothedDeltas,
parsing: false,
borderColor: ACCENT,
borderWidth: 2.5,
pointRadius: 0,
pointHoverRadius: 4,
tension: 0.35,
fill: false,
segment: {
borderColor: function (c) { return c.p1.parsed.y >= 0 ? UP : DOWN; }
}
}
]
},
options: {
responsive: true,
maintainAspectRatio: false,
normalized: true,
interaction: { mode: 'nearest', axis: 'x', intersect: false },
scales: {
x: baseScales(function () { return ''; }).x,
y: {
ticks: { color: mutedColor(), callback: function (v) { return fmtSigned(v); } },
grid: {
color: function (c) { return c.tick.value === 0 ? 'rgba(128,128,128,0.5)' : grid; }
}
}
},
plugins: {
legend: {
display: true,
labels: {
usePointStyle: true,
boxWidth: 8,
color: mutedColor(),
filter: function (item) { return item.text !== 'Raw'; }
}
},
tooltip: {
filter: function (item) { return item.datasetIndex === 1; },
callbacks: {
title: tooltipTitle,
label: function (item) {
const raw = item.raw;
return [
'Smoothed: ' + fmtSigned(raw.y) + ' Pages per 5 min',
'Raw: ' + fmtSigned(raw.raw) + ' per 5 min (change ' + fmtSigned(raw.diff) + ' in ' + fmtDur(raw.dt) + ')'
];
}
}
}
}
}
});
/* 3. cumulative net growth */
const cum = points.map(function (p, i) {
return { x: p.x, y: p.y - points[0].y };
});
charts.cum = new Chart(document.getElementById('cumChart'), {
type: 'line',
data: {
datasets: [{
label: 'Net growth since range start',
data: cum,
parsing: false,
borderColor: UP,
borderWidth: 2,
pointRadius: 0,
pointHoverRadius: 4,
tension: 0.25,
fill: true,
backgroundColor: gradientFill('rgba(46,204,113,0.22)', 'rgba(46,204,113,0.02)')
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
normalized: true,
interaction: { mode: 'nearest', axis: 'x', intersect: false },
scales: {
x: baseScales(function () { return ''; }).x,
y: {
ticks: { color: mutedColor(), callback: function (v) { return fmtSigned(v); } },
grid: {
color: function (c) { return c.tick.value === 0 ? 'rgba(128,128,128,0.5)' : grid; }
}
}
},
plugins: {
legend: { labels: { usePointStyle: true, boxWidth: 8, color: mutedColor() } },
tooltip: {
callbacks: {
title: tooltipTitle,
label: function (item) { return fmtSigned(item.parsed.y) + ' Pages since range start'; }
}
}
}
}
});
/* 4. hourly growth, last 48 h */
const buckets = {};
deltas.forEach(function (d) {
const h = Math.floor(d.x / 3600000) * 3600000;
buckets[h] = (buckets[h] || 0) + d.y;
});
const hours = Object.keys(buckets).map(Number).sort(function (a, b) { return a - b; }).slice(-48);
const hourlyLabels = hours.map(function (h) { return fmtDate(h); });
const hourlyValues = hours.map(function (h) { return Math.round(buckets[h]); });
charts.hourly = new Chart(document.getElementById('hourlyChart'), {
type: 'bar',
data: {
labels: hourlyLabels,
datasets: [{
label: 'Gain per hour',
data: hourlyValues,
backgroundColor: hourlyValues.map(function (v) { return v >= 0 ? 'rgba(46,204,113,0.65)' : 'rgba(255,107,107,0.65)'; }),
borderRadius: 4
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
interaction: { mode: 'nearest', axis: 'x', intersect: false },
scales: {
x: { ticks: { color: mutedColor(), maxRotation: 45, autoSkip: true, maxTicksLimit: 12 }, grid: { display: false } },
y: {
ticks: { color: mutedColor(), callback: function (v) { return fmtSigned(v); } },
grid: {
color: function (c) { return c.tick.value === 0 ? 'rgba(128,128,128,0.5)' : grid; }
}
}
},
plugins: {
legend: { labels: { usePointStyle: true, boxWidth: 8, color: mutedColor() } },
tooltip: { callbacks: { label: function (item) { return fmtSigned(item.parsed.y) + ' Pages in this hour'; } } }
}
}
});
/* 5. growth by hour of day (whole history) */
const hodSum = new Array(24).fill(0);
const hodCount = new Array(24).fill(0);
deltas.forEach(function (d) {
const h = new Date(d.x).getHours();
hodSum[h] += d.y;
hodCount[h] += 1;
});
const hodValues = hodSum.map(function (s, i) {
return hodCount[i] ? Math.round(s / hodCount[i] * 10) / 10 : 0;
});
charts.hod = new Chart(document.getElementById('hodChart'), {
type: 'bar',
data: {
labels: hodValues.map(function (_, i) { return (i < 10 ? '0' : '') + i + ':00'; }),
datasets: [{
label: 'Avg gain per 5 min',
data: hodValues,
backgroundColor: hodValues.map(function (v) { return v >= 0 ? 'rgba(79,140,255,0.65)' : 'rgba(255,107,107,0.65)'; }),
borderRadius: 4
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
scales: {
x: { ticks: { color: mutedColor(), maxRotation: 0, autoSkip: true }, grid: { display: false } },
y: {
ticks: { color: mutedColor(), callback: function (v) { return fmtSigned(v); } },
grid: {
color: function (c) { return c.tick.value === 0 ? 'rgba(128,128,128,0.5)' : grid; }
}
}
},
plugins: {
legend: { labels: { usePointStyle: true, boxWidth: 8, color: mutedColor() } },
tooltip: { callbacks: { label: function (item) { return fmtSigned(item.parsed.y) + ' Pages per 5 min (avg of ' + hodCount[item.dataIndex] + ' measurements)'; } } }
}
}
});
}
/* ---------- records table ---------- */
function renderRecords(deltas) {
const el = document.getElementById('records');
if (deltas.length < 2) {
el.innerHTML = '<div class="chart-sub">Not enough data yet.</div>';
return;
}
const sorted = deltas.slice().sort(function (a, b) { return b.diff - a.diff; });
const gains = sorted.slice(0, 5).filter(function (d) { return d.diff > 0; });
const losses = sorted.slice().reverse().slice(0, 5).filter(function (d) { return d.diff < 0; }).reverse();
function rows(items, cls) {
return items.map(function (d) {
return '<tr><td>' + fmtDate(d.x) + '</td><td class="num ' + cls + '">' + fmtSigned(d.diff) + '</td><td class="num">' + fmtDur(d.dt) + '</td></tr>';
}).join('');
}
el.innerHTML =
'<table class="records">' +
'<tr><th>Time</th><th style="text-align:right">Change</th><th style="text-align:right">Span</th></tr>' +
rows(gains, 'pos') +
rows(losses, 'neg') +
'</table>';
}
/* ---------- heatmap ---------- */
function renderHeatmap(points) {
const el = document.getElementById('heatmap');
if (!points.length) { el.innerHTML = ''; return; }
function localDayKey(d) {
return d.getFullYear() + '-' +
('0' + (d.getMonth() + 1)).slice(-2) + '-' +
('0' + d.getDate()).slice(-2);
}
const map = {};
points.forEach(function (p) {
const d = new Date(p.x);
map[localDayKey(d) + '|' + d.getHours()] = p.y;
});
const days = {};
Object.keys(map).forEach(function (k) { days[k.split('|')[0]] = true; });
const dayList = Object.keys(days).sort();
// deltas per day+hour for coloring
const cellDeltas = {};
let maxAbs = 0;
const sorted = points.slice().sort(function (a, b) { return a.x - b.x; });
const byDayHour = {};
sorted.forEach(function (p) {
const d = new Date(p.x);
const key = localDayKey(d) + '|' + d.getHours();
if (byDayHour[key] !== undefined) {
cellDeltas[key] = p.y - byDayHour[key];
maxAbs = Math.max(maxAbs, Math.abs(cellDeltas[key]));
}
byDayHour[key] = p.y;
});
let html = '<table><tr><th></th>';
for (let h = 0; h < 24; h++) html += '<th>' + h + '</th>';
html += '</tr>';
dayList.forEach(function (day) {
html += '<tr><td class="day">' + fmtDay(new Date(day + 'T12:00:00').getTime()) + '</td>';
for (let h = 0; h < 24; h++) {
const key = day + '|' + h;
const val = map[key];
let style = '';
let title = fmtDay(new Date(day + 'T12:00:00').getTime()) + ' ' + h + ':00 – no data';
if (val !== undefined && cellDeltas[key] !== undefined) {
const dv = cellDeltas[key];
const op = maxAbs ? Math.min(0.85, 0.15 + 0.7 * Math.abs(dv) / maxAbs) : 0.3;
const color = dv >= 0 ? '46,204,113' : '255,107,107';
style = ' style="background:rgba(' + color + ',' + op.toFixed(2) + ')"';
title = fmtDay(new Date(day + 'T12:00:00').getTime()) + ' ' + h + ':00 – ' + fmtSigned(dv) + ' Pages (' + fmt(val) + ' total)';
}
html += '<td><div class="cell"' + style + ' title="' + title + '"></div></td>';
}
html += '</tr>';
});
html += '</table>';
el.innerHTML = html;
}
/* ---------- main update ---------- */
function renderAll() {
const all = parsePoints(rawValues);
const points = filterRange(all);
const now = Date.now();
const elCurrent = document.getElementById('current');
const meta = document.getElementById('current-meta');
document.getElementById('badge-count').textContent = 'Measurements: ' + all.length;
if (!all.length) {
elCurrent.textContent = '–';
meta.textContent = 'No measurements yet – the collector stores one every 15 minutes.';
setStatus('offline', 'No data');
return;
}
const last = all[all.length - 1];
const first = all[0];
const dts = [];
for (let i = 1; i < all.length; i++) dts.push(all[i].x - all[i - 1].x);
intervalMs = median(dts) || FALLBACK_INTERVAL_MS;
document.getElementById('badge-interval').textContent = 'Interval: ~' + fmtDur(intervalMs);
elCurrent.textContent = fmt(last.y);
meta.textContent = 'Last measurement: ' + fmtDateFull(last.x) +
' · next expected in ~' + fmtDur(Math.max(0, last.x + intervalMs - now));
document.getElementById('badge-since').textContent = 'First measurement: ' + fmtDateFull(first.x);
document.getElementById('badge-updated').textContent = 'Last update: ' + fmtDate(last.x);
const age = now - last.x;
if (age <= 3 * intervalMs) {
setStatus('ok', 'Live · ' + fmtDur(age) + ' ago');
} else if (age <= 24 * 3600000) {
setStatus('stale', 'Stale · last update ' + fmtDur(age) + ' ago');
} else {
setStatus('offline', 'No data for ' + fmtDur(age));
}
/* fixed-window cards (full history) */
const totalHours = (last.x - first.x) / 3600000;
const perHour = totalHours > 0 ? (last.y - first.y) / totalHours : null;
const elPh = document.getElementById('stat-per-hour');
const elPd = document.getElementById('stat-per-day');
if (perHour === null) {
elPh.textContent = '–';
elPd.textContent = '–';
} else {
setSigned('stat-per-hour', perHour);
setSigned('stat-per-day', perHour * 24);
}
const span24h = closestBefore(all, last.x - 24 * 3600000, Math.max(6 * intervalMs, 3 * 3600000));
setDeltaCard('stat-delta-24h', 'stat-delta-24h-hint',
span24h ? last.y - span24h.y : null,
span24h ? last.x - span24h.x : 0,
span24h ? (last.y - span24h.y) * (3600000 / Math.max(1, last.x - span24h.x)) * (5 / 60) : 0);
const span7d = closestBefore(all, last.x - 7 * 24 * 3600000, Math.max(12 * intervalMs, 6 * 3600000));
setDeltaCard('stat-delta-7d', 'stat-delta-7d-hint',
span7d ? last.y - span7d.y : null,
span7d ? last.x - span7d.x : 0,
span7d ? (last.y - span7d.y) * (3600000 / Math.max(1, last.x - span7d.x)) * (5 / 60) : 0);
/* range-dependent cards + charts */
const fpoints = points.length >= 2 ? points : all;
const dd = computeDeltas(fpoints);
const deltas = dd.deltas;
const smoothY = movingAverage(deltas.map(function (d) { return d.y; }), DELTA_SMOOTH_WINDOW);
const smoothedDeltas = deltas.map(function (d, i) {
return { x: d.x, y: smoothY[i], raw: d.y, dt: d.dt, diff: d.diff };
});
/* avg/h over last 24 h: sum of all changes within 24 h ÷ hours actually covered */
let sum24 = 0;
let windowStart = Math.max(fpoints[0].x, last.x - 24 * 3600000);
let coveredH = (last.x - windowStart) / 3600000;
deltas.forEach(function (d) {
if (d.x > last.x - 24 * 3600000) sum24 += d.diff;
});
if (coveredH >= 0.5) {
setSigned('stat-avg-hour', sum24 / coveredH);
} else {
document.getElementById('stat-avg-hour').textContent = '–';
}
/* best / worst hour: best total gain within a sliding ~1 h window */
let best = null, worst = null;
for (let i = 0; i < deltas.length; i++) {
let gain = 0, endMs = deltas[i].x;
for (let j = i; j < deltas.length; j++) {
if (deltas[j].x - deltas[i].x > 3600000) break;
gain += deltas[j].diff;
endMs = deltas[j].x;
}
if (!best || gain > best.gain) best = { gain: gain, x: endMs };
if (!worst || gain < worst.gain) worst = { gain: gain, x: endMs };
}
if (best && best.gain !== 0) {
setSigned('stat-best-hour', best.gain);
document.getElementById('stat-best-hour-hint').textContent = 'Around ' + fmtDate(best.x);
}
if (worst && worst.gain !== 0) {
setSigned('stat-worst-hour', worst.gain);
document.getElementById('stat-worst-hour-hint').textContent = 'Around ' + fmtDate(worst.x);
}
/* peak / lowest (whole history) */
let peak = all[0], low = all[0];
all.forEach(function (p) {
if (p.y > peak.y) peak = p;
if (p.y < low.y) low = p;
});
document.getElementById('stat-peak').textContent = fmt(peak.y);
document.getElementById('stat-peak-hint').textContent = 'On ' + fmtDateFull(peak.x);
document.getElementById('stat-min').textContent = fmt(low.y);
document.getElementById('stat-min-hint').textContent = 'On ' + fmtDateFull(low.x);
/* forecast */
const regWindow = [];
for (let i = fpoints.length - 1; i >= 0; i--) {
regWindow.unshift(fpoints[i]);
if (fpoints.length - i >= 96) break;
if (fpoints[fpoints.length - 1].x - fpoints[i].x >= 2 * 3600000) break;
}
const reg = regress(regWindow);
if (reg) {
const elF = document.getElementById('stat-forecast');
const fc = fpoints[fpoints.length - 1].y + reg.slope;
elF.textContent = fmt(fc);
elF.classList.remove('pos', 'neg');
elF.classList.add(fc >= fpoints[fpoints.length - 1].y ? 'pos' : 'neg');
document.getElementById('stat-forecast-hint').textContent =
'Trend ' + fmtSigned(reg.slope) + '/h · R² = ' + reg.r2.toFixed(3);
document.getElementById('footer').textContent =
'Forecast: linear least-squares regression over the last ' + regWindow.length +
' measurements (' + fmtDur(fpoints[fpoints.length - 1].x - regWindow[0].x) +
'), extrapolated +24 h · R² = ' + reg.r2.toFixed(3) +
' · Range: ' + (range === 'all' ? 'all' : range) + ' · All statistics are computed in the browser.';
} else {
document.getElementById('stat-forecast').textContent = '–';
document.getElementById('stat-forecast-hint').textContent = 'Needs at least 2 measurements';
document.getElementById('footer').textContent = 'Forecast available once there are at least 2 measurements.';
}
renderCharts(fpoints, deltas, smoothedDeltas);
renderRecords(deltas);
renderHeatmap(all);
}
async function update() {
refreshLeft = REFRESH_S;
try {
const response = await fetch('/api/data');
rawValues = (await response.json()).values || [];
} catch (e) {
setStatus('offline', 'Fetch error – retrying…');
return;
}
renderAll();
}
setInterval(function () {
refreshLeft -= 1;
if (refreshLeft <= 0) {
document.getElementById('badge-refresh').textContent = 'Refreshing…';
} else {
document.getElementById('badge-refresh').textContent = 'Next refresh: ' + refreshLeft + ' s';
}
}, 1000);
update();
setInterval(update, REFRESH_S * 1000);
</script>
</body>
</html>
"""
def load_data():
if not DB_DATA.exists():
return []
try:
with DB_DATA.open("r", encoding="utf-8") as f:
return json.load(f)
except Exception:
return []
def save_data(data):
tmp = DB_DATA.with_suffix(".tmp")
with tmp.open("w", encoding="utf-8") as f:
json.dump(data, f, indent=2)
tmp.replace(DB_DATA)
def get_page_count():
command = [
"sudo",
"-u",
"postgres",
"psql",
"-p",
"25432",
"-d",
"jonassearch",
"-t",
"-A",
"-c",
"SELECT COUNT(*) FROM pages;"
]
result = subprocess.run(
command,
capture_output=True,
text=True,
check=True
)
return int(result.stdout.strip())
def collect():
while True:
try:
count = get_page_count()
entry = {
"timestamp": datetime.now().isoformat(),
"count": count
}
data = load_data()
data.append(entry)
save_data(data)
print(
f"[{entry['timestamp']}] "
f"Pages: {count}"
)
except Exception as e:
print(f"Fehler beim Abrufen der Seitenzahl: {e}")
time.sleep(INTERVAL)
@app.route("/")
def index():
return render_template_string(HTML)
@app.route("/api/data")
def api_data():
return jsonify({
"values": load_data()
})
@app.route("/api/export")
def api_export():
lines = ["timestamp;count"]
for entry in load_data():
lines.append(f"{entry.get('timestamp', '')};{entry.get('count', '')}")
return Response("\n".join(lines), mimetype="text/csv")
if __name__ == "__main__":
# Collector im Hintergrund starten
thread = threading.Thread(target=collect, daemon=True)
thread.start()
# Webserver
app.run(
host="0.0.0.0",
port=8081,
threaded=True,
load_dotenv=False # sonst liest Flask beim Start .env im Projektordner -> PermissionError
)