Files
search/web/app.py
T
2026-09-05 21:50:42 +02:00

724 lines
23 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
import subprocess
import json
import time
import threading
from datetime import datetime
from pathlib import Path
from flask import Flask, jsonify, render_template_string
DB_DATA = Path(__file__).parent / "page_counts.json"
INTERVAL = 15 * 60 # 15 Minuten
app = Flask(__name__)
HTML = """
<!DOCTYPE html>
<html lang="de">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>JonasSearch – Pages</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<style>
:root {
--bg: #0f1115;
--card: #1b1e24;
--border: #2c313a;
--text: #e8eaed;
--muted: #9aa0a6;
--accent: #4f8cff;
--accent2: #b47aff;
--up: #2ecc71;
--down: #ff6b6b;
--warn: #f5a623;
}
* { box-sizing: border-box; }
body {
margin: 0;
background: var(--bg);
color: var(--text);
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Arial, sans-serif;
}
.container { max-width: 1200px; margin: auto; padding: 30px 24px 60px; }
header { display: flex; flex-wrap: wrap; align-items: baseline; gap: 8px 16px; }
h1 { margin: 0; font-size: 1.9rem; letter-spacing: 0.3px; }
.subtitle { color: var(--muted); }
.current {
font-size: 3.2rem;
font-weight: 700;
margin: 18px 0 4px;
line-height: 1;
}
.current .unit {
font-size: 1.1rem;
font-weight: 400;
color: var(--muted);
margin-left: 8px;
}
.current-meta { color: var(--muted); margin-bottom: 24px; }
.badges { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 22px; }
.badge {
display: inline-flex;
align-items: center;
gap: 6px;
background: var(--card);
border: 1px solid var(--border);
color: var(--muted);
font-size: 0.8rem;
padding: 5px 10px;
border-radius: 999px;
}
.dot { width: 8px; height: 8px; border-radius: 50%; background: var(--warn); display: inline-block; }
.dot.ok { background: var(--up); }
.dot.stale { background: var(--warn); }
.dot.offline { background: var(--down); }
.dot.live { animation: pulse 2s ease-in-out infinite; }
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.35; }
}
.grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(165px, 1fr));
gap: 14px;
margin-bottom: 22px;
}
.card {
background: var(--card);
border: 1px solid var(--border);
border-radius: 15px;
padding: 16px 18px;
min-width: 0;
}
.card .label {
color: var(--muted);
font-size: 0.78rem;
text-transform: uppercase;
letter-spacing: 0.8px;
margin-bottom: 8px;
}
.card .value { font-size: 1.45rem; font-weight: 600; }
.card .hint { color: var(--muted); font-size: 0.78rem; margin-top: 6px; }
.pos { color: var(--up); }
.neg { color: var(--down); }
.chart-container {
background: var(--card);
border: 1px solid var(--border);
padding: 20px;
border-radius: 15px;
margin-bottom: 22px;
}
.chart-box { position: relative; height: 340px; }
.chart-box.small { height: 260px; }
.chart-title { font-weight: 600; margin-bottom: 2px; }
.chart-sub { color: var(--muted); font-size: 0.82rem; margin-bottom: 14px; }
.footer { color: var(--muted); font-size: 0.8rem; margin-top: 8px; }
@media (max-width: 600px) {
.current { font-size: 2.4rem; }
.container { padding: 20px 14px 40px; }
.chart-container { padding: 14px; }
.chart-box { height: 260px; }
.chart-box.small { height: 200px; }
}
</style>
</head>
<body>
<div class="container">
<header>
<h1>JonasSearch</h1>
<div class="subtitle">Indexed Pages – Live Statistics</div>
</header>
<div class="current">
<span id="current">–</span><span class="unit">Pages</span>
</div>
<div class="current-meta" id="current-meta">Loading…</div>
<div class="badges">
<span class="badge"><span class="dot" id="status-dot"></span><span id="status-text">Loading…</span></span>
<span class="badge" id="badge-interval">Interval: –</span>
<span class="badge" id="badge-since">First measurement: –</span>
<span class="badge" id="badge-count">Measurements: –</span>
<span class="badge" id="badge-updated">Last update: –</span>
</div>
<div class="grid">
<div class="card">
<div class="label">Growth / hour</div>
<div class="value" id="stat-per-hour">–</div>
<div class="hint">Average over full history</div>
</div>
<div class="card">
<div class="label">Growth / day</div>
<div class="value" id="stat-per-day">–</div>
<div class="hint">Average over full history</div>
</div>
<div class="card">
<div class="label">Last 5 min</div>
<div class="value" id="stat-delta-5m">–</div>
<div class="hint" id="stat-delta-5m-hint">Latest interval change</div>
</div>
<div class="card">
<div class="label">Last hour</div>
<div class="value" id="stat-delta-1h">–</div>
<div class="hint" id="stat-delta-1h-hint">vs. measurement ~1 h ago</div>
</div>
<div class="card">
<div class="label">Last 24 h</div>
<div class="value" id="stat-delta-24h">–</div>
<div class="hint" id="stat-delta-24h-hint">vs. measurement ~24 h ago</div>
</div>
<div class="card">
<div class="label">Forecast next hour</div>
<div class="value" id="stat-forecast">–</div>
<div class="hint" id="stat-forecast-hint">Linear trend extrapolation</div>
</div>
</div>
<div class="chart-container">
<div class="chart-title">Indexed pages over time</div>
<div class="chart-sub">X-axis shows the real measurement timestamps · dashed line = linear-regression trend &amp; 24 h forecast</div>
<div class="chart-box"><canvas id="mainChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Change per 5 minutes</div>
<div class="chart-sub">Computed from the raw measurements: each change is normalized to a 5-minute interval</div>
<div class="chart-box small"><canvas id="deltaChart"></canvas></div>
</div>
<div class="footer" id="footer"></div>
</div>
<script>
'use strict';
if (window.Chart && Chart.registerables) {
Chart.register.apply(Chart, Chart.registerables);
}
Chart.defaults.color = '#9aa0a6';
Chart.defaults.borderColor = '#2c313a';
Chart.defaults.font.family = "-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Arial, sans-serif";
const ACCENT = '#4f8cff';
const ACCENT2 = '#b47aff';
const UP = '#2ecc71';
const DOWN = '#ff6b6b';
const FALLBACK_INTERVAL_MS = 15 * 60 * 1000;
function fmt(n) { return Math.round(n).toLocaleString('de-DE'); }
function fmtSigned(n) {
const s = Math.round(n).toLocaleString('de-DE');
return n > 0 ? '+' + s : s;
}
function fmtDate(ms) {
return new Date(ms).toLocaleString('de-DE', {
day: '2-digit', month: '2-digit', hour: '2-digit', minute: '2-digit'
});
}
function fmtDateFull(ms) {
return new Date(ms).toLocaleString('de-DE');
}
function fmtDur(ms) {
const min = Math.round(ms / 60000);
if (min < 1) return '<1 min';
if (min < 60) return min + ' min';
const h = Math.floor(min / 60);
const m = min % 60;
if (h < 24) return m ? (h + ' h ' + m + ' min') : (h + ' h');
const d = Math.floor(h / 24);
return d + ' d ' + (h % 24) + ' h';
}
function regress(points) {
const n = points.length;
if (n < 2) return null;
const t0 = points[0].x;
let sx = 0, sy = 0, sxy = 0, sxx = 0;
for (const p of points) {
const x = (p.x - t0) / 3600000;
sx += x; sy += p.y; sxy += x * p.y; sxx += x * x;
}
const den = n * sxx - sx * sx;
if (den === 0) return null;
const b = (n * sxy - sx * sy) / den;
const a = (sy - b * sx) / n;
let ssTot = 0, ssRes = 0;
const mean = sy / n;
for (const p of points) {
const x = (p.x - t0) / 3600000;
const pred = a + b * x;
ssTot += Math.pow(p.y - mean, 2);
ssRes += Math.pow(p.y - pred, 2);
}
const r2 = ssTot === 0 ? 1 : Math.max(0, Math.min(1, 1 - ssRes / ssTot));
return {
slope: b,
r2: r2,
at: function (ms) { return a + b * ((ms - t0) / 3600000); }
};
}
function median(arr) {
if (!arr.length) return null;
const s = arr.slice().sort(function (a, b) { return a - b; });
const mid = Math.floor(s.length / 2);
return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
}
function closestBefore(points, targetMs, maxLagMs) {
let best = null;
for (const p of points) {
if (p.x <= targetMs && (!best || p.x > best.x)) best = p;
}
if (best && targetMs - best.x <= maxLagMs) return best;
return null;
}
function setDeltaCard(id, idHint, delta, spanMs, per5) {
const el = document.getElementById(id);
const hint = document.getElementById(idHint);
if (delta === null) {
el.textContent = '–';
el.classList.remove('pos', 'neg');
hint.textContent = 'Not enough data yet';
return;
}
el.textContent = fmtSigned(delta);
el.classList.remove('pos', 'neg');
el.classList.add(delta >= 0 ? 'pos' : 'neg');
hint.textContent = fmtSigned(spanMs > 0 ? delta * (3600000 / spanMs) : 0) + ' Pages/h · ' + fmtSigned(per5) + ' per 5 min over ' + fmtDur(spanMs);
}
let mainChart = null;
let deltaChart = null;
let intervalMs = FALLBACK_INTERVAL_MS;
function gradientFill(context) {
const chart = context.chart;
const ctx = chart.ctx;
const area = chart.chartArea;
if (!area) return 'rgba(79,140,255,0.15)';
const g = ctx.createLinearGradient(0, area.top, 0, area.bottom);
g.addColorStop(0, 'rgba(79,140,255,0.28)');
g.addColorStop(1, 'rgba(79,140,255,0.02)');
return g;
}
const xAxisConfig = {
type: 'linear',
ticks: {
maxTicksLimit: 8,
maxRotation: 0,
autoSkip: true,
callback: function (v) { return fmtDate(v); }
},
grid: { color: 'rgba(255,255,255,0.05)' }
};
function buildCharts(points, trendPoints, deltas) {
mainChart = new Chart(document.getElementById('mainChart'), {
type: 'line',
data: {
datasets: [
{
label: 'Pages',
data: points,
parsing: false,
borderColor: ACCENT,
borderWidth: 2,
pointRadius: 2.5,
pointHoverRadius: 5,
tension: 0.25,
fill: true,
backgroundColor: gradientFill,
spanGaps: false
},
{
label: 'Trend / Forecast',
data: trendPoints,
parsing: false,
borderColor: ACCENT2,
borderDash: [6, 6],
borderWidth: 2,
pointRadius: 0,
fill: false,
spanGaps: true
}
]
},
options: {
responsive: true,
maintainAspectRatio: false,
normalized: true,
interaction: { mode: 'nearest', axis: 'x', intersect: false },
scales: {
x: xAxisConfig,
y: {
grace: '5%',
ticks: { callback: function (v) { return fmt(v); } },
grid: { color: 'rgba(255,255,255,0.05)' }
}
},
plugins: {
legend: { labels: { usePointStyle: true, boxWidth: 8 } },
tooltip: {
callbacks: {
title: function (items) { return fmtDateFull(items[0].parsed.x); },
label: function (item) {
return item.datasetIndex === 0
? (fmt(item.parsed.y) + ' Pages')
: ('Trend: ' + fmt(item.parsed.y) + ' Pages');
}
}
}
}
}
});
deltaChart = new Chart(document.getElementById('deltaChart'), {
type: 'bar',
data: {
datasets: [{
label: 'Change per 5 min',
data: deltas,
parsing: false,
backgroundColor: function (c) {
return c.raw && c.raw.y >= 0 ? 'rgba(46,204,113,0.75)' : 'rgba(255,107,107,0.75)';
},
borderColor: function (c) {
return c.raw && c.raw.y >= 0 ? UP : DOWN;
},
borderWidth: 1,
maxBarThickness: 28
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
normalized: true,
interaction: { mode: 'nearest', axis: 'x', intersect: false },
scales: {
x: xAxisConfig,
y: {
ticks: { callback: function (v) { return fmtSigned(v); } },
grid: {
color: function (c) { return c.tick.value === 0 ? 'rgba(255,255,255,0.35)' : 'rgba(255,255,255,0.05)'; }
}
}
},
plugins: {
legend: { display: false },
tooltip: {
callbacks: {
title: function (items) { return fmtDateFull(items[0].parsed.x); },
label: function (item) {
const raw = item.raw;
return [
fmtSigned(raw.y) + ' Pages per 5 min',
'Actual change: ' + fmtSigned(raw.diff) + ' in ' + fmtDur(raw.dt)
];
}
}
}
}
}
});
}
async function update() {
let payload;
try {
const response = await fetch('/api/data');
payload = await response.json();
} catch (e) {
setStatus('offline', 'Fetch error – retrying…');
return;
}
const measurements = (payload && payload.values) ? payload.values.slice() : [];
measurements.sort(function (a, b) { return new Date(a.timestamp) - new Date(b.timestamp); });
const points = measurements.map(function (v) {
return { x: new Date(v.timestamp).getTime(), y: v.count };
});
const now = Date.now();
const elCurrent = document.getElementById('current');
const meta = document.getElementById('current-meta');
document.getElementById('badge-count').textContent = 'Measurements: ' + points.length;
if (!points.length) {
elCurrent.textContent = '–';
meta.textContent = 'No measurements yet – the collector stores one every 15 minutes.';
setStatus('offline', 'No data');
return;
}
const last = points[points.length - 1];
const first = points[0];
const dts = [];
for (let i = 1; i < points.length; i++) dts.push(points[i].x - points[i - 1].x);
intervalMs = median(dts) || FALLBACK_INTERVAL_MS;
document.getElementById('badge-interval').textContent =
'Interval: ~' + fmtDur(intervalMs);
elCurrent.textContent = fmt(last.y);
meta.textContent = 'Last measurement: ' + fmtDateFull(last.x) +
' · next expected in ~' + fmtDur(Math.max(0, last.x + intervalMs - now));
document.getElementById('badge-since').textContent = 'First measurement: ' + fmtDateFull(first.x);
document.getElementById('badge-updated').textContent = 'Last update: ' + fmtDate(last.x);
const age = now - last.x;
if (age <= 3 * intervalMs) {
setStatus('ok', 'Live · ' + fmtDur(age) + ' ago');
} else if (age <= 24 * 3600000) {
setStatus('stale', 'Stale · last update ' + fmtDur(age) + ' ago');
} else {
setStatus('offline', 'No data for ' + fmtDur(age));
}
const totalHours = (last.x - first.x) / 3600000;
const perHour = totalHours > 0 ? (last.y - first.y) / totalHours : null;
const elPh = document.getElementById('stat-per-hour');
const elPd = document.getElementById('stat-per-day');
if (perHour === null) {
elPh.textContent = '–';
elPd.textContent = '–';
} else {
elPh.textContent = fmtSigned(perHour);
elPh.classList.remove('pos', 'neg');
elPh.classList.add(perHour >= 0 ? 'pos' : 'neg');
elPd.textContent = fmtSigned(perHour * 24);
elPd.classList.remove('pos', 'neg');
elPd.classList.add(perHour >= 0 ? 'pos' : 'neg');
}
let deltas = [];
for (let i = 1; i < points.length; i++) {
const dt = points[i].x - points[i - 1].x;
const diff = points[i].y - points[i - 1].y;
if (dt <= 0 || dt > 4 * intervalMs) continue;
const per5 = dt < 60000 ? diff : diff * (5 * 60000 / dt);
deltas.push({ x: points[i].x, y: per5, dt: dt, diff: diff });
}
const lastDelta = deltas.length ? deltas[deltas.length - 1] : null;
if (lastDelta) {
const el = document.getElementById('stat-delta-5m');
el.textContent = fmtSigned(lastDelta.diff);
el.classList.remove('pos', 'neg');
el.classList.add(lastDelta.diff >= 0 ? 'pos' : 'neg');
document.getElementById('stat-delta-5m-hint').textContent =
fmtSigned(lastDelta.y) + ' per 5 min · over ' + fmtDur(lastDelta.dt);
} else {
document.getElementById('stat-delta-5m').textContent = '–';
document.getElementById('stat-delta-5m-hint').textContent = 'Needs at least 2 measurements';
}
const span1h = closestBefore(points, last.x - 3600000, Math.max(3 * intervalMs, 30 * 60000));
setDeltaCard('stat-delta-1h', 'stat-delta-1h-hint',
span1h ? last.y - span1h.y : null,
span1h ? last.x - span1h.x : 0,
span1h ? (last.y - span1h.y) * (3600000 / Math.max(1, last.x - span1h.x)) * (5 / 60) : 0);
const span24h = closestBefore(points, last.x - 24 * 3600000, Math.max(6 * intervalMs, 3 * 3600000));
setDeltaCard('stat-delta-24h', 'stat-delta-24h-hint',
span24h ? last.y - span24h.y : null,
span24h ? last.x - span24h.x : 0,
span24h ? (last.y - span24h.y) * (3600000 / Math.max(1, last.x - span24h.x)) * (5 / 60) : 0);
const regWindow = [];
for (let i = points.length - 1; i >= 0; i--) {
regWindow.unshift(points[i]);
if (points.length - i >= 96) break;
if (last.x - points[i].x >= 2 * 3600000) break;
}
const reg = regress(regWindow);
let trendPoints = [];
if (reg) {
const horizon = last.x + 24 * 3600000;
trendPoints = [
{ x: regWindow[0].x, y: reg.at(regWindow[0].x) },
{ x: horizon, y: reg.at(horizon) }
];
const elF = document.getElementById('stat-forecast');
const fc = last.y + reg.slope;
elF.textContent = fmt(fc);
elF.classList.remove('pos', 'neg');
elF.classList.add(fc >= last.y ? 'pos' : 'neg');
document.getElementById('stat-forecast-hint').textContent =
'Trend ' + fmtSigned(reg.slope) + '/h · R² = ' + reg.r2.toFixed(3);
document.getElementById('footer').textContent =
'Forecast: linear least-squares regression over the last ' + regWindow.length +
' measurements (' + fmtDur(last.x - regWindow[0].x) + '), extrapolated +24 h · R² = ' +
reg.r2.toFixed(3) + ' · All statistics are computed in the browser.';
} else {
document.getElementById('stat-forecast').textContent = '–';
document.getElementById('stat-forecast-hint').textContent = 'Needs at least 2 measurements';
document.getElementById('footer').textContent =
'Forecast available once there are at least 2 measurements.';
}
if (!mainChart) {
buildCharts(points, trendPoints, deltas);
} else {
mainChart.data.datasets[0].data = points;
mainChart.data.datasets[1].data = trendPoints;
mainChart.update('none');
deltaChart.data.datasets[0].data = deltas;
deltaChart.update('none');
}
}
function setStatus(kind, text) {
const dot = document.getElementById('status-dot');
dot.className = 'dot ' + kind + (kind === 'ok' ? ' live' : '');
document.getElementById('status-text').textContent = text;
}
update();
setInterval(update, 30000);
</script>
</body>
</html>
"""
def load_data():
if not DB_DATA.exists():
return []
try:
with DB_DATA.open("r", encoding="utf-8") as f:
return json.load(f)
except Exception:
return []
def save_data(data):
tmp = DB_DATA.with_suffix(".tmp")
with tmp.open("w", encoding="utf-8") as f:
json.dump(data, f, indent=2)
tmp.replace(DB_DATA)
def get_page_count():
command = [
"sudo",
"-u",
"postgres",
"psql",
"-p",
"25432",
"-d",
"jonassearch",
"-t",
"-A",
"-c",
"SELECT COUNT(*) FROM pages;"
]
result = subprocess.run(
command,
capture_output=True,
text=True,
check=True
)
return int(result.stdout.strip())
def collect():
while True:
try:
count = get_page_count()
entry = {
"timestamp": datetime.now().isoformat(),
"count": count
}
data = load_data()
data.append(entry)
save_data(data)
print(
f"[{entry['timestamp']}] "
f"Pages: {count}"
)
except Exception as e:
print(f"Fehler beim Abrufen der Seitenzahl: {e}")
time.sleep(INTERVAL)
@app.route("/")
def index():
return render_template_string(HTML)
@app.route("/api/data")
def api_data():
return jsonify({
"values": load_data()
})
if __name__ == "__main__":
# Collector im Hintergrund starten
thread = threading.Thread(target=collect, daemon=True)
thread.start()
# Webserver
app.run(
host="0.0.0.0",
port=8081,
threaded=True
)