Files
search/web/app.py
T

2016 lines
73 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
import subprocess
import json
import time
import threading
from datetime import datetime, timedelta
from pathlib import Path
from flask import Flask, jsonify, render_template_string, Response
DB_DATA = Path(__file__).parent / "page_counts.json"
INTERVAL = 30 * 60 # 30 Minuten
# Bilder werden nur alle 2 Stunden exakt gezählt (SUM über die ~10-GB-
# Tabelle). Dazwischen wird eine 1-%-Stichprobe (TABLESAMPLE SYSTEM)
# hochgerechnet - das kostet nur einen Bruchteil der IO.
EXACT_IMAGE_INTERVAL = 2 * 60 * 60 # Sekunden
IMAGE_SAMPLE_PERCENT = 1
# Sehr kleine Tabellen: Stichprobe kann leer sein; dort ist der exakte
# Count ohnehin billig.
SAMPLE_EXACT_FALLBACK_ROWS = 100_000
app = Flask(__name__)
# r""" raw string: preserves \n etc. for the embedded JavaScript
# (a normal string would eat the backslashes and break the JS)
HTML = r"""
<!DOCTYPE html>
<html lang="de">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>JonasSearch – Pages</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<style>
:root {
--bg: #0f1115;
--card: #1b1e24;
--border: #2c313a;
--text: #e8eaed;
--muted: #9aa0a6;
--accent: #4f8cff;
--accent2: #b47aff;
--up: #2ecc71;
--down: #ff6b6b;
--warn: #f5a623;
}
html[data-theme="light"] {
--bg: #f4f6fa;
--card: #ffffff;
--border: #d8dde5;
--text: #1a1d23;
--muted: #6b7280;
--accent: #2f6fe4;
--accent2: #8a4fe0;
}
* { box-sizing: border-box; }
body {
margin: 0;
background: var(--bg);
color: var(--text);
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Arial, sans-serif;
}
.container { max-width: 1200px; margin: auto; padding: 30px 24px 60px; }
header {
display: flex;
flex-wrap: wrap;
align-items: baseline;
gap: 8px 16px;
position: sticky;
top: 0;
z-index: 10;
background: var(--bg);
padding: 10px 0 14px;
border-bottom: 1px solid var(--border);
}
h1 { margin: 0; font-size: 1.9rem; letter-spacing: 0.3px; }
.subtitle { color: var(--muted); }
.header-right { margin-left: auto; display: flex; gap: 8px; align-items: center; }
.btn {
background: var(--card);
border: 1px solid var(--border);
color: var(--text);
font-size: 0.8rem;
padding: 6px 12px;
border-radius: 999px;
cursor: pointer;
text-decoration: none;
}
.btn:hover { border-color: var(--accent); }
.current {
font-size: 3.2rem;
font-weight: 700;
margin: 18px 0 4px;
line-height: 1;
}
.current .unit {
font-size: 1.1rem;
font-weight: 400;
color: var(--muted);
margin-left: 8px;
}
.current-meta { color: var(--muted); margin-bottom: 24px; }
.current-row { display: flex; flex-wrap: wrap; gap: 0 56px; }
.badges { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 14px; }
.badge {
display: inline-flex;
align-items: center;
gap: 6px;
background: var(--card);
border: 1px solid var(--border);
color: var(--muted);
font-size: 0.8rem;
padding: 5px 10px;
border-radius: 999px;
}
.dot { width: 8px; height: 8px; border-radius: 50%; background: var(--warn); display: inline-block; }
.dot.ok { background: var(--up); }
.dot.stale { background: var(--warn); }
.dot.offline { background: var(--down); }
.dot.live { animation: pulse 2s ease-in-out infinite; }
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.35; }
}
.ranges { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 18px; }
.image-options { display: flex; align-items: center; gap: 9px; margin: 0 0 12px; color: var(--muted); font-size: 0.88rem; }
.image-options input { accent-color: var(--accent); }
.image-options label { cursor: pointer; }
.image-options-hint { color: var(--muted); font-size: 0.78rem; margin-bottom: 14px; }
.metric-tabs {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(160px, 1fr));
gap: 10px;
margin: 22px 0 16px;
}
.metric-btn {
display: flex;
align-items: center;
gap: 12px;
text-align: left;
background: var(--card);
border: 1px solid var(--border);
color: var(--muted);
border-radius: 15px;
padding: 13px 16px;
font-size: 1rem;
font-weight: 600;
font-family: inherit;
cursor: pointer;
transition: border-color .15s, color .15s, background .15s;
}
.metric-btn:hover { border-color: var(--accent); color: var(--text); }
.metric-btn.active {
color: var(--text);
border-color: var(--accent);
background: linear-gradient(180deg, rgba(79,140,255,0.18), rgba(79,140,255,0.04));
box-shadow: inset 0 0 0 1px rgba(79,140,255,0.35);
}
.metric-icon { font-size: 1.6rem; line-height: 1; }
.metric-name { display: block; }
.metric-count {
display: block;
font-size: 0.78rem;
font-weight: 400;
color: var(--muted);
margin-top: 2px;
}
.metric-panel { display: none; }
.metric-panel.active { display: block; }
.section-title-first { margin-top: 6px; }
.range-btn {
background: var(--card);
border: 1px solid var(--border);
color: var(--muted);
font-size: 0.82rem;
padding: 6px 14px;
border-radius: 999px;
cursor: pointer;
}
.range-btn.active {
color: #fff;
background: var(--accent);
border-color: var(--accent);
}
.grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(165px, 1fr));
gap: 14px;
margin-bottom: 22px;
}
.card {
background: var(--card);
border: 1px solid var(--border);
border-radius: 15px;
padding: 16px 18px;
min-width: 0;
}
.card .label {
color: var(--muted);
font-size: 0.78rem;
text-transform: uppercase;
letter-spacing: 0.8px;
margin-bottom: 8px;
}
.card .value { font-size: 1.45rem; font-weight: 600; }
.card .hint { color: var(--muted); font-size: 0.78rem; margin-top: 6px; }
.pos { color: var(--up); }
.neg { color: var(--down); }
.chart-container {
background: var(--card);
border: 1px solid var(--border);
padding: 20px;
border-radius: 15px;
margin-bottom: 22px;
}
.chart-row { display: grid; grid-template-columns: repeat(auto-fit, minmax(420px, 1fr)); gap: 22px; }
.chart-row .chart-container { margin-bottom: 0; }
@media (max-width: 950px) {
.chart-row { grid-template-columns: 1fr; }
.chart-row .chart-container { margin-bottom: 22px; }
}
.chart-box { position: relative; height: 340px; }
.chart-box.small { height: 260px; }
.chart-title { font-weight: 600; margin-bottom: 2px; }
.chart-sub { color: var(--muted); font-size: 0.82rem; margin-bottom: 14px; }
.heatmap { overflow-x: auto; }
.heatmap table { border-collapse: collapse; width: 100%; min-width: 720px; }
.heatmap td, .heatmap th {
font-size: 0.68rem;
color: var(--muted);
padding: 2px;
text-align: center;
}
.heatmap td.day { text-align: right; padding-right: 8px; white-space: nowrap; }
.heatmap .cell {
width: 100%;
min-width: 22px;
height: 18px;
border-radius: 4px;
background: rgba(128,128,128,0.12);
}
.footer { color: var(--muted); font-size: 0.8rem; margin-top: 8px; }
table.records { width: 100%; border-collapse: collapse; font-size: 0.85rem; }
table.records th, table.records td {
text-align: left;
padding: 7px 8px;
border-bottom: 1px solid var(--border);
}
table.records th { color: var(--muted); font-weight: 500; font-size: 0.75rem; text-transform: uppercase; letter-spacing: 0.6px; }
table.records td.num { text-align: right; font-variant-numeric: tabular-nums; }
.section-title { font-weight: 600; margin: 30px 0 12px; font-size: 1.05rem; }
@media (max-width: 600px) {
.metric-btn { padding: 10px 12px; font-size: 0.9rem; gap: 9px; }
.metric-icon { font-size: 1.3rem; }
.current { font-size: 2.4rem; }
.container { padding: 20px 14px 40px; }
.chart-container { padding: 14px; }
.chart-box { height: 260px; }
.chart-box.small { height: 200px; }
}
</style>
</head>
<body>
<div class="container">
<header>
<h1>JonasSearch</h1>
<div class="subtitle" id="page-subtitle">Pages – Live Statistics</div>
<div class="header-right">
<a class="btn" id="btn-csv" href="#">Export CSV</a>
<a class="btn" id="btn-json" href="#">Export JSON</a>
<button class="btn" id="btn-theme" title="Toggle light/dark">☀️ Light</button>
</div>
</header>
<div class="metric-tabs" role="tablist" aria-label="Data type">
<button class="metric-btn active" data-metric="pages" role="tab" aria-selected="true">
<span class="metric-icon" aria-hidden="true">📄</span>
<span>
<span class="metric-name">Pages</span>
<span class="metric-count" id="tab-count-pages">– total</span>
</span>
</button>
<button class="metric-btn" data-metric="images" role="tab" aria-selected="false">
<span class="metric-icon" aria-hidden="true">🖼️</span>
<span>
<span class="metric-name">Images</span>
<span class="metric-count" id="tab-count-images">– total</span>
</span>
</button>
<button class="metric-btn" data-metric="queue" role="tab" aria-selected="false">
<span class="metric-icon" aria-hidden="true">⏳</span>
<span>
<span class="metric-name">Queue</span>
<span class="metric-count" id="tab-count-queue">– pending</span>
</span>
</button>
</div>
<div class="badges">
<span class="badge"><span class="dot" id="status-dot"></span><span id="status-text">Loading…</span></span>
<span class="badge" id="badge-interval">Interval: –</span>
<span class="badge" id="badge-since">First measurement: –</span>
<span class="badge" id="badge-count">Measurements: –</span>
<span class="badge" id="badge-updated">Last update: –</span>
<span class="badge" id="badge-refresh">Next refresh: –</span>
</div>
<div class="ranges">
<button class="range-btn active" data-range="all">All</button>
<button class="range-btn" data-range="7d">Last 7 days</button>
<button class="range-btn" data-range="24h">Last 24 h</button>
<button class="range-btn" data-range="6h">Last 6 h</button>
</div>
<div class="metric-panel active" data-metric="pages" role="tabpanel">
<div class="current-row">
<div>
<div class="current">
<span id="current">–</span><span class="unit">Pages</span>
</div>
<div class="current-meta" id="current-meta">Loading…</div>
</div>
</div>
<div class="grid">
<div class="card">
<div class="label">Growth / hour</div>
<div class="value" id="stat-per-hour">–</div>
<div class="hint">Average over full history</div>
</div>
<div class="card">
<div class="label">Growth / day</div>
<div class="value" id="stat-per-day">–</div>
<div class="hint">Average over full history</div>
</div>
<div class="card">
<div class="label">Growth 6 h</div>
<div class="value" id="stat-delta-6h">–</div>
<div class="hint" id="stat-delta-6h-hint">vs. measurement ~6 h ago</div>
</div>
<div class="card">
<div class="label">Growth 24 h</div>
<div class="value" id="stat-delta-24h">–</div>
<div class="hint" id="stat-delta-24h-hint">vs. measurement ~24 h ago</div>
</div>
<div class="card">
<div class="label">Growth 7 days</div>
<div class="value" id="stat-delta-7d">–</div>
<div class="hint" id="stat-delta-7d-hint">vs. measurement ~7 d ago</div>
</div>
<div class="card">
<div class="label">Avg / h (24 h)</div>
<div class="value" id="stat-avg-hour">–</div>
<div class="hint">Mean of last 24 h changes</div>
</div>
<div class="card">
<div class="label">Best hour</div>
<div class="value" id="stat-best-hour">–</div>
<div class="hint" id="stat-best-hour-hint">Largest gain within one hour</div>
</div>
<div class="card">
<div class="label">Worst hour</div>
<div class="value" id="stat-worst-hour">–</div>
<div class="hint" id="stat-worst-hour-hint">Largest loss within one hour</div>
</div>
<div class="card">
<div class="label">Peak</div>
<div class="value" id="stat-peak">–</div>
<div class="hint" id="stat-peak-hint">Highest measurement ever</div>
</div>
<div class="card">
<div class="label">Lowest</div>
<div class="value" id="stat-min">–</div>
<div class="hint" id="stat-min-hint">Lowest measurement ever</div>
</div>
<div class="card">
<div class="label">Forecast next hour</div>
<div class="value" id="stat-forecast">–</div>
<div class="hint" id="stat-forecast-hint">Linear trend extrapolation</div>
</div>
</div>
<div class="chart-container">
<div class="chart-title">Indexed pages over time</div>
<div class="chart-sub">Real measurement timestamps · dashed line = linear-regression trend &amp; 24 h forecast</div>
<div class="chart-box"><canvas id="mainChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Change per 5 minutes</div>
<div class="chart-sub">Normalized to a 5-minute interval · grey = raw · colored = smoothed (moving average over 5 measurements)</div>
<div class="chart-box small"><canvas id="deltaChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Cumulative net growth</div>
<div class="chart-sub">Total pages gained/lost since the start of the selected range · linear time axis</div>
<div class="chart-box small"><canvas id="cumChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Growth per hour (last 48 h)</div>
<div class="chart-sub">Sum of changes per hour bucket · green = growth, red = loss</div>
<div class="chart-box small"><canvas id="hourlyChart"></canvas></div>
</div>
<div class="chart-row">
<div class="chart-container">
<div class="chart-title">Growth by hour of day</div>
<div class="chart-sub">Average gain per 5 min, grouped by hour of day (whole history) · shows when the crawler is most active</div>
<div class="chart-box small"><canvas id="hodChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Records – biggest changes</div>
<div class="chart-sub">Largest gains and losses between two consecutive measurements (selected range)</div>
<div id="records"></div>
</div>
</div>
<div class="section-title">Activity heatmap – gain per hour of day</div>
<div class="chart-container heatmap">
<div class="chart-sub">Rows = days · columns = hours (0–23) · green = growth, red = loss, grey = no data</div>
<div id="heatmap"></div>
</div>
</div>
<div class="metric-panel" data-metric="images" role="tabpanel">
<div class="current-row">
<div>
<div class="current">
<span id="current-images">–</span><span class="unit">Images</span>
</div>
<div class="current-meta" id="current-images-meta">Loading…</div>
</div>
</div>
<div class="grid" id="summary-images"></div>
<div class="section-title section-title-first">Images – all graphs</div>
<div class="image-options">
<input type="checkbox" id="show-estimated-images">
<label for="show-estimated-images">Include estimated image counts in graphs</label>
</div>
<div class="image-options-hint" id="images-mode-note">Only exact measurements (counted every 2 h) are plotted. This setting is saved in this browser.</div>
<div class="chart-sub" id="images-empty" style="display:none">
No image counts yet – they appear after the next collector run with the image migration in place.
</div>
<div class="chart-container">
<div class="chart-title">Images over time</div>
<div class="chart-sub">Image URLs found while crawling · dashed line = trend &amp; 24 h forecast</div>
<div class="chart-box"><canvas id="imagesMainChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Change per 5 minutes</div>
<div class="chart-sub">Normalized to a 5-minute interval between available image measurements · grey = raw · colored = smoothed (moving average over 5 measurements)</div>
<div class="chart-box small"><canvas id="imagesDeltaChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Cumulative net growth</div>
<div class="chart-sub">Total images gained/lost since the start of the selected range · linear time axis</div>
<div class="chart-box small"><canvas id="imagesCumChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Images per hour (last 48 h)</div>
<div class="chart-sub">Sum of changes per hour bucket · green = growth, red = loss</div>
<div class="chart-box small"><canvas id="imagesHourlyChart"></canvas></div>
</div>
<div class="chart-row">
<div class="chart-container">
<div class="chart-title">Image growth by hour of day</div>
<div class="chart-sub">Average gain per 5 min, grouped by hour of day (whole history)</div>
<div class="chart-box small"><canvas id="imagesHodChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Records – biggest image changes</div>
<div class="chart-sub">Largest gains and losses between two consecutive measurements (selected range)</div>
<div id="records-images"></div>
</div>
</div>
<div class="section-title">Activity heatmap – image gain per hour of day</div>
<div class="chart-container heatmap">
<div class="chart-sub">Rows = days · columns = hours (0–23) · green = growth, red = loss, grey = no data</div>
<div id="heatmap-images"></div>
</div>
</div>
<div class="metric-panel" data-metric="queue" role="tabpanel">
<div class="current-row">
<div>
<div class="current">
<span id="current-pending">–</span><span class="unit">Pending</span>
</div>
<div class="current-meta" id="current-pending-meta">Loading…</div>
</div>
</div>
<div class="grid" id="summary-queue"></div>
<div class="section-title section-title-first">Queue – all graphs</div>
<div class="chart-sub">URLs waiting in crawl_queue (status = 'pending'). Counted with every 30-minute sample (exact COUNT). Falling graphs mean the crawler is working through the queue.</div>
<div class="chart-sub" id="pending-empty" style="display:none">
No queue counts yet – they appear after the next collector run.
</div>
<div class="chart-container">
<div class="chart-title">Pending URLs over time</div>
<div class="chart-sub">Queued URLs not yet crawled · dashed line = trend &amp; 24 h forecast</div>
<div class="chart-box"><canvas id="pendingMainChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Change per 5 minutes</div>
<div class="chart-sub">Normalized to a 5-minute interval · negative = the crawler worked the queue down · grey = raw · colored = smoothed (moving average over 5 measurements)</div>
<div class="chart-box small"><canvas id="pendingDeltaChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Cumulative net change</div>
<div class="chart-sub">Total queued URLs gained/lost since the start of the selected range · linear time axis</div>
<div class="chart-box small"><canvas id="pendingCumChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Queue change per hour (last 48 h)</div>
<div class="chart-sub">Sum of changes per hour bucket · green = queue grew, red = queue shrank</div>
<div class="chart-box small"><canvas id="pendingHourlyChart"></canvas></div>
</div>
<div class="chart-row">
<div class="chart-container">
<div class="chart-title">Queue change by hour of day</div>
<div class="chart-sub">Average change per 5 min, grouped by hour of day (whole history)</div>
<div class="chart-box small"><canvas id="pendingHodChart"></canvas></div>
</div>
<div class="chart-container">
<div class="chart-title">Records – biggest queue changes</div>
<div class="chart-sub">Largest gains and losses between two consecutive measurements (selected range)</div>
<div id="records-pending"></div>
</div>
</div>
<div class="section-title">Activity heatmap – queue change per hour of day</div>
<div class="chart-container heatmap">
<div class="chart-sub">Rows = days · columns = hours (0–23) · green = queue grew, red = queue shrank, grey = no data</div>
<div id="heatmap-pending"></div>
</div>
</div>
<div class="footer" id="footer"></div>
</div>
<script>
'use strict';
if (window.Chart && Chart.registerables) {
Chart.register.apply(Chart, Chart.registerables);
}
const ACCENT = '#4f8cff';
const ACCENT2 = '#b47aff';
const ACCENT3 = '#f5a623';
const UP = '#2ecc71';
const DOWN = '#ff6b6b';
const FALLBACK_INTERVAL_MS = 15 * 60 * 1000;
const DELTA_SMOOTH_WINDOW = 5;
const REFRESH_S = 30;
let rawValues = [];
let intervalMs = FALLBACK_INTERVAL_MS;
let range = 'all';
let activeMetric = 'pages';
let showEstimatedImages = false;
let charts = {};
let refreshLeft = REFRESH_S;
function cssVar(name) {
return getComputedStyle(document.documentElement).getPropertyValue(name).trim();
}
function borderColor() { return cssVar('--border') || '#2c313a'; }
function mutedColor() { return cssVar('--muted') || '#9aa0a6'; }
function fmt(n) { return Math.round(n).toLocaleString('de-DE'); }
function fmtSigned(n) {
const s = Math.round(n).toLocaleString('de-DE');
return n > 0 ? '+' + s : s;
}
function fmtDate(ms) {
return new Date(ms).toLocaleString('de-DE', {
day: '2-digit', month: '2-digit', hour: '2-digit', minute: '2-digit'
});
}
function fmtDateFull(ms) {
return new Date(ms).toLocaleString('de-DE');
}
function fmtDay(ms) {
return new Date(ms).toLocaleDateString('de-DE', { day: '2-digit', month: '2-digit' });
}
function fmtDur(ms) {
const min = Math.round(ms / 60000);
if (min < 1) return '<1 min';
if (min < 60) return min + ' min';
const h = Math.floor(min / 60);
const m = min % 60;
if (h < 24) return m ? (h + ' h ' + m + ' min') : (h + ' h');
const d = Math.floor(h / 24);
return d + ' d ' + (h % 24) + ' h';
}
function regress(points) {
const n = points.length;
if (n < 2) return null;
const t0 = points[0].x;
let sx = 0, sy = 0, sxy = 0, sxx = 0;
for (const p of points) {
const x = (p.x - t0) / 3600000;
sx += x; sy += p.y; sxy += x * p.y; sxx += x * x;
}
const den = n * sxx - sx * sx;
if (den === 0) return null;
const b = (n * sxy - sx * sy) / den;
const a = (sy - b * sx) / n;
let ssTot = 0, ssRes = 0;
const mean = sy / n;
for (const p of points) {
const x = (p.x - t0) / 3600000;
const pred = a + b * x;
ssTot += Math.pow(p.y - mean, 2);
ssRes += Math.pow(p.y - pred, 2);
}
const r2 = ssTot === 0 ? 1 : Math.max(0, Math.min(1, 1 - ssRes / ssTot));
return {
slope: b,
r2: r2,
at: function (ms) { return a + b * ((ms - t0) / 3600000); }
};
}
function movingAverage(values, window) {
const half = Math.floor(window / 2);
return values.map(function (_, i) {
let sum = 0, count = 0;
for (let j = i - half; j <= i + half; j++) {
if (j >= 0 && j < values.length) { sum += values[j]; count++; }
}
return sum / count;
});
}
function median(arr) {
if (!arr.length) return null;
const s = arr.slice().sort(function (a, b) { return a - b; });
const mid = Math.floor(s.length / 2);
return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
}
function closestBefore(points, targetMs, maxLagMs) {
let best = null;
for (const p of points) {
if (p.x <= targetMs && (!best || p.x > best.x)) best = p;
}
if (best && targetMs - best.x <= maxLagMs) return best;
return null;
}
function setDeltaCard(id, idHint, delta, spanMs, per5) {
const el = document.getElementById(id);
const hint = document.getElementById(idHint);
if (delta === null) {
el.textContent = '–';
el.classList.remove('pos', 'neg');
hint.textContent = 'Not enough data yet';
return;
}
el.textContent = fmtSigned(delta);
el.classList.remove('pos', 'neg');
el.classList.add(delta >= 0 ? 'pos' : 'neg');
hint.textContent = fmtSigned(spanMs > 0 ? delta * (3600000 / spanMs) : 0) + ' Pages/h · ' + fmtSigned(per5) + ' per 5 min over ' + fmtDur(spanMs);
}
function setSigned(id, value) {
const el = document.getElementById(id);
el.textContent = fmtSigned(value);
el.classList.remove('pos', 'neg');
el.classList.add(value >= 0 ? 'pos' : 'neg');
}
/* ---------- data helpers ---------- */
function parsePoints(values) {
return (values || [])
.map(function (v) {
const images = (v.images === null || v.images === undefined)
? null : Number(v.images);
const pending = (v.pending === null || v.pending === undefined)
? null : Number(v.pending);
return {
x: new Date(v.timestamp).getTime(),
y: v.count,
images: Number.isFinite(images) ? images : null,
imagesExact: v.images_exact === true,
pending: Number.isFinite(pending) ? pending : null
};
})
.filter(function (p) { return !isNaN(p.x); })
.sort(function (a, b) { return a.x - b.x; });
}
// Estimates are excluded by default; the Images section toggle can include them.
function toImagePoints(points, includeEstimated) {
return (points || [])
.filter(function (p) { return p.images !== null && p.images !== undefined; })
.filter(function (p) { return includeEstimated || p.imagesExact; })
.map(function (p) { return { x: p.x, y: p.images }; });
}
// Pending queue entries are counted with every sample (exact COUNT), so
// no filtering beyond "value present" is needed.
function toPendingPoints(points) {
return (points || [])
.filter(function (p) { return p.pending !== null && p.pending !== undefined; })
.map(function (p) { return { x: p.x, y: p.pending }; });
}
const RANGE_SPANS = {
'6h': 6 * 3600000,
'24h': 24 * 3600000,
'7d': 7 * 24 * 3600000
};
function filterRange(points, anchorX) {
if (range === 'all' || !points.length) return points;
const last = anchorX === undefined ? points[points.length - 1].x : anchorX;
const span = RANGE_SPANS[range] || RANGE_SPANS['24h'];
return points.filter(function (p) { return p.x >= last - span; });
}
function computeDeltas(points) {
const dts = [];
for (let i = 1; i < points.length; i++) dts.push(points[i].x - points[i - 1].x);
const iv = median(dts) || FALLBACK_INTERVAL_MS;
const deltas = [];
for (let i = 1; i < points.length; i++) {
const dt = points[i].x - points[i - 1].x;
const diff = points[i].y - points[i - 1].y;
if (dt <= 0 || dt > 4 * iv) continue;
const per5 = dt < 60000 ? diff : diff * (5 * 60000 / dt);
deltas.push({ x: points[i].x, y: per5, dt: dt, diff: diff });
}
return { deltas: deltas, interval: iv };
}
/* ---------- status ---------- */
function setStatus(kind, text) {
const dot = document.getElementById('status-dot');
dot.className = 'dot ' + kind + (kind === 'ok' ? ' live' : '');
document.getElementById('status-text').textContent = text;
}
/* ---------- theme ---------- */
function applyTheme(theme) {
document.documentElement.dataset.theme = theme;
document.getElementById('btn-theme').textContent = theme === 'light' ? '🌙 Dark' : '☀️ Light';
try { localStorage.setItem('js-theme', theme); } catch (e) {}
if (rawValues.length) renderAll(); // re-render charts with new palette
}
document.getElementById('btn-theme').addEventListener('click', function () {
applyTheme(document.documentElement.dataset.theme === 'light' ? 'dark' : 'light');
});
(function () {
const toggle = document.getElementById('show-estimated-images');
try { showEstimatedImages = localStorage.getItem('js-show-estimated-images') === 'true'; } catch (e) {}
toggle.checked = showEstimatedImages;
toggle.addEventListener('change', function () {
showEstimatedImages = toggle.checked;
try { localStorage.setItem('js-show-estimated-images', String(showEstimatedImages)); } catch (e) {}
if (rawValues.length) renderAll();
});
})();
(function () {
let saved = null;
try { saved = localStorage.getItem('js-theme'); } catch (e) {}
if (saved) document.documentElement.dataset.theme = saved;
document.getElementById('btn-theme').textContent = saved === 'light' ? '🌙 Dark' : '☀️ Light';
})();
/* ---------- metric tabs (pages / images / queue) ---------- */
const METRIC_LABELS = { pages: 'Pages', images: 'Images', queue: 'Queue' };
// Hidden panels must not render Chart.js instances: a canvas inside a
// display:none container measures 0 px and breaks on later resize.
function isVisible(el) {
const panel = el.closest('.metric-panel');
return !panel || panel.classList.contains('active');
}
function setMetric(next, persist) {
activeMetric = METRIC_LABELS[next] ? next : 'pages';
document.querySelectorAll('.metric-btn').forEach(function (btn) {
const on = btn.dataset.metric === activeMetric;
btn.classList.toggle('active', on);
btn.setAttribute('aria-selected', on ? 'true' : 'false');
});
document.querySelectorAll('.metric-panel').forEach(function (panel) {
panel.classList.toggle('active', panel.dataset.metric === activeMetric);
});
const label = METRIC_LABELS[activeMetric];
document.getElementById('page-subtitle').textContent = label + ' – Live Statistics';
document.title = 'JonasSearch – ' + label;
if (persist !== false) {
try { localStorage.setItem('js-metric', activeMetric); } catch (e) {}
}
if (rawValues.length) renderAll();
}
document.querySelectorAll('.metric-btn').forEach(function (btn) {
btn.addEventListener('click', function () { setMetric(btn.dataset.metric); });
});
/* ---------- range buttons ---------- */
document.querySelectorAll('.range-btn').forEach(function (btn) {
btn.addEventListener('click', function () {
document.querySelectorAll('.range-btn').forEach(function (b) { b.classList.remove('active'); });
btn.classList.add('active');
range = btn.dataset.range;
update();
});
});
/* ---------- export ---------- */
function download(name, text, mime) {
const blob = new Blob([text], { type: mime });
const a = document.createElement('a');
a.href = URL.createObjectURL(blob);
a.download = name;
a.click();
URL.revokeObjectURL(a.href);
}
// Export what the selected tab shows, so the CSV/JSON match the graphs.
function exportSeries() {
const all = parsePoints(rawValues);
if (activeMetric === 'images') {
const values = all
.filter(function (p) { return p.images !== null; })
.map(function (p) {
return { timestamp: new Date(p.x).toISOString(), images: p.images, images_exact: p.imagesExact };
});
return { name: 'jonassearch_images', values: values };
}
if (activeMetric === 'queue') {
const values = all
.filter(function (p) { return p.pending !== null; })
.map(function (p) {
return { timestamp: new Date(p.x).toISOString(), pending: p.pending };
});
return { name: 'jonassearch_queue', values: values };
}
return { name: 'jonassearch_pages', values: rawValues };
}
document.getElementById('btn-csv').addEventListener('click', function (e) {
e.preventDefault();
const series = exportSeries();
const keys = series.values.length ? Object.keys(series.values[0]) : ['timestamp'];
const lines = [keys.join(';')];
series.values.forEach(function (row) {
lines.push(keys.map(function (k) { return row[k] === undefined ? '' : row[k]; }).join(';'));
});
download(series.name + '.csv', lines.join('\n'), 'text/csv');
});
document.getElementById('btn-json').addEventListener('click', function (e) {
e.preventDefault();
const series = exportSeries();
download(
series.name + '.json',
JSON.stringify({ metric: activeMetric, values: series.values }, null, 2),
'application/json'
);
});
/* ---------- chart helpers ---------- */
function destroyCharts() {
Object.keys(charts).forEach(function (k) {
if (charts[k]) charts[k].destroy();
});
charts = {};
}
function baseScales(yFmt) {
return {
x: {
type: 'linear',
ticks: {
maxTicksLimit: 8,
maxRotation: 0,
autoSkip: true,
color: mutedColor(),
callback: function (v) { return fmtDate(v); }
},
grid: { color: 'rgba(128,128,128,0.15)' }
},
y: {
grace: '5%',
ticks: { color: mutedColor(), callback: yFmt },
grid: { color: 'rgba(128,128,128,0.15)' }
}
};
}
function tooltipTitle(items) { return fmtDateFull(items[0].parsed.x); }
// Pages keep the existing camelCase ids (mainChart); other metrics get a prefix
// (imagesMainChart).
function chartId(prefix, suffix) {
return prefix ? prefix + suffix : suffix.charAt(0).toLowerCase() + suffix.slice(1);
}
function gradientFill(colorTop, colorBottom) {
return function (context) {
const chart = context.chart;
const area = chart.chartArea;
if (!area) return colorBottom;
const g = chart.ctx.createLinearGradient(0, area.top, 0, area.bottom);
g.addColorStop(0, colorTop);
g.addColorStop(1, colorBottom);
return g;
};
}
/* ---------- charts ---------- */
function renderCharts(cfg) {
const target = document.getElementById(chartId(cfg.prefix, 'MainChart'));
if (!target || !isVisible(target)) return;
const prefix = cfg.prefix;
const points = cfg.points;
const deltas = cfg.deltas;
const smoothedDeltas = cfg.smoothedDeltas;
const unit = cfg.unit;
const accent = cfg.accent;
const accentBar = cfg.accentBar;
const grid = 'rgba(128,128,128,0.15)';
/* 1. main chart + trend */
const regWindow = [];
for (let i = points.length - 1; i >= 0; i--) {
regWindow.unshift(points[i]);
if (points.length - i >= 96) break;
if (points[points.length - 1].x - points[i].x >= 2 * 3600000) break;
}
const reg = regress(regWindow);
let trendPoints = [];
if (reg) {
const horizon = points[points.length - 1].x + 24 * 3600000;
trendPoints = [
{ x: regWindow[0].x, y: reg.at(regWindow[0].x) },
{ x: horizon, y: reg.at(horizon) }
];
}
charts[prefix + 'main'] = new Chart(document.getElementById(chartId(prefix, 'MainChart')), {
type: 'line',
data: {
datasets: [
{
label: unit,
data: points,
parsing: false,
borderColor: accent,
borderWidth: 2,
pointRadius: 2.5,
pointHoverRadius: 5,
tension: 0.25,
fill: true,
backgroundColor: gradientFill(cfg.gradTop, cfg.gradBottom),
spanGaps: false
},
{
label: 'Trend / Forecast',
data: trendPoints,
parsing: false,
borderColor: ACCENT2,
borderDash: [6, 6],
borderWidth: 2,
pointRadius: 0,
fill: false,
spanGaps: true
}
]
},
options: {
responsive: true,
maintainAspectRatio: false,
normalized: true,
interaction: { mode: 'nearest', axis: 'x', intersect: false },
scales: baseScales(function (v) { return fmt(v); }),
plugins: {
legend: { labels: { usePointStyle: true, boxWidth: 8, color: mutedColor() } },
tooltip: {
callbacks: {
title: tooltipTitle,
label: function (item) {
return item.datasetIndex === 0
? (fmt(item.parsed.y) + ' ' + unit)
: ('Trend: ' + fmt(item.parsed.y) + ' ' + unit);
}
}
}
}
}
});
/* 2. delta chart */
charts[prefix + 'delta'] = new Chart(document.getElementById(chartId(prefix, 'DeltaChart')), {
type: 'line',
data: {
datasets: [
{
label: 'Raw',
data: deltas,
parsing: false,
borderColor: 'rgba(128,128,128,0.30)',
borderWidth: 1,
pointRadius: 0,
tension: 0,
fill: false
},
{
label: 'Change per 5 min',
data: smoothedDeltas,
parsing: false,
borderColor: accent,
borderWidth: 2.5,
pointRadius: 0,
pointHoverRadius: 4,
tension: 0.35,
fill: false,
segment: {
borderColor: function (c) { return c.p1.parsed.y >= 0 ? UP : DOWN; }
}
}
]
},
options: {
responsive: true,
maintainAspectRatio: false,
normalized: true,
interaction: { mode: 'nearest', axis: 'x', intersect: false },
scales: {
x: baseScales(function () { return ''; }).x,
y: {
ticks: { color: mutedColor(), callback: function (v) { return fmtSigned(v); } },
grid: {
color: function (c) { return c.tick.value === 0 ? 'rgba(128,128,128,0.5)' : grid; }
}
}
},
plugins: {
legend: {
display: true,
labels: {
usePointStyle: true,
boxWidth: 8,
color: mutedColor(),
filter: function (item) { return item.text !== 'Raw'; }
}
},
tooltip: {
filter: function (item) { return item.datasetIndex === 1; },
callbacks: {
title: tooltipTitle,
label: function (item) {
const raw = item.raw;
return [
'Smoothed: ' + fmtSigned(raw.y) + ' ' + unit + ' per 5 min',
'Raw: ' + fmtSigned(raw.raw) + ' per 5 min (change ' + fmtSigned(raw.diff) + ' in ' + fmtDur(raw.dt) + ')'
];
}
}
}
}
}
});
/* 3. cumulative net growth */
const cum = points.map(function (p, i) {
return { x: p.x, y: p.y - points[0].y };
});
charts[prefix + 'cum'] = new Chart(document.getElementById(chartId(prefix, 'CumChart')), {
type: 'line',
data: {
datasets: [{
label: 'Net growth since range start',
data: cum,
parsing: false,
borderColor: UP,
borderWidth: 2,
pointRadius: 0,
pointHoverRadius: 4,
tension: 0.25,
fill: true,
backgroundColor: gradientFill('rgba(46,204,113,0.22)', 'rgba(46,204,113,0.02)')
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
normalized: true,
interaction: { mode: 'nearest', axis: 'x', intersect: false },
scales: {
x: baseScales(function () { return ''; }).x,
y: {
ticks: { color: mutedColor(), callback: function (v) { return fmtSigned(v); } },
grid: {
color: function (c) { return c.tick.value === 0 ? 'rgba(128,128,128,0.5)' : grid; }
}
}
},
plugins: {
legend: { labels: { usePointStyle: true, boxWidth: 8, color: mutedColor() } },
tooltip: {
callbacks: {
title: tooltipTitle,
label: function (item) { return fmtSigned(item.parsed.y) + ' ' + unit + ' since range start'; }
}
}
}
}
});
/* 4. hourly growth, last 48 h */
const buckets = {};
deltas.forEach(function (d) {
const h = Math.floor(d.x / 3600000) * 3600000;
buckets[h] = (buckets[h] || 0) + d.y;
});
const hours = Object.keys(buckets).map(Number).sort(function (a, b) { return a - b; }).slice(-48);
const hourlyLabels = hours.map(function (h) { return fmtDate(h); });
const hourlyValues = hours.map(function (h) { return Math.round(buckets[h]); });
charts[prefix + 'hourly'] = new Chart(document.getElementById(chartId(prefix, 'HourlyChart')), {
type: 'bar',
data: {
labels: hourlyLabels,
datasets: [{
label: 'Gain per hour',
data: hourlyValues,
backgroundColor: hourlyValues.map(function (v) { return v >= 0 ? 'rgba(46,204,113,0.65)' : 'rgba(255,107,107,0.65)'; }),
borderRadius: 4
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
interaction: { mode: 'nearest', axis: 'x', intersect: false },
scales: {
x: { ticks: { color: mutedColor(), maxRotation: 45, autoSkip: true, maxTicksLimit: 12 }, grid: { display: false } },
y: {
ticks: { color: mutedColor(), callback: function (v) { return fmtSigned(v); } },
grid: {
color: function (c) { return c.tick.value === 0 ? 'rgba(128,128,128,0.5)' : grid; }
}
}
},
plugins: {
legend: { labels: { usePointStyle: true, boxWidth: 8, color: mutedColor() } },
tooltip: { callbacks: { label: function (item) { return fmtSigned(item.parsed.y) + ' ' + unit + ' in this hour'; } } }
}
}
});
/* 5. growth by hour of day (whole history) */
const hodSum = new Array(24).fill(0);
const hodCount = new Array(24).fill(0);
deltas.forEach(function (d) {
const h = new Date(d.x).getHours();
hodSum[h] += d.y;
hodCount[h] += 1;
});
const hodValues = hodSum.map(function (s, i) {
return hodCount[i] ? Math.round(s / hodCount[i] * 10) / 10 : 0;
});
charts[prefix + 'hod'] = new Chart(document.getElementById(chartId(prefix, 'HodChart')), {
type: 'bar',
data: {
labels: hodValues.map(function (_, i) { return (i < 10 ? '0' : '') + i + ':00'; }),
datasets: [{
label: 'Avg gain per 5 min',
data: hodValues,
backgroundColor: hodValues.map(function (v) { return v >= 0 ? accentBar : 'rgba(255,107,107,0.65)'; }),
borderRadius: 4
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
scales: {
x: { ticks: { color: mutedColor(), maxRotation: 0, autoSkip: true }, grid: { display: false } },
y: {
ticks: { color: mutedColor(), callback: function (v) { return fmtSigned(v); } },
grid: {
color: function (c) { return c.tick.value === 0 ? 'rgba(128,128,128,0.5)' : grid; }
}
}
},
plugins: {
legend: { labels: { usePointStyle: true, boxWidth: 8, color: mutedColor() } },
tooltip: { callbacks: { label: function (item) { return fmtSigned(item.parsed.y) + ' ' + unit + ' per 5 min (avg of ' + hodCount[item.dataIndex] + ' measurements)'; } } }
}
}
});
}
/* ---------- records table ---------- */
function renderRecords(deltas, id, unit) {
const el = document.getElementById(id);
if (!el || !isVisible(el)) return;
if (deltas.length < 2) {
el.innerHTML = '<div class="chart-sub">Not enough data yet.</div>';
return;
}
const sorted = deltas.slice().sort(function (a, b) { return b.diff - a.diff; });
const gains = sorted.slice(0, 5).filter(function (d) { return d.diff > 0; });
const losses = sorted.slice().reverse().slice(0, 5).filter(function (d) { return d.diff < 0; }).reverse();
function rows(items, cls) {
return items.map(function (d) {
return '<tr><td>' + fmtDate(d.x) + '</td><td class="num ' + cls + '">' + fmtSigned(d.diff) + '</td><td class="num">' + fmtDur(d.dt) + '</td></tr>';
}).join('');
}
el.innerHTML =
'<table class="records">' +
'<tr><th>Time</th><th style="text-align:right">Change (' + unit + ')</th><th style="text-align:right">Span</th></tr>' +
rows(gains, 'pos') +
rows(losses, 'neg') +
'</table>';
}
/* ---------- heatmap ---------- */
function renderHeatmap(points, id, unit) {
const el = document.getElementById(id);
if (!el || !isVisible(el)) return;
if (!points.length) { el.innerHTML = ''; return; }
function localDayKey(d) {
return d.getFullYear() + '-' +
('0' + (d.getMonth() + 1)).slice(-2) + '-' +
('0' + d.getDate()).slice(-2);
}
const map = {};
points.forEach(function (p) {
const d = new Date(p.x);
map[localDayKey(d) + '|' + d.getHours()] = p.y;
});
const days = {};
Object.keys(map).forEach(function (k) { days[k.split('|')[0]] = true; });
const dayList = Object.keys(days).sort();
// deltas per day+hour for coloring
const cellDeltas = {};
let maxAbs = 0;
const sorted = points.slice().sort(function (a, b) { return a.x - b.x; });
const byDayHour = {};
sorted.forEach(function (p) {
const d = new Date(p.x);
const key = localDayKey(d) + '|' + d.getHours();
if (byDayHour[key] !== undefined) {
cellDeltas[key] = p.y - byDayHour[key];
maxAbs = Math.max(maxAbs, Math.abs(cellDeltas[key]));
}
byDayHour[key] = p.y;
});
let html = '<table><tr><th></th>';
for (let h = 0; h < 24; h++) html += '<th>' + h + '</th>';
html += '</tr>';
dayList.forEach(function (day) {
html += '<tr><td class="day">' + fmtDay(new Date(day + 'T12:00:00').getTime()) + '</td>';
for (let h = 0; h < 24; h++) {
const key = day + '|' + h;
const val = map[key];
let style = '';
let title = fmtDay(new Date(day + 'T12:00:00').getTime()) + ' ' + h + ':00 – no data';
if (val !== undefined && cellDeltas[key] !== undefined) {
const dv = cellDeltas[key];
const op = maxAbs ? Math.min(0.85, 0.15 + 0.7 * Math.abs(dv) / maxAbs) : 0.3;
const color = dv >= 0 ? '46,204,113' : '255,107,107';
style = ' style="background:rgba(' + color + ',' + op.toFixed(2) + ')"';
title = fmtDay(new Date(day + 'T12:00:00').getTime()) + ' ' + h + ':00 – ' + fmtSigned(dv) + ' ' + unit + ' (' + fmt(val) + ' total)';
}
html += '<td><div class="cell"' + style + ' title="' + title + '"></div></td>';
}
html += '</tr>';
});
html += '</table>';
el.innerHTML = html;
}
/* ---------- summary cards for images / queue ---------- */
// The pages tab keeps its static card grid (ids like stat-delta-6h);
// images and queue get the same key numbers generated from their series.
function renderSummary(containerId, all, points) {
const el = document.getElementById(containerId);
if (!el || !isVisible(el)) return;
if (!points.length) {
el.innerHTML = '<div class="chart-sub">No data in the selected range yet.</div>';
return;
}
const anchor = all[all.length - 1];
const first = points[0];
const hours = (anchor.x - first.x) / 3600000;
const perHour = hours > 0 ? (anchor.y - first.y) / hours : null;
const deltas = computeDeltas(points).deltas;
let peak = all[0], low = all[0];
all.forEach(function (p) {
if (p.y > peak.y) peak = p;
if (p.y < low.y) low = p;
});
const cards = [];
function signedCard(label, value, hint) {
const cls = value === null ? '' : (value >= 0 ? 'pos' : 'neg');
cards.push('<div class="card"><div class="label">' + label +
'</div><div class="value ' + cls + '">' +
(value === null ? '–' : fmtSigned(value)) +
'</div><div class="hint">' + hint + '</div></div>');
}
function plainCard(label, value, hint) {
cards.push('<div class="card"><div class="label">' + label +
'</div><div class="value">' + value +
'</div><div class="hint">' + hint + '</div></div>');
}
signedCard('Growth / hour', perHour, 'Average over full history');
signedCard('Growth / day', perHour === null ? null : perHour * 24, 'Average over full history');
const windows = [
['6 h', 6 * 3600000, Math.max(3 * intervalMs, 3600000)],
['24 h', 24 * 3600000, Math.max(6 * intervalMs, 3 * 3600000)],
['7 days', 7 * 24 * 3600000, Math.max(12 * intervalMs, 6 * 3600000)]
];
windows.forEach(function (win) {
const base = closestBefore(all, anchor.x - win[1], win[2]);
signedCard('Growth ' + win[0], base ? anchor.y - base.y : null,
base ? 'vs. measurement ' + fmtDur(anchor.x - base.x) + ' ago' : 'Not enough data yet');
});
let sum24 = 0;
const windowStart = Math.max(first.x, anchor.x - 24 * 3600000);
const coveredH = (anchor.x - windowStart) / 3600000;
deltas.forEach(function (d) {
if (d.x > anchor.x - 24 * 3600000) sum24 += d.diff;
});
signedCard('Avg / h (24 h)', coveredH >= 0.5 ? sum24 / coveredH : null, 'Mean of last 24 h changes');
plainCard('Peak', fmt(peak.y), 'On ' + fmtDateFull(peak.x));
plainCard('Lowest', fmt(low.y), 'On ' + fmtDateFull(low.x));
const regWindow = points.filter(function (p) { return p.x >= anchor.x - 2 * 3600000; }).slice(-96);
const reg = regress(regWindow);
if (reg) {
plainCard('Forecast next hour', fmt(anchor.y + reg.slope),
'Trend ' + fmtSigned(reg.slope) + '/h · R² = ' + reg.r2.toFixed(3));
} else {
plainCard('Forecast next hour', '–', 'Needs at least 2 measurements');
}
el.innerHTML = cards.join('');
}
/* ---------- main update ---------- */
function renderAll() {
const all = parsePoints(rawValues);
const points = filterRange(all);
const now = Date.now();
const elCurrent = document.getElementById('current');
const meta = document.getElementById('current-meta');
const elImages = document.getElementById('current-images');
const imagesMeta = document.getElementById('current-images-meta');
const elPending = document.getElementById('current-pending');
const pendingMeta = document.getElementById('current-pending-meta');
document.getElementById('badge-count').textContent = 'Measurements: ' + all.length;
if (!all.length) {
elCurrent.textContent = '–';
meta.textContent = 'No measurements yet – the collector stores one every 30 minutes.';
elImages.textContent = '–';
imagesMeta.textContent = 'No image count yet';
elPending.textContent = '–';
pendingMeta.textContent = 'No queue count yet';
setStatus('offline', 'No data');
return;
}
const last = all[all.length - 1];
const first = all[0];
const dts = [];
for (let i = 1; i < all.length; i++) dts.push(all[i].x - all[i - 1].x);
intervalMs = median(dts) || FALLBACK_INTERVAL_MS;
document.getElementById('badge-interval').textContent = 'Interval: ~' + fmtDur(intervalMs);
elCurrent.textContent = fmt(last.y);
meta.textContent = 'Last measurement: ' + fmtDateFull(last.x) +
' · next expected in ~' + fmtDur(Math.max(0, last.x + intervalMs - now));
if (last.images === null) {
elImages.textContent = '–';
imagesMeta.textContent = 'No image count yet';
} else {
elImages.textContent = fmt(last.images);
imagesMeta.textContent = (last.imagesExact
? 'Exact count' : 'Estimated · exact every 2 h') +
' · ' + fmtDateFull(last.x);
}
if (last.pending === null) {
elPending.textContent = '–';
pendingMeta.textContent = 'No queue count yet';
} else {
elPending.textContent = fmt(last.pending);
pendingMeta.textContent = 'Waiting in queue · exact count · ' + fmtDateFull(last.x);
}
/* tab badges: always show the latest value of each data type */
document.getElementById('tab-count-pages').textContent = fmt(last.y) + ' total';
document.getElementById('tab-count-images').textContent = last.images === null
? 'no data yet' : fmt(last.images) + ' total';
document.getElementById('tab-count-queue').textContent = last.pending === null
? 'no data yet' : fmt(last.pending) + ' pending';
document.getElementById('badge-since').textContent = 'First measurement: ' + fmtDateFull(first.x);
document.getElementById('badge-updated').textContent = 'Last update: ' + fmtDate(last.x);
const age = now - last.x;
if (age <= 3 * intervalMs) {
setStatus('ok', 'Live · ' + fmtDur(age) + ' ago');
} else if (age <= 24 * 3600000) {
setStatus('stale', 'Stale · last update ' + fmtDur(age) + ' ago');
} else {
setStatus('offline', 'No data for ' + fmtDur(age));
}
/* fixed-window cards (full history) */
const totalHours = (last.x - first.x) / 3600000;
const perHour = totalHours > 0 ? (last.y - first.y) / totalHours : null;
const elPh = document.getElementById('stat-per-hour');
const elPd = document.getElementById('stat-per-day');
if (perHour === null) {
elPh.textContent = '–';
elPd.textContent = '–';
} else {
setSigned('stat-per-hour', perHour);
setSigned('stat-per-day', perHour * 24);
}
const span6h = closestBefore(all, last.x - 6 * 3600000, Math.max(3 * intervalMs, 3600000));
setDeltaCard('stat-delta-6h', 'stat-delta-6h-hint',
span6h ? last.y - span6h.y : null,
span6h ? last.x - span6h.x : 0,
span6h ? (last.y - span6h.y) * (3600000 / Math.max(1, last.x - span6h.x)) * (5 / 60) : 0);
const span24h = closestBefore(all, last.x - 24 * 3600000, Math.max(6 * intervalMs, 3 * 3600000));
setDeltaCard('stat-delta-24h', 'stat-delta-24h-hint',
span24h ? last.y - span24h.y : null,
span24h ? last.x - span24h.x : 0,
span24h ? (last.y - span24h.y) * (3600000 / Math.max(1, last.x - span24h.x)) * (5 / 60) : 0);
const span7d = closestBefore(all, last.x - 7 * 24 * 3600000, Math.max(12 * intervalMs, 6 * 3600000));
setDeltaCard('stat-delta-7d', 'stat-delta-7d-hint',
span7d ? last.y - span7d.y : null,
span7d ? last.x - span7d.x : 0,
span7d ? (last.y - span7d.y) * (3600000 / Math.max(1, last.x - span7d.x)) * (5 / 60) : 0);
/* range-dependent cards + charts */
const fpoints = points.length >= 2 ? points : all;
const dd = computeDeltas(fpoints);
const deltas = dd.deltas;
const smoothY = movingAverage(deltas.map(function (d) { return d.y; }), DELTA_SMOOTH_WINDOW);
const smoothedDeltas = deltas.map(function (d, i) {
return { x: d.x, y: smoothY[i], raw: d.y, dt: d.dt, diff: d.diff };
});
/* avg/h over last 24 h: sum of all changes within 24 h ÷ hours actually covered */
let sum24 = 0;
let windowStart = Math.max(fpoints[0].x, last.x - 24 * 3600000);
let coveredH = (last.x - windowStart) / 3600000;
deltas.forEach(function (d) {
if (d.x > last.x - 24 * 3600000) sum24 += d.diff;
});
if (coveredH >= 0.5) {
setSigned('stat-avg-hour', sum24 / coveredH);
} else {
document.getElementById('stat-avg-hour').textContent = '–';
}
/* best / worst hour: best total gain within a sliding ~1 h window */
let best = null, worst = null;
for (let i = 0; i < deltas.length; i++) {
let gain = 0, endMs = deltas[i].x;
for (let j = i; j < deltas.length; j++) {
if (deltas[j].x - deltas[i].x > 3600000) break;
gain += deltas[j].diff;
endMs = deltas[j].x;
}
if (!best || gain > best.gain) best = { gain: gain, x: endMs };
if (!worst || gain < worst.gain) worst = { gain: gain, x: endMs };
}
if (best && best.gain !== 0) {
setSigned('stat-best-hour', best.gain);
document.getElementById('stat-best-hour-hint').textContent = 'Around ' + fmtDate(best.x);
}
if (worst && worst.gain !== 0) {
setSigned('stat-worst-hour', worst.gain);
document.getElementById('stat-worst-hour-hint').textContent = 'Around ' + fmtDate(worst.x);
}
/* peak / lowest (whole history) */
let peak = all[0], low = all[0];
all.forEach(function (p) {
if (p.y > peak.y) peak = p;
if (p.y < low.y) low = p;
});
document.getElementById('stat-peak').textContent = fmt(peak.y);
document.getElementById('stat-peak-hint').textContent = 'On ' + fmtDateFull(peak.x);
document.getElementById('stat-min').textContent = fmt(low.y);
document.getElementById('stat-min-hint').textContent = 'On ' + fmtDateFull(low.x);
/* forecast */
const regWindow = [];
for (let i = fpoints.length - 1; i >= 0; i--) {
regWindow.unshift(fpoints[i]);
if (fpoints.length - i >= 96) break;
if (fpoints[fpoints.length - 1].x - fpoints[i].x >= 2 * 3600000) break;
}
const reg = regress(regWindow);
if (reg) {
const elF = document.getElementById('stat-forecast');
const fc = fpoints[fpoints.length - 1].y + reg.slope;
elF.textContent = fmt(fc);
elF.classList.remove('pos', 'neg');
elF.classList.add(fc >= fpoints[fpoints.length - 1].y ? 'pos' : 'neg');
document.getElementById('stat-forecast-hint').textContent =
'Trend ' + fmtSigned(reg.slope) + '/h · R² = ' + reg.r2.toFixed(3);
} else {
document.getElementById('stat-forecast').textContent = '–';
document.getElementById('stat-forecast-hint').textContent = 'Needs at least 2 measurements';
}
document.getElementById('footer').textContent =
'Showing ' + METRIC_LABELS[activeMetric] +
' · Range: ' + (range === 'all' ? 'all data' : range) +
' · All statistics are computed in the browser.';
/* queue: same graphs for the pending URLs in crawl_queue */
const pendingAll = toPendingPoints(all);
const pendingFpoints = filterRange(pendingAll, last.x);
const pendingDeltas = computeDeltas(pendingFpoints).deltas;
const pendingSmoothY = movingAverage(pendingDeltas.map(function (d) { return d.y; }), DELTA_SMOOTH_WINDOW);
const pendingSmoothedDeltas = pendingDeltas.map(function (d, i) {
return { x: d.x, y: pendingSmoothY[i], raw: d.y, dt: d.dt, diff: d.diff };
});
const pendingEmpty = document.getElementById('pending-empty');
if (!pendingAll.length) {
pendingEmpty.textContent = all.some(function (p) { return p.pending !== null; })
? 'No queue counts in the selected range.'
: 'No queue counts yet – they appear after the next collector run.';
pendingEmpty.style.display = '';
} else if (!pendingFpoints.length) {
pendingEmpty.textContent = 'No queue counts in the selected range.';
pendingEmpty.style.display = '';
} else {
pendingEmpty.style.display = 'none';
}
/* Images: estimates are optional and controlled by the Images-section toggle. */
const imageAll = toImagePoints(all, showEstimatedImages);
document.getElementById('images-mode-note').textContent = showEstimatedImages
? 'Exact and estimated measurements are plotted. Estimates come from a 1% table sample; this setting is saved in this browser.'
: 'Only exact measurements (counted every 2 h) are plotted. This setting is saved in this browser.';
const imageFpoints = filterRange(imageAll, last.x);
const imageDeltas = computeDeltas(imageFpoints).deltas;
const imageSmoothY = movingAverage(imageDeltas.map(function (d) { return d.y; }), DELTA_SMOOTH_WINDOW);
const imageSmoothedDeltas = imageDeltas.map(function (d, i) {
return { x: d.x, y: imageSmoothY[i], raw: d.y, dt: d.dt, diff: d.diff };
});
const imagesEmpty = document.getElementById('images-empty');
const hasImageValues = all.some(function (p) { return p.images !== null; });
if (!imageAll.length) {
imagesEmpty.textContent = hasImageValues
? 'Only estimated image counts so far – turn on “Include estimated image counts in graphs” to show them.'
: 'No image counts yet – they appear after the next collector run with the image migration in place.';
imagesEmpty.style.display = '';
} else if (!imageFpoints.length) {
imagesEmpty.textContent = 'No image counts in the selected range.';
imagesEmpty.style.display = '';
} else {
imagesEmpty.style.display = 'none';
}
destroyCharts();
renderCharts({
prefix: '',
points: fpoints,
deltas: deltas,
smoothedDeltas: smoothedDeltas,
unit: 'Pages',
accent: ACCENT,
accentBar: 'rgba(79,140,255,0.65)',
gradTop: 'rgba(79,140,255,0.28)',
gradBottom: 'rgba(79,140,255,0.02)'
});
renderCharts({
prefix: 'images',
points: imageFpoints,
deltas: imageDeltas,
smoothedDeltas: imageSmoothedDeltas,
unit: 'Images',
accent: ACCENT2,
accentBar: 'rgba(180,122,255,0.65)',
gradTop: 'rgba(180,122,255,0.28)',
gradBottom: 'rgba(180,122,255,0.02)'
});
renderCharts({
prefix: 'pending',
points: pendingFpoints,
deltas: pendingDeltas,
smoothedDeltas: pendingSmoothedDeltas,
unit: 'Pending',
accent: ACCENT3,
accentBar: 'rgba(245,166,35,0.65)',
gradTop: 'rgba(245,166,35,0.28)',
gradBottom: 'rgba(245,166,35,0.02)'
});
renderRecords(deltas, 'records', 'Pages');
renderRecords(imageDeltas, 'records-images', 'Images');
renderRecords(pendingDeltas, 'records-pending', 'Pending');
renderHeatmap(all, 'heatmap', 'Pages');
renderHeatmap(imageAll, 'heatmap-images', 'Images');
renderHeatmap(pendingAll, 'heatmap-pending', 'Pending');
renderSummary('summary-images', imageAll, imageFpoints);
renderSummary('summary-queue', pendingAll, pendingFpoints);
}
async function update() {
refreshLeft = REFRESH_S;
try {
const response = await fetch('/api/data');
rawValues = (await response.json()).values || [];
} catch (e) {
setStatus('offline', 'Fetch error – retrying…');
return;
}
renderAll();
}
setInterval(function () {
refreshLeft -= 1;
if (refreshLeft <= 0) {
document.getElementById('badge-refresh').textContent = 'Refreshing…';
} else {
document.getElementById('badge-refresh').textContent = 'Next refresh: ' + refreshLeft + ' s';
}
}, 1000);
update();
setInterval(update, REFRESH_S * 1000);
// Restore the last selected tab (pages / images / queue).
(function () {
let saved = 'pages';
try { saved = localStorage.getItem('js-metric') || 'pages'; } catch (e) {}
setMetric(saved, false);
})();
</script>
</body>
</html>
"""
def load_data():
if not DB_DATA.exists():
return []
try:
with DB_DATA.open("r", encoding="utf-8") as f:
return json.load(f)
except Exception:
return []
def save_data(data):
tmp = DB_DATA.with_suffix(".tmp")
with tmp.open("w", encoding="utf-8") as f:
json.dump(data, f, indent=2)
tmp.replace(DB_DATA)
# Exakt: ein Scan, der COUNT(*) und die Bilder in einem Durchgang
# ermittelt. Wird nur alle EXACT_IMAGE_INTERVAL Sekunden ausgeführt.
EXACT_IMAGE_SQL = (
"SELECT COUNT(*), "
"COALESCE(SUM(cardinality(pictures)), 0)::bigint "
"FROM pages;"
)
# Schätzung: COUNT(*) wie bisher, Bilder aus 1 % der Tabellenblöcke
# (TABLESAMPLE SYSTEM), hochgerechnet. Die zweite Spalte ist NULL, wenn
# kein Block ausgewählt wurde (sehr kleine Tabelle).
ESTIMATE_IMAGE_SQL = (
"SELECT (SELECT COUNT(*) FROM pages), "
"(SELECT (SUM(cardinality(pictures)) * {scale})::bigint "
"FROM pages TABLESAMPLE SYSTEM ({percent}));"
).format(
scale=100 // IMAGE_SAMPLE_PERCENT,
percent=IMAGE_SAMPLE_PERCENT,
)
# URLs waiting in the crawler queue. Nutzt den Index
# (status, priority, discovered_at); kein Sampling, der Wert ist exakt.
PENDING_QUEUE_SQL = "SELECT COUNT(*) FROM crawl_queue WHERE status = 'pending';"
def _psql_scalar(sql):
"""Eine einzelne psql-Zeile ("count|images") abholen."""
command = [
"sudo",
"-u",
"postgres",
"psql",
"-p",
"25432",
"-d",
"jonassearch",
"-t",
"-A",
"-F",
"|",
"-c",
sql,
]
result = subprocess.run(
command,
capture_output=True,
text=True,
check=True
)
return result.stdout.strip()
def _parse_stats_row(output):
""""count|images" -> (count, images|None)."""
count_raw, _sep, images_raw = (output or "").strip().partition("|")
count = int(count_raw.strip())
images = int(images_raw.strip()) if images_raw.strip() else None
return count, images
def images_exact_due(data, now=None):
"""
True, wenn der letzte exakte Bild-Count fehlt oder mindestens
EXACT_IMAGE_INTERVAL alt ist. Einträge aus der Zeit vor dieser
Funktion (ohne images_exact) zählen nicht als exakt.
"""
now = now or datetime.now()
for entry in reversed(data or []):
if entry.get("images_exact") is not True:
continue
try:
timestamp = datetime.fromisoformat(str(entry.get("timestamp", "")))
except (TypeError, ValueError):
continue
return now - timestamp >= timedelta(seconds=EXACT_IMAGE_INTERVAL)
return True
def get_stats(exact_images):
"""
Seitenzahl + Bildanzahl holen.
exact_images=True: ein Scan, exakter Bild-Count.
exact_images=False: Seitenzahl wie bisher, Bilder aus einer
1-%-Stichprobe hochgerechnet. Liefert die Stichprobe nichts (kleine
Tabelle), wird exakt gezählt - das ist dort billig.
Rückgabe: (pages, images, images_exact). images kann None sein,
wenn eine große Tabelle wider Erwarten keinen Block gesampelt hat.
"""
try:
if exact_images:
count, images = _parse_stats_row(_psql_scalar(EXACT_IMAGE_SQL))
return count, images, True
count, images = _parse_stats_row(_psql_scalar(ESTIMATE_IMAGE_SQL))
except subprocess.CalledProcessError as exc:
# Z. B. noch keine Migration (pages.pictures fehlt): die
# Seitenzahl soll trotzdem weiter erfasst werden.
detail = ((exc.stderr or "").strip().splitlines() or [str(exc)])[-1]
print(f"Bild-Statistik nicht verfügbar ({detail}); zähle nur Seiten.")
return int(_psql_scalar("SELECT COUNT(*) FROM pages;").strip()), None, False
if images is None:
if count < SAMPLE_EXACT_FALLBACK_ROWS:
return get_stats(True)
return count, None, False
return count, images, False
def get_pending():
"""
Anzahl der URLs mit status='pending' in der crawl_queue.
Ist die Tabelle (noch) nicht erreichbar, wird None geliefert - die
Messung wird trotzdem gespeichert, nur ohne Queue-Wert.
"""
try:
return int(_psql_scalar(PENDING_QUEUE_SQL).strip())
except subprocess.CalledProcessError as exc:
detail = ((exc.stderr or "").strip().splitlines() or [str(exc)])[-1]
print(f"Queue-Statistik nicht verfügbar ({detail}); zähle nur Seiten/Bilder.")
return None
except (TypeError, ValueError):
# Leere/überraschende psql-Ausgabe darf die übrigen Werte nicht kosten.
print("Queue-Statistik nicht verfügbar (unerwartete psql-Ausgabe).")
return None
def collect():
while True:
try:
data = load_data()
exact = images_exact_due(data)
count, images, images_exact = get_stats(exact)
pending = get_pending()
if images is None:
# Stichprobe ohne Treffer (praktisch nur bei sehr großen
# Tabellen): letzten bekannten Wert fortschreiben.
for previous in reversed(data):
if previous.get("images") is not None:
images = previous["images"]
break
entry = {
"timestamp": datetime.now().isoformat(),
"count": count,
"images": images,
"images_exact": images_exact,
"pending": pending,
}
data.append(entry)
save_data(data)
print(
f"[{entry['timestamp']}] "
f"Pages: {count} · Images: {images} "
f"({'exakt' if images_exact else 'geschätzt'}) "
f"· Pending: {pending if pending is not None else 'n/a'}"
)
except Exception as e:
print(f"Fehler beim Abrufen der Statistik: {e}")
time.sleep(INTERVAL)
@app.route("/")
def index():
return render_template_string(HTML)
@app.route("/api/data")
def api_data():
return jsonify({
"values": load_data()
})
@app.route("/api/export")
def api_export():
lines = ["timestamp;count"]
for entry in load_data():
lines.append(f"{entry.get('timestamp', '')};{entry.get('count', '')}")
return Response("\n".join(lines), mimetype="text/csv")
if __name__ == "__main__":
# Collector im Hintergrund starten
thread = threading.Thread(target=collect, daemon=True)
thread.start()
# Webserver
app.run(
host="0.0.0.0",
port=8081,
threaded=True,
load_dotenv=False # sonst liest Flask beim Start .env im Projektordner -> PermissionError
)