#!/usr/bin/env python3
import subprocess
import json
import time
import threading
from datetime import datetime, timedelta
from pathlib import Path
from flask import Flask, jsonify, render_template_string, Response
DB_DATA = Path(__file__).parent / "page_counts.json"
INTERVAL = 30 * 60 # 30 Minuten
# Bilder werden nur alle 2 Stunden exakt gezählt (SUM über die ~10-GB-
# Tabelle). Dazwischen wird eine 1-%-Stichprobe (TABLESAMPLE SYSTEM)
# hochgerechnet - das kostet nur einen Bruchteil der IO.
EXACT_IMAGE_INTERVAL = 2 * 60 * 60 # Sekunden
IMAGE_SAMPLE_PERCENT = 1
# Sehr kleine Tabellen: Stichprobe kann leer sein; dort ist der exakte
# Count ohnehin billig.
SAMPLE_EXACT_FALLBACK_ROWS = 100_000
app = Flask(__name__)
# r""" raw string: preserves \n etc. for the embedded JavaScript
# (a normal string would eat the backslashes and break the JS)
HTML = r"""
JonasSearch – Pages
Real measurement timestamps · dashed line = linear-regression trend & 24 h forecast
Change per 5 minutes
Normalized to a 5-minute interval · grey = raw · colored = smoothed (moving average over 5 measurements)
Cumulative net growth
Total pages gained/lost since the start of the selected range · linear time axis
Growth per hour (last 48 h)
Sum of changes per hour bucket · green = growth, red = loss
Growth by hour of day
Average gain per 5 min, grouped by hour of day (whole history) · shows when the crawler is most active
Records – biggest changes
Largest gains and losses between two consecutive measurements (selected range)
Activity heatmap – gain per hour of day
Rows = days · columns = hours (0–23) · green = growth, red = loss, grey = no data
–Images
Loading…
Images – all graphs
Only exact measurements (counted every 2 h) are plotted. This setting is saved in this browser.
No image counts yet – they appear after the next collector run with the image migration in place.
Images over time
Image URLs found while crawling · dashed line = trend & 24 h forecast
Change per 5 minutes
Normalized to a 5-minute interval between available image measurements · grey = raw · colored = smoothed (moving average over 5 measurements)
Cumulative net growth
Total images gained/lost since the start of the selected range · linear time axis
Images per hour (last 48 h)
Sum of changes per hour bucket · green = growth, red = loss
Image growth by hour of day
Average gain per 5 min, grouped by hour of day (whole history)
Records – biggest image changes
Largest gains and losses between two consecutive measurements (selected range)
Activity heatmap – image gain per hour of day
Rows = days · columns = hours (0–23) · green = growth, red = loss, grey = no data
–Pending
Loading…
Queue – all graphs
URLs waiting in crawl_queue (status = 'pending'). Counted with every 30-minute sample (exact COUNT). Falling graphs mean the crawler is working through the queue.
No queue counts yet – they appear after the next collector run.
Pending URLs over time
Queued URLs not yet crawled · dashed line = trend & 24 h forecast
Change per 5 minutes
Normalized to a 5-minute interval · negative = the crawler worked the queue down · grey = raw · colored = smoothed (moving average over 5 measurements)
Cumulative net change
Total queued URLs gained/lost since the start of the selected range · linear time axis
Queue change per hour (last 48 h)
Sum of changes per hour bucket · green = queue grew, red = queue shrank
Queue change by hour of day
Average change per 5 min, grouped by hour of day (whole history)
Records – biggest queue changes
Largest gains and losses between two consecutive measurements (selected range)
Activity heatmap – queue change per hour of day
Rows = days · columns = hours (0–23) · green = queue grew, red = queue shrank, grey = no data
"""
def load_data():
if not DB_DATA.exists():
return []
try:
with DB_DATA.open("r", encoding="utf-8") as f:
return json.load(f)
except Exception:
return []
def save_data(data):
tmp = DB_DATA.with_suffix(".tmp")
with tmp.open("w", encoding="utf-8") as f:
json.dump(data, f, indent=2)
tmp.replace(DB_DATA)
# Exakt: ein Scan, der COUNT(*) und die Bilder in einem Durchgang
# ermittelt. Wird nur alle EXACT_IMAGE_INTERVAL Sekunden ausgeführt.
EXACT_IMAGE_SQL = (
"SELECT COUNT(*), "
"COALESCE(SUM(cardinality(pictures)), 0)::bigint "
"FROM pages;"
)
# Schätzung: COUNT(*) wie bisher, Bilder aus 1 % der Tabellenblöcke
# (TABLESAMPLE SYSTEM), hochgerechnet. Die zweite Spalte ist NULL, wenn
# kein Block ausgewählt wurde (sehr kleine Tabelle).
ESTIMATE_IMAGE_SQL = (
"SELECT (SELECT COUNT(*) FROM pages), "
"(SELECT (SUM(cardinality(pictures)) * {scale})::bigint "
"FROM pages TABLESAMPLE SYSTEM ({percent}));"
).format(
scale=100 // IMAGE_SAMPLE_PERCENT,
percent=IMAGE_SAMPLE_PERCENT,
)
# URLs waiting in the crawler queue. Nutzt den Index
# (status, priority, discovered_at); kein Sampling, der Wert ist exakt.
PENDING_QUEUE_SQL = "SELECT COUNT(*) FROM crawl_queue WHERE status = 'pending';"
def _psql_scalar(sql):
"""Eine einzelne psql-Zeile ("count|images") abholen."""
command = [
"sudo",
"-u",
"postgres",
"psql",
"-p",
"25432",
"-d",
"jonassearch",
"-t",
"-A",
"-F",
"|",
"-c",
sql,
]
result = subprocess.run(
command,
capture_output=True,
text=True,
check=True
)
return result.stdout.strip()
def _parse_stats_row(output):
""""count|images" -> (count, images|None)."""
count_raw, _sep, images_raw = (output or "").strip().partition("|")
count = int(count_raw.strip())
images = int(images_raw.strip()) if images_raw.strip() else None
return count, images
def images_exact_due(data, now=None):
"""
True, wenn der letzte exakte Bild-Count fehlt oder mindestens
EXACT_IMAGE_INTERVAL alt ist. Einträge aus der Zeit vor dieser
Funktion (ohne images_exact) zählen nicht als exakt.
"""
now = now or datetime.now()
for entry in reversed(data or []):
if entry.get("images_exact") is not True:
continue
try:
timestamp = datetime.fromisoformat(str(entry.get("timestamp", "")))
except (TypeError, ValueError):
continue
return now - timestamp >= timedelta(seconds=EXACT_IMAGE_INTERVAL)
return True
def get_stats(exact_images):
"""
Seitenzahl + Bildanzahl holen.
exact_images=True: ein Scan, exakter Bild-Count.
exact_images=False: Seitenzahl wie bisher, Bilder aus einer
1-%-Stichprobe hochgerechnet. Liefert die Stichprobe nichts (kleine
Tabelle), wird exakt gezählt - das ist dort billig.
Rückgabe: (pages, images, images_exact). images kann None sein,
wenn eine große Tabelle wider Erwarten keinen Block gesampelt hat.
"""
try:
if exact_images:
count, images = _parse_stats_row(_psql_scalar(EXACT_IMAGE_SQL))
return count, images, True
count, images = _parse_stats_row(_psql_scalar(ESTIMATE_IMAGE_SQL))
except subprocess.CalledProcessError as exc:
# Z. B. noch keine Migration (pages.pictures fehlt): die
# Seitenzahl soll trotzdem weiter erfasst werden.
detail = ((exc.stderr or "").strip().splitlines() or [str(exc)])[-1]
print(f"Bild-Statistik nicht verfügbar ({detail}); zähle nur Seiten.")
return int(_psql_scalar("SELECT COUNT(*) FROM pages;").strip()), None, False
if images is None:
if count < SAMPLE_EXACT_FALLBACK_ROWS:
return get_stats(True)
return count, None, False
return count, images, False
def get_pending():
"""
Anzahl der URLs mit status='pending' in der crawl_queue.
Ist die Tabelle (noch) nicht erreichbar, wird None geliefert - die
Messung wird trotzdem gespeichert, nur ohne Queue-Wert.
"""
try:
return int(_psql_scalar(PENDING_QUEUE_SQL).strip())
except subprocess.CalledProcessError as exc:
detail = ((exc.stderr or "").strip().splitlines() or [str(exc)])[-1]
print(f"Queue-Statistik nicht verfügbar ({detail}); zähle nur Seiten/Bilder.")
return None
except (TypeError, ValueError):
# Leere/überraschende psql-Ausgabe darf die übrigen Werte nicht kosten.
print("Queue-Statistik nicht verfügbar (unerwartete psql-Ausgabe).")
return None
def collect():
while True:
try:
data = load_data()
exact = images_exact_due(data)
count, images, images_exact = get_stats(exact)
pending = get_pending()
if images is None:
# Stichprobe ohne Treffer (praktisch nur bei sehr großen
# Tabellen): letzten bekannten Wert fortschreiben.
for previous in reversed(data):
if previous.get("images") is not None:
images = previous["images"]
break
entry = {
"timestamp": datetime.now().isoformat(),
"count": count,
"images": images,
"images_exact": images_exact,
"pending": pending,
}
data.append(entry)
save_data(data)
print(
f"[{entry['timestamp']}] "
f"Pages: {count} · Images: {images} "
f"({'exakt' if images_exact else 'geschätzt'}) "
f"· Pending: {pending if pending is not None else 'n/a'}"
)
except Exception as e:
print(f"Fehler beim Abrufen der Statistik: {e}")
time.sleep(INTERVAL)
@app.route("/")
def index():
return render_template_string(HTML)
@app.route("/api/data")
def api_data():
return jsonify({
"values": load_data()
})
@app.route("/api/export")
def api_export():
lines = ["timestamp;count"]
for entry in load_data():
lines.append(f"{entry.get('timestamp', '')};{entry.get('count', '')}")
return Response("\n".join(lines), mimetype="text/csv")
if __name__ == "__main__":
# Collector im Hintergrund starten
thread = threading.Thread(target=collect, daemon=True)
thread.start()
# Webserver
app.run(
host="0.0.0.0",
port=8081,
threaded=True,
load_dotenv=False # sonst liest Flask beim Start .env im Projektordner -> PermissionError
)