#!/usr/bin/env python3 import subprocess import json import time import threading from datetime import datetime from pathlib import Path from flask import Flask, jsonify, render_template_string, Response DB_DATA = Path(__file__).parent / "page_counts.json" INTERVAL = 15 * 60 # 15 Minuten app = Flask(__name__) # r""" raw string: preserves \n etc. for the embedded JavaScript # (a normal string would eat the backslashes and break the JS) HTML = r""" JonasSearch – Pages

JonasSearch

Indexed Pages – Live Statistics
Export CSV Export JSON
–Pages
Loading…
Loading… Interval: – First measurement: – Measurements: – Last update: – Next refresh: –
Growth / hour
–
Average over full history
Growth / day
–
Average over full history
Growth 24 h
–
vs. measurement ~24 h ago
Growth 7 days
–
vs. measurement ~7 d ago
Avg / h (24 h)
–
Mean of last 24 h changes
Best hour
–
Largest gain within one hour
Worst hour
–
Largest loss within one hour
Peak
–
Highest measurement ever
Lowest
–
Lowest measurement ever
Forecast next hour
–
Linear trend extrapolation
Indexed pages over time
Real measurement timestamps · dashed line = linear-regression trend & 24 h forecast
Change per 5 minutes
Normalized to a 5-minute interval · grey = raw · colored = smoothed (moving average over 5 measurements)
Cumulative net growth
Total pages gained/lost since the start of the selected range · linear time axis
Growth per hour (last 48 h)
Sum of changes per hour bucket · green = growth, red = loss
Growth by hour of day
Average gain per 5 min, grouped by hour of day (whole history) · shows when the crawler is most active
Records – biggest changes
Largest gains and losses between two consecutive measurements (selected range)
Activity heatmap – gain per hour of day
Rows = days · columns = hours (0–23) · green = growth, red = loss, grey = no data
""" def load_data(): if not DB_DATA.exists(): return [] try: with DB_DATA.open("r", encoding="utf-8") as f: return json.load(f) except Exception: return [] def save_data(data): tmp = DB_DATA.with_suffix(".tmp") with tmp.open("w", encoding="utf-8") as f: json.dump(data, f, indent=2) tmp.replace(DB_DATA) def get_page_count(): command = [ "sudo", "-u", "postgres", "psql", "-p", "25432", "-d", "jonassearch", "-t", "-A", "-c", "SELECT COUNT(*) FROM pages;" ] result = subprocess.run( command, capture_output=True, text=True, check=True ) return int(result.stdout.strip()) def collect(): while True: try: count = get_page_count() entry = { "timestamp": datetime.now().isoformat(), "count": count } data = load_data() data.append(entry) save_data(data) print( f"[{entry['timestamp']}] " f"Pages: {count}" ) except Exception as e: print(f"Fehler beim Abrufen der Seitenzahl: {e}") time.sleep(INTERVAL) @app.route("/") def index(): return render_template_string(HTML) @app.route("/api/data") def api_data(): return jsonify({ "values": load_data() }) @app.route("/api/export") def api_export(): lines = ["timestamp;count"] for entry in load_data(): lines.append(f"{entry.get('timestamp', '')};{entry.get('count', '')}") return Response("\n".join(lines), mimetype="text/csv") if __name__ == "__main__": # Collector im Hintergrund starten thread = threading.Thread(target=collect, daemon=True) thread.start() # Webserver app.run( host="0.0.0.0", port=8081, threaded=True, load_dotenv=False # sonst liest Flask beim Start .env im Projektordner -> PermissionError )