From ca5af75bbd27632a775c5e80a96b66df4d9982ec Mon Sep 17 00:00:00 2001 From: jhkaktus Date: Sat, 17 Oct 2026 21:42:06 +0200 Subject: [PATCH] make stats count images --- web/app.py | 170 ++++++++++++++++++++++++++++++++++++++++++---- web/test_stats.py | 134 ++++++++++++++++++++++++++++++++++++ 2 files changed, 289 insertions(+), 15 deletions(-) create mode 100644 web/test_stats.py diff --git a/web/app.py b/web/app.py index 0c88876..473fc43 100644 --- a/web/app.py +++ b/web/app.py @@ -4,7 +4,7 @@ import subprocess import json import time import threading -from datetime import datetime +from datetime import datetime, timedelta from pathlib import Path from flask import Flask, jsonify, render_template_string, Response @@ -12,6 +12,15 @@ from flask import Flask, jsonify, render_template_string, Response DB_DATA = Path(__file__).parent / "page_counts.json" INTERVAL = 30 * 60 # 30 Minuten +# Bilder werden nur alle 2 Stunden exakt gezählt (SUM über die ~10-GB- +# Tabelle). Dazwischen wird eine 1-%-Stichprobe (TABLESAMPLE SYSTEM) +# hochgerechnet - das kostet nur einen Bruchteil der IO. +EXACT_IMAGE_INTERVAL = 2 * 60 * 60 # Sekunden +IMAGE_SAMPLE_PERCENT = 1 +# Sehr kleine Tabellen: Stichprobe kann leer sein; dort ist der exakte +# Count ohnehin billig. +SAMPLE_EXACT_FALLBACK_ROWS = 100_000 + app = Flask(__name__) # r""" raw string: preserves \n etc. for the embedded JavaScript @@ -109,6 +118,8 @@ HTML = r""" .current-meta { color: var(--muted); margin-bottom: 24px; } + .current-row { display: flex; flex-wrap: wrap; gap: 0 56px; } + .badges { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 14px; } .badge { @@ -263,10 +274,20 @@ HTML = r""" -
- –Pages +
+
+
+ –Pages +
+
Loading…
+
+
+
+ –Images +
+
Loading…
+
-
Loading…
Loading… @@ -524,7 +545,16 @@ function setSigned(id, value) { function parsePoints(values) { return (values || []) - .map(function (v) { return { x: new Date(v.timestamp).getTime(), y: v.count }; }) + .map(function (v) { + const images = (v.images === null || v.images === undefined) + ? null : Number(v.images); + return { + x: new Date(v.timestamp).getTime(), + y: v.count, + images: Number.isFinite(images) ? images : null, + imagesExact: v.images_exact === true + }; + }) .filter(function (p) { return !isNaN(p.x); }) .sort(function (a, b) { return a.x - b.x; }); } @@ -1032,12 +1062,16 @@ function renderAll() { const now = Date.now(); const elCurrent = document.getElementById('current'); const meta = document.getElementById('current-meta'); + const elImages = document.getElementById('current-images'); + const imagesMeta = document.getElementById('current-images-meta'); document.getElementById('badge-count').textContent = 'Measurements: ' + all.length; if (!all.length) { elCurrent.textContent = '–'; - meta.textContent = 'No measurements yet – the collector stores one every 15 minutes.'; + meta.textContent = 'No measurements yet – the collector stores one every 30 minutes.'; + elImages.textContent = '–'; + imagesMeta.textContent = 'No image count yet'; setStatus('offline', 'No data'); return; } @@ -1054,6 +1088,16 @@ function renderAll() { meta.textContent = 'Last measurement: ' + fmtDateFull(last.x) + ' · next expected in ~' + fmtDur(Math.max(0, last.x + intervalMs - now)); + if (last.images === null) { + elImages.textContent = '–'; + imagesMeta.textContent = 'No image count yet'; + } else { + elImages.textContent = fmt(last.images); + imagesMeta.textContent = (last.imagesExact + ? 'Exact count' : 'Estimated · exact every 2 h') + + ' · ' + fmtDateFull(last.x); + } + document.getElementById('badge-since').textContent = 'First measurement: ' + fmtDateFull(first.x); document.getElementById('badge-updated').textContent = 'Last update: ' + fmtDate(last.x); @@ -1228,7 +1272,29 @@ def save_data(data): tmp.replace(DB_DATA) -def get_page_count(): +# Exakt: ein Scan, der COUNT(*) und die Bilder in einem Durchgang +# ermittelt. Wird nur alle EXACT_IMAGE_INTERVAL Sekunden ausgeführt. +EXACT_IMAGE_SQL = ( + "SELECT COUNT(*), " + "COALESCE(SUM(cardinality(pictures)), 0)::bigint " + "FROM pages;" +) + +# Schätzung: COUNT(*) wie bisher, Bilder aus 1 % der Tabellenblöcke +# (TABLESAMPLE SYSTEM), hochgerechnet. Die zweite Spalte ist NULL, wenn +# kein Block ausgewählt wurde (sehr kleine Tabelle). +ESTIMATE_IMAGE_SQL = ( + "SELECT (SELECT COUNT(*) FROM pages), " + "(SELECT (SUM(cardinality(pictures)) * {scale})::bigint " + "FROM pages TABLESAMPLE SYSTEM ({percent}));" +).format( + scale=100 // IMAGE_SAMPLE_PERCENT, + percent=IMAGE_SAMPLE_PERCENT, +) + + +def _psql_scalar(sql): + """Eine einzelne psql-Zeile ("count|images") abholen.""" command = [ "sudo", "-u", @@ -1240,8 +1306,10 @@ def get_page_count(): "jonassearch", "-t", "-A", + "-F", + "|", "-c", - "SELECT COUNT(*) FROM pages;" + sql, ] result = subprocess.run( @@ -1251,31 +1319,103 @@ def get_page_count(): check=True ) - return int(result.stdout.strip()) + return result.stdout.strip() + + +def _parse_stats_row(output): + """"count|images" -> (count, images|None).""" + count_raw, _sep, images_raw = (output or "").strip().partition("|") + count = int(count_raw.strip()) + images = int(images_raw.strip()) if images_raw.strip() else None + return count, images + + +def images_exact_due(data, now=None): + """ + True, wenn der letzte exakte Bild-Count fehlt oder mindestens + EXACT_IMAGE_INTERVAL alt ist. Einträge aus der Zeit vor dieser + Funktion (ohne images_exact) zählen nicht als exakt. + """ + now = now or datetime.now() + + for entry in reversed(data or []): + if entry.get("images_exact") is not True: + continue + try: + timestamp = datetime.fromisoformat(str(entry.get("timestamp", ""))) + except (TypeError, ValueError): + continue + return now - timestamp >= timedelta(seconds=EXACT_IMAGE_INTERVAL) + + return True + + +def get_stats(exact_images): + """ + Seitenzahl + Bildanzahl holen. + + exact_images=True: ein Scan, exakter Bild-Count. + exact_images=False: Seitenzahl wie bisher, Bilder aus einer + 1-%-Stichprobe hochgerechnet. Liefert die Stichprobe nichts (kleine + Tabelle), wird exakt gezählt - das ist dort billig. + + Rückgabe: (pages, images, images_exact). images kann None sein, + wenn eine große Tabelle wider Erwarten keinen Block gesampelt hat. + """ + try: + if exact_images: + count, images = _parse_stats_row(_psql_scalar(EXACT_IMAGE_SQL)) + return count, images, True + + count, images = _parse_stats_row(_psql_scalar(ESTIMATE_IMAGE_SQL)) + except subprocess.CalledProcessError as exc: + # Z. B. noch keine Migration (pages.pictures fehlt): die + # Seitenzahl soll trotzdem weiter erfasst werden. + detail = ((exc.stderr or "").strip().splitlines() or [str(exc)])[-1] + print(f"Bild-Statistik nicht verfügbar ({detail}); zähle nur Seiten.") + return int(_psql_scalar("SELECT COUNT(*) FROM pages;").strip()), None, False + + if images is None: + if count < SAMPLE_EXACT_FALLBACK_ROWS: + return get_stats(True) + return count, None, False + + return count, images, False def collect(): while True: try: - count = get_page_count() + data = load_data() + exact = images_exact_due(data) + count, images, images_exact = get_stats(exact) + + if images is None: + # Stichprobe ohne Treffer (praktisch nur bei sehr großen + # Tabellen): letzten bekannten Wert fortschreiben. + for previous in reversed(data): + if previous.get("images") is not None: + images = previous["images"] + break entry = { "timestamp": datetime.now().isoformat(), - "count": count + "count": count, + "images": images, + "images_exact": images_exact, } - data = load_data() data.append(entry) - save_data(data) print( f"[{entry['timestamp']}] " - f"Pages: {count}" + f"Pages: {count} · Images: {images} " + f"({'exakt' if images_exact else 'geschätzt'})" ) except Exception as e: - print(f"Fehler beim Abrufen der Seitenzahl: {e}") + print(f"Fehler beim Abrufen der Statistik: {e}") time.sleep(INTERVAL) diff --git a/web/test_stats.py b/web/test_stats.py new file mode 100644 index 0000000..813987a --- /dev/null +++ b/web/test_stats.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +""" +Tests für den Statserver (web/app.py), ohne Datenbank. + +Geprüft werden die DB-freien Teile: Parsen der psql-Zeile, der +2-Stunden-Rhythmus für den exakten Bild-Count, die Wahl zwischen +exakter Zählung und Stichprobe sowie die Einbindung der neuen +Images-Anzeige in die HTML-Seite. +""" + +import importlib.util +import os +import subprocess +import unittest +from datetime import datetime, timedelta +from unittest import mock + +HERE = os.path.dirname(os.path.abspath(__file__)) + +# web/app.py unter einem eindeutigen Modulnamen laden, damit es nicht mit +# priorityweb/app.py (gleicher Dateiname) kollidiert. +_spec = importlib.util.spec_from_file_location( + "jonassearch_stats_app", os.path.join(HERE, "app.py") +) +stats_app = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(stats_app) + + +class ParseStatsRowTest(unittest.TestCase): + + def test_count_and_images(self): + self.assertEqual( + stats_app._parse_stats_row("981981|12345"), (981981, 12345) + ) + + def test_empty_sample_is_none(self): + self.assertEqual(stats_app._parse_stats_row("10|"), (10, None)) + self.assertEqual(stats_app._parse_stats_row("10|\n"), (10, None)) + + +class ImagesExactDueTest(unittest.TestCase): + + NOW = datetime(2026, 10, 17, 12, 0, 0) + + def _entry(self, age, exact): + return { + "timestamp": (self.NOW - age).isoformat(), + "count": 1, + "images": 2, + "images_exact": exact, + } + + def test_no_data_means_due(self): + self.assertTrue(stats_app.images_exact_due([], now=self.NOW)) + + def test_recent_exact_is_not_due(self): + data = [self._entry(timedelta(hours=1), True)] + self.assertFalse(stats_app.images_exact_due(data, now=self.NOW)) + + def test_two_hours_old_is_due(self): + data = [self._entry(timedelta(hours=2), True)] + self.assertTrue(stats_app.images_exact_due(data, now=self.NOW)) + + def test_estimates_in_between_do_not_reset_schedule(self): + data = [ + self._entry(timedelta(hours=1, minutes=30), True), + self._entry(timedelta(minutes=30), False), + ] + self.assertFalse(stats_app.images_exact_due(data, now=self.NOW)) + + def test_old_entries_without_flag_are_ignored(self): + data = [{"timestamp": self.NOW.isoformat(), "count": 1}] + self.assertTrue(stats_app.images_exact_due(data, now=self.NOW)) + + def test_malformed_exact_timestamp_is_ignored(self): + data = [{"timestamp": "kaputt", "images_exact": True}] + self.assertTrue(stats_app.images_exact_due(data, now=self.NOW)) + + +class GetStatsTest(unittest.TestCase): + + @mock.patch.object(stats_app, "_psql_scalar") + def test_exact_query(self, psql): + psql.return_value = "100|250" + self.assertEqual(stats_app.get_stats(True), (100, 250, True)) + self.assertEqual(psql.call_args[0][0], stats_app.EXACT_IMAGE_SQL) + self.assertNotIn("TABLESAMPLE", stats_app.EXACT_IMAGE_SQL) + + @mock.patch.object(stats_app, "_psql_scalar") + def test_estimate_query(self, psql): + psql.return_value = "100|250" + self.assertEqual(stats_app.get_stats(False), (100, 250, False)) + self.assertEqual(psql.call_args[0][0], stats_app.ESTIMATE_IMAGE_SQL) + self.assertIn("TABLESAMPLE SYSTEM (1)", stats_app.ESTIMATE_IMAGE_SQL) + + @mock.patch.object(stats_app, "_psql_scalar") + def test_empty_sample_on_small_table_falls_back_to_exact(self, psql): + psql.side_effect = ["10|", "10|20"] + self.assertEqual(stats_app.get_stats(False), (10, 20, True)) + self.assertEqual(psql.call_count, 2) + self.assertEqual(psql.call_args_list[1][0][0], + stats_app.EXACT_IMAGE_SQL) + + @mock.patch.object(stats_app, "_psql_scalar") + def test_empty_sample_on_large_table_stays_unknown(self, psql): + psql.return_value = "200000|" + self.assertEqual(stats_app.get_stats(False), (200000, None, False)) + + @mock.patch.object(stats_app, "_psql_scalar") + def test_missing_pictures_column_falls_back_to_page_count(self, psql): + error = subprocess.CalledProcessError( + 1, ["psql"], stderr='ERROR: column "pictures" does not exist' + ) + psql.side_effect = [error, "10"] + self.assertEqual(stats_app.get_stats(False), (10, None, False)) + self.assertEqual(psql.call_args_list[1][0][0], + "SELECT COUNT(*) FROM pages;") + + +class StatsPageTest(unittest.TestCase): + + def test_page_contains_image_counter(self): + client = stats_app.app.test_client() + response = client.get("/") + self.assertEqual(response.status_code, 200) + html = response.get_data(as_text=True) + self.assertIn('id="current-images"', html) + self.assertIn('id="current-images-meta"', html) + self.assertIn("Estimated · exact every 2 h", html) + self.assertIn("imagesExact", html) + + +if __name__ == "__main__": + unittest.main(verbosity=2)