make stats count images

web/app.py

@@ -4,7 +4,7 @@ import subprocess
 import json
 import time
 import threading
-from datetime import datetime
+from datetime import datetime, timedelta
 from pathlib import Path
 
 from flask import Flask, jsonify, render_template_string, Response
@@ -12,6 +12,15 @@ from flask import Flask, jsonify, render_template_string, Response
 DB_DATA = Path(__file__).parent / "page_counts.json"
 INTERVAL = 30 * 60  # 30 Minuten
 
+# Bilder werden nur alle 2 Stunden exakt gezählt (SUM über die ~10-GB-
+# Tabelle). Dazwischen wird eine 1-%-Stichprobe (TABLESAMPLE SYSTEM)
+# hochgerechnet - das kostet nur einen Bruchteil der IO.
+EXACT_IMAGE_INTERVAL = 2 * 60 * 60  # Sekunden
+IMAGE_SAMPLE_PERCENT = 1
+# Sehr kleine Tabellen: Stichprobe kann leer sein; dort ist der exakte
+# Count ohnehin billig.
+SAMPLE_EXACT_FALLBACK_ROWS = 100_000
+
 app = Flask(__name__)
 
 # r""" raw string: preserves \n etc. for the embedded JavaScript
@@ -109,6 +118,8 @@ HTML = r"""
 
         .current-meta { color: var(--muted); margin-bottom: 24px; }
 
+        .current-row { display: flex; flex-wrap: wrap; gap: 0 56px; }
+
         .badges { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 14px; }
 
         .badge {
@@ -263,10 +274,20 @@ HTML = r"""
         </div>
     </header>
 
-    <div class="current">
+    <div class="current-row">
-        <span id="current">–</span><span class="unit">Pages</span>
+        <div>
+            <div class="current">
+                <span id="current">–</span><span class="unit">Pages</span>
+            </div>
+            <div class="current-meta" id="current-meta">Loading…</div>
+        </div>
+        <div>
+            <div class="current">
+                <span id="current-images">–</span><span class="unit">Images</span>
+            </div>
+            <div class="current-meta" id="current-images-meta">Loading…</div>
+        </div>
     </div>
-    <div class="current-meta" id="current-meta">Loading…</div>
 
     <div class="badges">
         <span class="badge"><span class="dot" id="status-dot"></span><span id="status-text">Loading…</span></span>
@@ -524,7 +545,16 @@ function setSigned(id, value) {
 
 function parsePoints(values) {
     return (values || [])
-        .map(function (v) { return { x: new Date(v.timestamp).getTime(), y: v.count }; })
+        .map(function (v) {
+            const images = (v.images === null || v.images === undefined)
+                ? null : Number(v.images);
+            return {
+                x: new Date(v.timestamp).getTime(),
+                y: v.count,
+                images: Number.isFinite(images) ? images : null,
+                imagesExact: v.images_exact === true
+            };
+        })
         .filter(function (p) { return !isNaN(p.x); })
         .sort(function (a, b) { return a.x - b.x; });
 }
@@ -1032,12 +1062,16 @@ function renderAll() {
     const now = Date.now();
     const elCurrent = document.getElementById('current');
     const meta = document.getElementById('current-meta');
+    const elImages = document.getElementById('current-images');
+    const imagesMeta = document.getElementById('current-images-meta');
 
     document.getElementById('badge-count').textContent = 'Measurements: ' + all.length;
 
     if (!all.length) {
         elCurrent.textContent = '–';
-        meta.textContent = 'No measurements yet – the collector stores one every 15 minutes.';
+        meta.textContent = 'No measurements yet – the collector stores one every 30 minutes.';
+        elImages.textContent = '–';
+        imagesMeta.textContent = 'No image count yet';
         setStatus('offline', 'No data');
         return;
     }
@@ -1054,6 +1088,16 @@ function renderAll() {
     meta.textContent = 'Last measurement: ' + fmtDateFull(last.x) +
         ' · next expected in ~' + fmtDur(Math.max(0, last.x + intervalMs - now));
 
+    if (last.images === null) {
+        elImages.textContent = '–';
+        imagesMeta.textContent = 'No image count yet';
+    } else {
+        elImages.textContent = fmt(last.images);
+        imagesMeta.textContent = (last.imagesExact
+            ? 'Exact count' : 'Estimated · exact every 2 h') +
+            ' · ' + fmtDateFull(last.x);
+    }
+
     document.getElementById('badge-since').textContent = 'First measurement: ' + fmtDateFull(first.x);
     document.getElementById('badge-updated').textContent = 'Last update: ' + fmtDate(last.x);
 
@@ -1228,7 +1272,29 @@ def save_data(data):
     tmp.replace(DB_DATA)
 
 
-def get_page_count():
+# Exakt: ein Scan, der COUNT(*) und die Bilder in einem Durchgang
+# ermittelt. Wird nur alle EXACT_IMAGE_INTERVAL Sekunden ausgeführt.
+EXACT_IMAGE_SQL = (
+    "SELECT COUNT(*), "
+    "COALESCE(SUM(cardinality(pictures)), 0)::bigint "
+    "FROM pages;"
+)
+
+# Schätzung: COUNT(*) wie bisher, Bilder aus 1 % der Tabellenblöcke
+# (TABLESAMPLE SYSTEM), hochgerechnet. Die zweite Spalte ist NULL, wenn
+# kein Block ausgewählt wurde (sehr kleine Tabelle).
+ESTIMATE_IMAGE_SQL = (
+    "SELECT (SELECT COUNT(*) FROM pages), "
+    "(SELECT (SUM(cardinality(pictures)) * {scale})::bigint "
+    "FROM pages TABLESAMPLE SYSTEM ({percent}));"
+).format(
+    scale=100 // IMAGE_SAMPLE_PERCENT,
+    percent=IMAGE_SAMPLE_PERCENT,
+)
+
+
+def _psql_scalar(sql):
+    """Eine einzelne psql-Zeile ("count|images") abholen."""
     command = [
         "sudo",
         "-u",
@@ -1240,8 +1306,10 @@ def get_page_count():
         "jonassearch",
         "-t",
         "-A",
+        "-F",
+        "|",
         "-c",
-        "SELECT COUNT(*) FROM pages;"
+        sql,
     ]
 
     result = subprocess.run(
@@ -1251,31 +1319,103 @@ def get_page_count():
         check=True
     )
 
-    return int(result.stdout.strip())
+    return result.stdout.strip()
+
+
+def _parse_stats_row(output):
+    """"count|images" -> (count, images|None)."""
+    count_raw, _sep, images_raw = (output or "").strip().partition("|")
+    count = int(count_raw.strip())
+    images = int(images_raw.strip()) if images_raw.strip() else None
+    return count, images
+
+
+def images_exact_due(data, now=None):
+    """
+    True, wenn der letzte exakte Bild-Count fehlt oder mindestens
+    EXACT_IMAGE_INTERVAL alt ist. Einträge aus der Zeit vor dieser
+    Funktion (ohne images_exact) zählen nicht als exakt.
+    """
+    now = now or datetime.now()
+
+    for entry in reversed(data or []):
+        if entry.get("images_exact") is not True:
+            continue
+        try:
+            timestamp = datetime.fromisoformat(str(entry.get("timestamp", "")))
+        except (TypeError, ValueError):
+            continue
+        return now - timestamp >= timedelta(seconds=EXACT_IMAGE_INTERVAL)
+
+    return True
+
+
+def get_stats(exact_images):
+    """
+    Seitenzahl + Bildanzahl holen.
+
+    exact_images=True: ein Scan, exakter Bild-Count.
+    exact_images=False: Seitenzahl wie bisher, Bilder aus einer
+    1-%-Stichprobe hochgerechnet. Liefert die Stichprobe nichts (kleine
+    Tabelle), wird exakt gezählt - das ist dort billig.
+
+    Rückgabe: (pages, images, images_exact). images kann None sein,
+    wenn eine große Tabelle wider Erwarten keinen Block gesampelt hat.
+    """
+    try:
+        if exact_images:
+            count, images = _parse_stats_row(_psql_scalar(EXACT_IMAGE_SQL))
+            return count, images, True
+
+        count, images = _parse_stats_row(_psql_scalar(ESTIMATE_IMAGE_SQL))
+    except subprocess.CalledProcessError as exc:
+        # Z. B. noch keine Migration (pages.pictures fehlt): die
+        # Seitenzahl soll trotzdem weiter erfasst werden.
+        detail = ((exc.stderr or "").strip().splitlines() or [str(exc)])[-1]
+        print(f"Bild-Statistik nicht verfügbar ({detail}); zähle nur Seiten.")
+        return int(_psql_scalar("SELECT COUNT(*) FROM pages;").strip()), None, False
+
+    if images is None:
+        if count < SAMPLE_EXACT_FALLBACK_ROWS:
+            return get_stats(True)
+        return count, None, False
+
+    return count, images, False
 
 
 def collect():
     while True:
         try:
-            count = get_page_count()
+            data = load_data()
+            exact = images_exact_due(data)
+            count, images, images_exact = get_stats(exact)
+
+            if images is None:
+                # Stichprobe ohne Treffer (praktisch nur bei sehr großen
+                # Tabellen): letzten bekannten Wert fortschreiben.
+                for previous in reversed(data):
+                    if previous.get("images") is not None:
+                        images = previous["images"]
+                        break
 
             entry = {
                 "timestamp": datetime.now().isoformat(),
-                "count": count
+                "count": count,
+                "images": images,
+                "images_exact": images_exact,
             }
 
-            data = load_data()
             data.append(entry)
-
             save_data(data)
 
             print(
                 f"[{entry['timestamp']}] "
-                f"Pages: {count}"
+                f"Pages: {count} · Images: {images} "
+                f"({'exakt' if images_exact else 'geschätzt'})"
             )
 
         except Exception as e:
-            print(f"Fehler beim Abrufen der Seitenzahl: {e}")
+            print(f"Fehler beim Abrufen der Statistik: {e}")
 
         time.sleep(INTERVAL)
 

web/test_stats.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""
+Tests für den Statserver (web/app.py), ohne Datenbank.
+
+Geprüft werden die DB-freien Teile: Parsen der psql-Zeile, der
+2-Stunden-Rhythmus für den exakten Bild-Count, die Wahl zwischen
+exakter Zählung und Stichprobe sowie die Einbindung der neuen
+Images-Anzeige in die HTML-Seite.
+"""
+
+import importlib.util
+import os
+import subprocess
+import unittest
+from datetime import datetime, timedelta
+from unittest import mock
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+
+# web/app.py unter einem eindeutigen Modulnamen laden, damit es nicht mit
+# priorityweb/app.py (gleicher Dateiname) kollidiert.
+_spec = importlib.util.spec_from_file_location(
+    "jonassearch_stats_app", os.path.join(HERE, "app.py")
+)
+stats_app = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(stats_app)
+
+
+class ParseStatsRowTest(unittest.TestCase):
+
+    def test_count_and_images(self):
+        self.assertEqual(
+            stats_app._parse_stats_row("981981|12345"), (981981, 12345)
+        )
+
+    def test_empty_sample_is_none(self):
+        self.assertEqual(stats_app._parse_stats_row("10|"), (10, None))
+        self.assertEqual(stats_app._parse_stats_row("10|\n"), (10, None))
+
+
+class ImagesExactDueTest(unittest.TestCase):
+
+    NOW = datetime(2026, 10, 17, 12, 0, 0)
+
+    def _entry(self, age, exact):
+        return {
+            "timestamp": (self.NOW - age).isoformat(),
+            "count": 1,
+            "images": 2,
+            "images_exact": exact,
+        }
+
+    def test_no_data_means_due(self):
+        self.assertTrue(stats_app.images_exact_due([], now=self.NOW))
+
+    def test_recent_exact_is_not_due(self):
+        data = [self._entry(timedelta(hours=1), True)]
+        self.assertFalse(stats_app.images_exact_due(data, now=self.NOW))
+
+    def test_two_hours_old_is_due(self):
+        data = [self._entry(timedelta(hours=2), True)]
+        self.assertTrue(stats_app.images_exact_due(data, now=self.NOW))
+
+    def test_estimates_in_between_do_not_reset_schedule(self):
+        data = [
+            self._entry(timedelta(hours=1, minutes=30), True),
+            self._entry(timedelta(minutes=30), False),
+        ]
+        self.assertFalse(stats_app.images_exact_due(data, now=self.NOW))
+
+    def test_old_entries_without_flag_are_ignored(self):
+        data = [{"timestamp": self.NOW.isoformat(), "count": 1}]
+        self.assertTrue(stats_app.images_exact_due(data, now=self.NOW))
+
+    def test_malformed_exact_timestamp_is_ignored(self):
+        data = [{"timestamp": "kaputt", "images_exact": True}]
+        self.assertTrue(stats_app.images_exact_due(data, now=self.NOW))
+
+
+class GetStatsTest(unittest.TestCase):
+
+    @mock.patch.object(stats_app, "_psql_scalar")
+    def test_exact_query(self, psql):
+        psql.return_value = "100|250"
+        self.assertEqual(stats_app.get_stats(True), (100, 250, True))
+        self.assertEqual(psql.call_args[0][0], stats_app.EXACT_IMAGE_SQL)
+        self.assertNotIn("TABLESAMPLE", stats_app.EXACT_IMAGE_SQL)
+
+    @mock.patch.object(stats_app, "_psql_scalar")
+    def test_estimate_query(self, psql):
+        psql.return_value = "100|250"
+        self.assertEqual(stats_app.get_stats(False), (100, 250, False))
+        self.assertEqual(psql.call_args[0][0], stats_app.ESTIMATE_IMAGE_SQL)
+        self.assertIn("TABLESAMPLE SYSTEM (1)", stats_app.ESTIMATE_IMAGE_SQL)
+
+    @mock.patch.object(stats_app, "_psql_scalar")
+    def test_empty_sample_on_small_table_falls_back_to_exact(self, psql):
+        psql.side_effect = ["10|", "10|20"]
+        self.assertEqual(stats_app.get_stats(False), (10, 20, True))
+        self.assertEqual(psql.call_count, 2)
+        self.assertEqual(psql.call_args_list[1][0][0],
+                         stats_app.EXACT_IMAGE_SQL)
+
+    @mock.patch.object(stats_app, "_psql_scalar")
+    def test_empty_sample_on_large_table_stays_unknown(self, psql):
+        psql.return_value = "200000|"
+        self.assertEqual(stats_app.get_stats(False), (200000, None, False))
+
+    @mock.patch.object(stats_app, "_psql_scalar")
+    def test_missing_pictures_column_falls_back_to_page_count(self, psql):
+        error = subprocess.CalledProcessError(
+            1, ["psql"], stderr='ERROR: column "pictures" does not exist'
+        )
+        psql.side_effect = [error, "10"]
+        self.assertEqual(stats_app.get_stats(False), (10, None, False))
+        self.assertEqual(psql.call_args_list[1][0][0],
+                         "SELECT COUNT(*) FROM pages;")
+
+
+class StatsPageTest(unittest.TestCase):
+
+    def test_page_contains_image_counter(self):
+        client = stats_app.app.test_client()
+        response = client.get("/")
+        self.assertEqual(response.status_code, 200)
+        html = response.get_data(as_text=True)
+        self.assertIn('id="current-images"', html)
+        self.assertIn('id="current-images-meta"', html)
+        self.assertIn("Estimated · exact every 2 h", html)
+        self.assertIn("imagesExact", html)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)