make stats count images
This commit is contained in:
+152
-12
@@ -4,7 +4,7 @@ import subprocess
|
||||
import json
|
||||
import time
|
||||
import threading
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from flask import Flask, jsonify, render_template_string, Response
|
||||
@@ -12,6 +12,15 @@ from flask import Flask, jsonify, render_template_string, Response
|
||||
DB_DATA = Path(__file__).parent / "page_counts.json"
|
||||
INTERVAL = 30 * 60 # 30 Minuten
|
||||
|
||||
# Bilder werden nur alle 2 Stunden exakt gezählt (SUM über die ~10-GB-
|
||||
# Tabelle). Dazwischen wird eine 1-%-Stichprobe (TABLESAMPLE SYSTEM)
|
||||
# hochgerechnet - das kostet nur einen Bruchteil der IO.
|
||||
EXACT_IMAGE_INTERVAL = 2 * 60 * 60 # Sekunden
|
||||
IMAGE_SAMPLE_PERCENT = 1
|
||||
# Sehr kleine Tabellen: Stichprobe kann leer sein; dort ist der exakte
|
||||
# Count ohnehin billig.
|
||||
SAMPLE_EXACT_FALLBACK_ROWS = 100_000
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
# r""" raw string: preserves \n etc. for the embedded JavaScript
|
||||
@@ -109,6 +118,8 @@ HTML = r"""
|
||||
|
||||
.current-meta { color: var(--muted); margin-bottom: 24px; }
|
||||
|
||||
.current-row { display: flex; flex-wrap: wrap; gap: 0 56px; }
|
||||
|
||||
.badges { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 14px; }
|
||||
|
||||
.badge {
|
||||
@@ -263,10 +274,20 @@ HTML = r"""
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div class="current-row">
|
||||
<div>
|
||||
<div class="current">
|
||||
<span id="current">–</span><span class="unit">Pages</span>
|
||||
</div>
|
||||
<div class="current-meta" id="current-meta">Loading…</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="current">
|
||||
<span id="current-images">–</span><span class="unit">Images</span>
|
||||
</div>
|
||||
<div class="current-meta" id="current-images-meta">Loading…</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="badges">
|
||||
<span class="badge"><span class="dot" id="status-dot"></span><span id="status-text">Loading…</span></span>
|
||||
@@ -524,7 +545,16 @@ function setSigned(id, value) {
|
||||
|
||||
function parsePoints(values) {
|
||||
return (values || [])
|
||||
.map(function (v) { return { x: new Date(v.timestamp).getTime(), y: v.count }; })
|
||||
.map(function (v) {
|
||||
const images = (v.images === null || v.images === undefined)
|
||||
? null : Number(v.images);
|
||||
return {
|
||||
x: new Date(v.timestamp).getTime(),
|
||||
y: v.count,
|
||||
images: Number.isFinite(images) ? images : null,
|
||||
imagesExact: v.images_exact === true
|
||||
};
|
||||
})
|
||||
.filter(function (p) { return !isNaN(p.x); })
|
||||
.sort(function (a, b) { return a.x - b.x; });
|
||||
}
|
||||
@@ -1032,12 +1062,16 @@ function renderAll() {
|
||||
const now = Date.now();
|
||||
const elCurrent = document.getElementById('current');
|
||||
const meta = document.getElementById('current-meta');
|
||||
const elImages = document.getElementById('current-images');
|
||||
const imagesMeta = document.getElementById('current-images-meta');
|
||||
|
||||
document.getElementById('badge-count').textContent = 'Measurements: ' + all.length;
|
||||
|
||||
if (!all.length) {
|
||||
elCurrent.textContent = '–';
|
||||
meta.textContent = 'No measurements yet – the collector stores one every 15 minutes.';
|
||||
meta.textContent = 'No measurements yet – the collector stores one every 30 minutes.';
|
||||
elImages.textContent = '–';
|
||||
imagesMeta.textContent = 'No image count yet';
|
||||
setStatus('offline', 'No data');
|
||||
return;
|
||||
}
|
||||
@@ -1054,6 +1088,16 @@ function renderAll() {
|
||||
meta.textContent = 'Last measurement: ' + fmtDateFull(last.x) +
|
||||
' · next expected in ~' + fmtDur(Math.max(0, last.x + intervalMs - now));
|
||||
|
||||
if (last.images === null) {
|
||||
elImages.textContent = '–';
|
||||
imagesMeta.textContent = 'No image count yet';
|
||||
} else {
|
||||
elImages.textContent = fmt(last.images);
|
||||
imagesMeta.textContent = (last.imagesExact
|
||||
? 'Exact count' : 'Estimated · exact every 2 h') +
|
||||
' · ' + fmtDateFull(last.x);
|
||||
}
|
||||
|
||||
document.getElementById('badge-since').textContent = 'First measurement: ' + fmtDateFull(first.x);
|
||||
document.getElementById('badge-updated').textContent = 'Last update: ' + fmtDate(last.x);
|
||||
|
||||
@@ -1228,7 +1272,29 @@ def save_data(data):
|
||||
tmp.replace(DB_DATA)
|
||||
|
||||
|
||||
def get_page_count():
|
||||
# Exakt: ein Scan, der COUNT(*) und die Bilder in einem Durchgang
|
||||
# ermittelt. Wird nur alle EXACT_IMAGE_INTERVAL Sekunden ausgeführt.
|
||||
EXACT_IMAGE_SQL = (
|
||||
"SELECT COUNT(*), "
|
||||
"COALESCE(SUM(cardinality(pictures)), 0)::bigint "
|
||||
"FROM pages;"
|
||||
)
|
||||
|
||||
# Schätzung: COUNT(*) wie bisher, Bilder aus 1 % der Tabellenblöcke
|
||||
# (TABLESAMPLE SYSTEM), hochgerechnet. Die zweite Spalte ist NULL, wenn
|
||||
# kein Block ausgewählt wurde (sehr kleine Tabelle).
|
||||
ESTIMATE_IMAGE_SQL = (
|
||||
"SELECT (SELECT COUNT(*) FROM pages), "
|
||||
"(SELECT (SUM(cardinality(pictures)) * {scale})::bigint "
|
||||
"FROM pages TABLESAMPLE SYSTEM ({percent}));"
|
||||
).format(
|
||||
scale=100 // IMAGE_SAMPLE_PERCENT,
|
||||
percent=IMAGE_SAMPLE_PERCENT,
|
||||
)
|
||||
|
||||
|
||||
def _psql_scalar(sql):
|
||||
"""Eine einzelne psql-Zeile ("count|images") abholen."""
|
||||
command = [
|
||||
"sudo",
|
||||
"-u",
|
||||
@@ -1240,8 +1306,10 @@ def get_page_count():
|
||||
"jonassearch",
|
||||
"-t",
|
||||
"-A",
|
||||
"-F",
|
||||
"|",
|
||||
"-c",
|
||||
"SELECT COUNT(*) FROM pages;"
|
||||
sql,
|
||||
]
|
||||
|
||||
result = subprocess.run(
|
||||
@@ -1251,31 +1319,103 @@ def get_page_count():
|
||||
check=True
|
||||
)
|
||||
|
||||
return int(result.stdout.strip())
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def _parse_stats_row(output):
|
||||
""""count|images" -> (count, images|None)."""
|
||||
count_raw, _sep, images_raw = (output or "").strip().partition("|")
|
||||
count = int(count_raw.strip())
|
||||
images = int(images_raw.strip()) if images_raw.strip() else None
|
||||
return count, images
|
||||
|
||||
|
||||
def images_exact_due(data, now=None):
|
||||
"""
|
||||
True, wenn der letzte exakte Bild-Count fehlt oder mindestens
|
||||
EXACT_IMAGE_INTERVAL alt ist. Einträge aus der Zeit vor dieser
|
||||
Funktion (ohne images_exact) zählen nicht als exakt.
|
||||
"""
|
||||
now = now or datetime.now()
|
||||
|
||||
for entry in reversed(data or []):
|
||||
if entry.get("images_exact") is not True:
|
||||
continue
|
||||
try:
|
||||
timestamp = datetime.fromisoformat(str(entry.get("timestamp", "")))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return now - timestamp >= timedelta(seconds=EXACT_IMAGE_INTERVAL)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def get_stats(exact_images):
|
||||
"""
|
||||
Seitenzahl + Bildanzahl holen.
|
||||
|
||||
exact_images=True: ein Scan, exakter Bild-Count.
|
||||
exact_images=False: Seitenzahl wie bisher, Bilder aus einer
|
||||
1-%-Stichprobe hochgerechnet. Liefert die Stichprobe nichts (kleine
|
||||
Tabelle), wird exakt gezählt - das ist dort billig.
|
||||
|
||||
Rückgabe: (pages, images, images_exact). images kann None sein,
|
||||
wenn eine große Tabelle wider Erwarten keinen Block gesampelt hat.
|
||||
"""
|
||||
try:
|
||||
if exact_images:
|
||||
count, images = _parse_stats_row(_psql_scalar(EXACT_IMAGE_SQL))
|
||||
return count, images, True
|
||||
|
||||
count, images = _parse_stats_row(_psql_scalar(ESTIMATE_IMAGE_SQL))
|
||||
except subprocess.CalledProcessError as exc:
|
||||
# Z. B. noch keine Migration (pages.pictures fehlt): die
|
||||
# Seitenzahl soll trotzdem weiter erfasst werden.
|
||||
detail = ((exc.stderr or "").strip().splitlines() or [str(exc)])[-1]
|
||||
print(f"Bild-Statistik nicht verfügbar ({detail}); zähle nur Seiten.")
|
||||
return int(_psql_scalar("SELECT COUNT(*) FROM pages;").strip()), None, False
|
||||
|
||||
if images is None:
|
||||
if count < SAMPLE_EXACT_FALLBACK_ROWS:
|
||||
return get_stats(True)
|
||||
return count, None, False
|
||||
|
||||
return count, images, False
|
||||
|
||||
|
||||
def collect():
|
||||
while True:
|
||||
try:
|
||||
count = get_page_count()
|
||||
data = load_data()
|
||||
exact = images_exact_due(data)
|
||||
count, images, images_exact = get_stats(exact)
|
||||
|
||||
if images is None:
|
||||
# Stichprobe ohne Treffer (praktisch nur bei sehr großen
|
||||
# Tabellen): letzten bekannten Wert fortschreiben.
|
||||
for previous in reversed(data):
|
||||
if previous.get("images") is not None:
|
||||
images = previous["images"]
|
||||
break
|
||||
|
||||
entry = {
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"count": count
|
||||
"count": count,
|
||||
"images": images,
|
||||
"images_exact": images_exact,
|
||||
}
|
||||
|
||||
data = load_data()
|
||||
data.append(entry)
|
||||
|
||||
save_data(data)
|
||||
|
||||
print(
|
||||
f"[{entry['timestamp']}] "
|
||||
f"Pages: {count}"
|
||||
f"Pages: {count} · Images: {images} "
|
||||
f"({'exakt' if images_exact else 'geschätzt'})"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Fehler beim Abrufen der Seitenzahl: {e}")
|
||||
print(f"Fehler beim Abrufen der Statistik: {e}")
|
||||
|
||||
time.sleep(INTERVAL)
|
||||
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tests für den Statserver (web/app.py), ohne Datenbank.
|
||||
|
||||
Geprüft werden die DB-freien Teile: Parsen der psql-Zeile, der
|
||||
2-Stunden-Rhythmus für den exakten Bild-Count, die Wahl zwischen
|
||||
exakter Zählung und Stichprobe sowie die Einbindung der neuen
|
||||
Images-Anzeige in die HTML-Seite.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import subprocess
|
||||
import unittest
|
||||
from datetime import datetime, timedelta
|
||||
from unittest import mock
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
# web/app.py unter einem eindeutigen Modulnamen laden, damit es nicht mit
|
||||
# priorityweb/app.py (gleicher Dateiname) kollidiert.
|
||||
_spec = importlib.util.spec_from_file_location(
|
||||
"jonassearch_stats_app", os.path.join(HERE, "app.py")
|
||||
)
|
||||
stats_app = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(stats_app)
|
||||
|
||||
|
||||
class ParseStatsRowTest(unittest.TestCase):
|
||||
|
||||
def test_count_and_images(self):
|
||||
self.assertEqual(
|
||||
stats_app._parse_stats_row("981981|12345"), (981981, 12345)
|
||||
)
|
||||
|
||||
def test_empty_sample_is_none(self):
|
||||
self.assertEqual(stats_app._parse_stats_row("10|"), (10, None))
|
||||
self.assertEqual(stats_app._parse_stats_row("10|\n"), (10, None))
|
||||
|
||||
|
||||
class ImagesExactDueTest(unittest.TestCase):
|
||||
|
||||
NOW = datetime(2026, 10, 17, 12, 0, 0)
|
||||
|
||||
def _entry(self, age, exact):
|
||||
return {
|
||||
"timestamp": (self.NOW - age).isoformat(),
|
||||
"count": 1,
|
||||
"images": 2,
|
||||
"images_exact": exact,
|
||||
}
|
||||
|
||||
def test_no_data_means_due(self):
|
||||
self.assertTrue(stats_app.images_exact_due([], now=self.NOW))
|
||||
|
||||
def test_recent_exact_is_not_due(self):
|
||||
data = [self._entry(timedelta(hours=1), True)]
|
||||
self.assertFalse(stats_app.images_exact_due(data, now=self.NOW))
|
||||
|
||||
def test_two_hours_old_is_due(self):
|
||||
data = [self._entry(timedelta(hours=2), True)]
|
||||
self.assertTrue(stats_app.images_exact_due(data, now=self.NOW))
|
||||
|
||||
def test_estimates_in_between_do_not_reset_schedule(self):
|
||||
data = [
|
||||
self._entry(timedelta(hours=1, minutes=30), True),
|
||||
self._entry(timedelta(minutes=30), False),
|
||||
]
|
||||
self.assertFalse(stats_app.images_exact_due(data, now=self.NOW))
|
||||
|
||||
def test_old_entries_without_flag_are_ignored(self):
|
||||
data = [{"timestamp": self.NOW.isoformat(), "count": 1}]
|
||||
self.assertTrue(stats_app.images_exact_due(data, now=self.NOW))
|
||||
|
||||
def test_malformed_exact_timestamp_is_ignored(self):
|
||||
data = [{"timestamp": "kaputt", "images_exact": True}]
|
||||
self.assertTrue(stats_app.images_exact_due(data, now=self.NOW))
|
||||
|
||||
|
||||
class GetStatsTest(unittest.TestCase):
|
||||
|
||||
@mock.patch.object(stats_app, "_psql_scalar")
|
||||
def test_exact_query(self, psql):
|
||||
psql.return_value = "100|250"
|
||||
self.assertEqual(stats_app.get_stats(True), (100, 250, True))
|
||||
self.assertEqual(psql.call_args[0][0], stats_app.EXACT_IMAGE_SQL)
|
||||
self.assertNotIn("TABLESAMPLE", stats_app.EXACT_IMAGE_SQL)
|
||||
|
||||
@mock.patch.object(stats_app, "_psql_scalar")
|
||||
def test_estimate_query(self, psql):
|
||||
psql.return_value = "100|250"
|
||||
self.assertEqual(stats_app.get_stats(False), (100, 250, False))
|
||||
self.assertEqual(psql.call_args[0][0], stats_app.ESTIMATE_IMAGE_SQL)
|
||||
self.assertIn("TABLESAMPLE SYSTEM (1)", stats_app.ESTIMATE_IMAGE_SQL)
|
||||
|
||||
@mock.patch.object(stats_app, "_psql_scalar")
|
||||
def test_empty_sample_on_small_table_falls_back_to_exact(self, psql):
|
||||
psql.side_effect = ["10|", "10|20"]
|
||||
self.assertEqual(stats_app.get_stats(False), (10, 20, True))
|
||||
self.assertEqual(psql.call_count, 2)
|
||||
self.assertEqual(psql.call_args_list[1][0][0],
|
||||
stats_app.EXACT_IMAGE_SQL)
|
||||
|
||||
@mock.patch.object(stats_app, "_psql_scalar")
|
||||
def test_empty_sample_on_large_table_stays_unknown(self, psql):
|
||||
psql.return_value = "200000|"
|
||||
self.assertEqual(stats_app.get_stats(False), (200000, None, False))
|
||||
|
||||
@mock.patch.object(stats_app, "_psql_scalar")
|
||||
def test_missing_pictures_column_falls_back_to_page_count(self, psql):
|
||||
error = subprocess.CalledProcessError(
|
||||
1, ["psql"], stderr='ERROR: column "pictures" does not exist'
|
||||
)
|
||||
psql.side_effect = [error, "10"]
|
||||
self.assertEqual(stats_app.get_stats(False), (10, None, False))
|
||||
self.assertEqual(psql.call_args_list[1][0][0],
|
||||
"SELECT COUNT(*) FROM pages;")
|
||||
|
||||
|
||||
class StatsPageTest(unittest.TestCase):
|
||||
|
||||
def test_page_contains_image_counter(self):
|
||||
client = stats_app.app.test_client()
|
||||
response = client.get("/")
|
||||
self.assertEqual(response.status_code, 200)
|
||||
html = response.get_data(as_text=True)
|
||||
self.assertIn('id="current-images"', html)
|
||||
self.assertIn('id="current-images-meta"', html)
|
||||
self.assertIn("Estimated · exact every 2 h", html)
|
||||
self.assertIn("imagesExact", html)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
Reference in New Issue
Block a user