Serverjonas/search
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

make stats count images

Browse Source
This commit is contained in:
jhkaktus
2026-10-17 21:42:06 +02:00
parent aa0bc4e960
commit ca5af75bbd
2 changed files with 289 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/app.py
+155 -15
View File
@@ -4,7 +4,7 @@ import subprocess
import json
import time
import threading
from datetime import datetime
from datetime import datetime, timedelta
from pathlib import Path
from flask import Flask, jsonify, render_template_string, Response
@@ -12,6 +12,15 @@ from flask import Flask, jsonify, render_template_string, Response
DB_DATA = Path(__file__).parent / "page_counts.json"
INTERVAL = 30 * 60 # 30 Minuten
# Bilder werden nur alle 2 Stunden exakt gezählt (SUM über die ~10-GB-
# Tabelle). Dazwischen wird eine 1-%-Stichprobe (TABLESAMPLE SYSTEM)
# hochgerechnet - das kostet nur einen Bruchteil der IO.
EXACT_IMAGE_INTERVAL = 2 * 60 * 60 # Sekunden
IMAGE_SAMPLE_PERCENT = 1
# Sehr kleine Tabellen: Stichprobe kann leer sein; dort ist der exakte
# Count ohnehin billig.
SAMPLE_EXACT_FALLBACK_ROWS = 100_000
app = Flask(__name__)
# r""" raw string: preserves \n etc. for the embedded JavaScript
@@ -109,6 +118,8 @@ HTML = r"""
.current-meta { color: var(--muted); margin-bottom: 24px; }
.current-row { display: flex; flex-wrap: wrap; gap: 0 56px; }
.badges { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 14px; }
.badge {
@@ -263,10 +274,20 @@ HTML = r"""
</div>
</header>
<div class="current">
<span id="current">–</span><span class="unit">Pages</span>
<div class="current-row">
<div>
<div class="current">
<span id="current">–</span><span class="unit">Pages</span>
</div>
<div class="current-meta" id="current-meta">Loading…</div>
</div>
<div>
<div class="current">
<span id="current-images">–</span><span class="unit">Images</span>
</div>
<div class="current-meta" id="current-images-meta">Loading…</div>
</div>
</div>
<div class="current-meta" id="current-meta">Loading…</div>
<div class="badges">
<span class="badge"><span class="dot" id="status-dot"></span><span id="status-text">Loading…</span></span>
@@ -524,7 +545,16 @@ function setSigned(id, value) {
function parsePoints(values) {
return (values || [])
.map(function (v) { return { x: new Date(v.timestamp).getTime(), y: v.count }; })
.map(function (v) {
const images = (v.images === null || v.images === undefined)
? null : Number(v.images);
return {
x: new Date(v.timestamp).getTime(),
y: v.count,
images: Number.isFinite(images) ? images : null,
imagesExact: v.images_exact === true
};
})
.filter(function (p) { return !isNaN(p.x); })
.sort(function (a, b) { return a.x - b.x; });
}
@@ -1032,12 +1062,16 @@ function renderAll() {
const now = Date.now();
const elCurrent = document.getElementById('current');
const meta = document.getElementById('current-meta');
const elImages = document.getElementById('current-images');
const imagesMeta = document.getElementById('current-images-meta');
document.getElementById('badge-count').textContent = 'Measurements: ' + all.length;
if (!all.length) {
elCurrent.textContent = '–';
meta.textContent = 'No measurements yet – the collector stores one every 15 minutes.';
meta.textContent = 'No measurements yet – the collector stores one every 30 minutes.';
elImages.textContent = '–';
imagesMeta.textContent = 'No image count yet';
setStatus('offline', 'No data');
return;
}
@@ -1054,6 +1088,16 @@ function renderAll() {
meta.textContent = 'Last measurement: ' + fmtDateFull(last.x) +
' · next expected in ~' + fmtDur(Math.max(0, last.x + intervalMs - now));
if (last.images === null) {
elImages.textContent = '–';
imagesMeta.textContent = 'No image count yet';
} else {
elImages.textContent = fmt(last.images);
imagesMeta.textContent = (last.imagesExact
? 'Exact count' : 'Estimated · exact every 2 h') +
' · ' + fmtDateFull(last.x);
}
document.getElementById('badge-since').textContent = 'First measurement: ' + fmtDateFull(first.x);
document.getElementById('badge-updated').textContent = 'Last update: ' + fmtDate(last.x);
@@ -1228,7 +1272,29 @@ def save_data(data):
tmp.replace(DB_DATA)
def get_page_count():
# Exakt: ein Scan, der COUNT(*) und die Bilder in einem Durchgang
# ermittelt. Wird nur alle EXACT_IMAGE_INTERVAL Sekunden ausgeführt.
EXACT_IMAGE_SQL = (
"SELECT COUNT(*), "
"COALESCE(SUM(cardinality(pictures)), 0)::bigint "
"FROM pages;"
)
# Schätzung: COUNT(*) wie bisher, Bilder aus 1 % der Tabellenblöcke
# (TABLESAMPLE SYSTEM), hochgerechnet. Die zweite Spalte ist NULL, wenn
# kein Block ausgewählt wurde (sehr kleine Tabelle).
ESTIMATE_IMAGE_SQL = (
"SELECT (SELECT COUNT(*) FROM pages), "
"(SELECT (SUM(cardinality(pictures)) * {scale})::bigint "
"FROM pages TABLESAMPLE SYSTEM ({percent}));"
).format(
scale=100 // IMAGE_SAMPLE_PERCENT,
percent=IMAGE_SAMPLE_PERCENT,
)
def _psql_scalar(sql):
"""Eine einzelne psql-Zeile ("count|images") abholen."""
command = [
"sudo",
"-u",
@@ -1240,8 +1306,10 @@ def get_page_count():
"jonassearch",
"-t",
"-A",
"-F",
"|",
"-c",
"SELECT COUNT(*) FROM pages;"
sql,
]
result = subprocess.run(
@@ -1251,31 +1319,103 @@ def get_page_count():
check=True
)
return int(result.stdout.strip())
return result.stdout.strip()
def _parse_stats_row(output):
""""count|images" -> (count, images|None)."""
count_raw, _sep, images_raw = (output or "").strip().partition("|")
count = int(count_raw.strip())
images = int(images_raw.strip()) if images_raw.strip() else None
return count, images
def images_exact_due(data, now=None):
"""
True, wenn der letzte exakte Bild-Count fehlt oder mindestens
EXACT_IMAGE_INTERVAL alt ist. Einträge aus der Zeit vor dieser
Funktion (ohne images_exact) zählen nicht als exakt.
"""
now = now or datetime.now()
for entry in reversed(data or []):
if entry.get("images_exact") is not True:
continue
try:
timestamp = datetime.fromisoformat(str(entry.get("timestamp", "")))
except (TypeError, ValueError):
continue
return now - timestamp >= timedelta(seconds=EXACT_IMAGE_INTERVAL)
return True
def get_stats(exact_images):
"""
Seitenzahl + Bildanzahl holen.
exact_images=True: ein Scan, exakter Bild-Count.
exact_images=False: Seitenzahl wie bisher, Bilder aus einer
1-%-Stichprobe hochgerechnet. Liefert die Stichprobe nichts (kleine
Tabelle), wird exakt gezählt - das ist dort billig.
Rückgabe: (pages, images, images_exact). images kann None sein,
wenn eine große Tabelle wider Erwarten keinen Block gesampelt hat.
"""
try:
if exact_images:
count, images = _parse_stats_row(_psql_scalar(EXACT_IMAGE_SQL))
return count, images, True
count, images = _parse_stats_row(_psql_scalar(ESTIMATE_IMAGE_SQL))
except subprocess.CalledProcessError as exc:
# Z. B. noch keine Migration (pages.pictures fehlt): die
# Seitenzahl soll trotzdem weiter erfasst werden.
detail = ((exc.stderr or "").strip().splitlines() or [str(exc)])[-1]
print(f"Bild-Statistik nicht verfügbar ({detail}); zähle nur Seiten.")
return int(_psql_scalar("SELECT COUNT(*) FROM pages;").strip()), None, False
if images is None:
if count < SAMPLE_EXACT_FALLBACK_ROWS:
return get_stats(True)
return count, None, False
return count, images, False
def collect():
while True:
try:
count = get_page_count()
data = load_data()
exact = images_exact_due(data)
count, images, images_exact = get_stats(exact)
if images is None:
# Stichprobe ohne Treffer (praktisch nur bei sehr großen
# Tabellen): letzten bekannten Wert fortschreiben.
for previous in reversed(data):
if previous.get("images") is not None:
images = previous["images"]
break
entry = {
"timestamp": datetime.now().isoformat(),
"count": count
"count": count,
"images": images,
"images_exact": images_exact,
}
data = load_data()
data.append(entry)
save_data(data)
print(
f"[{entry['timestamp']}] "
f"Pages: {count}"
f"Pages: {count} · Images: {images} "
f"({'exakt' if images_exact else 'geschätzt'})"
)
except Exception as e:
print(f"Fehler beim Abrufen der Seitenzahl: {e}")
print(f"Fehler beim Abrufen der Statistik: {e}")
time.sleep(INTERVAL)