Serverjonas/search
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

make the stats mor ereadaable

Browse Source
This commit is contained in:
jhkaktus
2026-10-21 18:32:31 +02:00
parent 3ebfeca7cc
commit 530c2b1ecc
3 changed files with 619 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/test_stats.py
+144 -6
View File
@@ -117,6 +117,59 @@ class GetStatsTest(unittest.TestCase):
"SELECT COUNT(*) FROM pages;")
class GetPendingTest(unittest.TestCase):
@mock.patch.object(stats_app, "_psql_scalar")
def test_counts_pending_queue_rows(self, psql):
psql.return_value = "1384711"
self.assertEqual(stats_app.get_pending(), 1384711)
self.assertIn("FROM crawl_queue", stats_app.PENDING_QUEUE_SQL)
self.assertIn("status = 'pending'", stats_app.PENDING_QUEUE_SQL)
@mock.patch.object(stats_app, "_psql_scalar")
def test_unavailable_queue_returns_none(self, psql):
psql.side_effect = subprocess.CalledProcessError(
1, ["psql"], stderr='ERROR: relation "crawl_queue" does not exist'
)
self.assertIsNone(stats_app.get_pending())
@mock.patch.object(stats_app, "_psql_scalar")
def test_unexpected_output_returns_none(self, psql):
psql.return_value = ""
self.assertIsNone(stats_app.get_pending())
class CollectTest(unittest.TestCase):
def _run_collect(self, get_stats, get_pending):
with mock.patch.object(stats_app, "load_data", return_value=[]), \
mock.patch.object(stats_app, "save_data") as save_data, \
mock.patch.object(stats_app, "get_stats", get_stats), \
mock.patch.object(stats_app, "get_pending", get_pending), \
mock.patch.object(stats_app, "time") as time_mock:
time_mock.sleep.side_effect = KeyboardInterrupt
with self.assertRaises(KeyboardInterrupt):
stats_app.collect()
return save_data.call_args[0][0][0]
def test_pending_is_stored_with_every_sample(self):
entry = self._run_collect(
mock.Mock(return_value=(1000, 2000, True)),
mock.Mock(return_value=4711),
)
self.assertEqual(entry["pending"], 4711)
self.assertEqual(entry["count"], 1000)
self.assertEqual(entry["images"], 2000)
def test_missing_queue_does_not_break_the_sample(self):
entry = self._run_collect(
mock.Mock(return_value=(1000, 2000, False)),
mock.Mock(return_value=None),
)
self.assertIsNone(entry["pending"])
self.assertEqual(entry["count"], 1000)
class StatsPageTest(unittest.TestCase):
def test_page_contains_image_counter(self):
@@ -159,15 +212,27 @@ class StatsPageTest(unittest.TestCase):
self.assertIn('id="heatmap-images"', html)
self.assertIn('id="images-empty"', html)
# the image series skips entries without an image count (old data)
# and only plots exact measurements - no estimates between the 2 h counts
self.assertIn("function toImagePoints(points)", html)
# Image estimates are excluded by default and can be included with the persistent toggle.
self.assertIn('id="show-estimated-images"', html)
self.assertIn('type="checkbox" id="show-estimated-images"', html)
self.assertIn("Include estimated image counts in graphs", html)
self.assertIn("function toImagePoints(points, includeEstimated)", html)
self.assertIn("p.images !== null && p.images !== undefined", html)
self.assertIn(".filter(function (p) { return p.imagesExact; })", html)
self.assertIn("only exact measurements are plotted (no estimates)", html)
self.assertIn("includeEstimated || p.imagesExact", html)
self.assertIn("localStorage.getItem('js-show-estimated-images')", html)
self.assertIn("localStorage.setItem('js-show-estimated-images'", html)
self.assertIn("toImagePoints(all, showEstimatedImages)", html)
self.assertIn("Only exact measurements (counted every 2 h) are plotted", html)
self.assertIn("Exact and estimated measurements are plotted", html)
self.assertIn("Only estimated image counts so far", html)
self.assertIn("No exact image counts in the selected range", html)
self.assertIn("No image counts in the selected range", html)
# The headline counter stays the latest collected value; only graphs toggle estimates.
self.assertIn("elImages.textContent = fmt(last.images)", html)
# Images estimates affect graphs only; other series keep their own points.
self.assertIn("renderCharts({\n prefix: '',\n points: fpoints", html)
self.assertIn("renderCharts({\n prefix: 'pending',\n points: pendingFpoints", html)
self.assertIn("renderHeatmap(imageAll, 'heatmap-images', 'Images')", html)
self.assertIn("renderRecords(imageDeltas, 'records-images', 'Images')", html)
# existing page chart ids must stay lowercase (mainChart, not MainChart)
self.assertIn("function chartId(prefix, suffix)", html)
self.assertIn("chartId(prefix, 'MainChart')", html)
@@ -182,6 +247,79 @@ class StatsPageTest(unittest.TestCase):
self.assertIn("renderHeatmap(all, 'heatmap', 'Pages')", html)
self.assertIn("renderHeatmap(imageAll, 'heatmap-images', 'Images')", html)
def test_page_has_metric_tabs_with_one_panel_each(self):
client = stats_app.app.test_client()
response = client.get("/")
self.assertEqual(response.status_code, 200)
html = response.get_data(as_text=True)
# three equally styled tab buttons (pages / images / queue) with icons
self.assertIn('class="metric-btn', html)
for metric in ("pages", "images", "queue"):
self.assertIn('data-metric="' + metric + '"', html)
self.assertIn("metric-icon", html)
self.assertIn("📄", html)
self.assertIn("🖼️", html)
self.assertIn("⏳", html)
# every metric has its own panel; only the initial one is active
self.assertEqual(html.count('class="metric-panel active" data-metric="pages"'), 1)
for metric in ("images", "queue"):
self.assertIn('class="metric-panel" data-metric="' + metric + '"', html)
self.assertIn('id="summary-images"', html)
self.assertIn('id="summary-queue"', html)
# switching tabs hides the other panels and is remembered
self.assertIn("function setMetric(next, persist)", html)
self.assertIn("panel.classList.toggle('active', panel.dataset.metric === activeMetric)", html)
self.assertIn("localStorage.setItem('js-metric', activeMetric)", html)
self.assertIn("localStorage.getItem('js-metric')", html)
# hidden panels must not create Chart.js instances (canvas has 0 px)
self.assertIn("function isVisible(el)", html)
self.assertIn("if (!target || !isVisible(target)) return;", html)
# summary cards are generated for the non-page tabs
self.assertIn("function renderSummary(containerId, all, points)", html)
self.assertIn("renderSummary('summary-images', imageAll, imageFpoints)", html)
self.assertIn("renderSummary('summary-queue', pendingAll, pendingFpoints)", html)
# exports follow the selected tab
self.assertIn("function exportSeries()", html)
self.assertIn("return { name: 'jonassearch_images'", html)
self.assertIn("return { name: 'jonassearch_queue'", html)
def test_page_contains_pending_queue_counter_and_graphs(self):
client = stats_app.app.test_client()
response = client.get("/")
self.assertEqual(response.status_code, 200)
html = response.get_data(as_text=True)
self.assertIn('id="current-pending"', html)
self.assertIn('id="current-pending-meta"', html)
self.assertIn("Queue – all graphs", html)
for canvas in (
"pendingMainChart",
"pendingDeltaChart",
"pendingCumChart",
"pendingHourlyChart",
"pendingHodChart",
):
self.assertIn('id="' + canvas + '"', html)
self.assertIn('id="records-pending"', html)
self.assertIn('id="heatmap-pending"', html)
self.assertIn('id="pending-empty"', html)
# pending values are parsed and plotted without sampling/filtering
self.assertIn("function toPendingPoints(points)", html)
self.assertIn("p.pending !== null && p.pending !== undefined", html)
self.assertIn("filterRange(pendingAll, last.x)", html)
self.assertIn("unit: 'Pending'", html)
self.assertIn("renderRecords(pendingDeltas, 'records-pending', 'Pending')", html)
self.assertIn("renderHeatmap(pendingAll, 'heatmap-pending', 'Pending')", html)
self.assertIn("No queue counts yet", html)
self.assertIn("No queue counts in the selected range.", html)
if __name__ == "__main__":
unittest.main(verbosity=2)