add vorschläge
This commit is contained in:
@@ -0,0 +1,474 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Simple local test suite for the Vorschlagsverfeinerer.
|
||||
|
||||
DB-backed tests use temp tables shadowing the real ones, so the
|
||||
production crawl_queue is never touched.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
|
||||
def _load_env_file():
|
||||
"""
|
||||
Load ../.env like the systemd EnvironmentFile does, so the tests
|
||||
also run outside systemd (local manual runs). Must happen before
|
||||
app.py is imported, which reads the DB variables at import time.
|
||||
"""
|
||||
env_path = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)),
|
||||
os.pardir,
|
||||
".env",
|
||||
)
|
||||
|
||||
try:
|
||||
with open(env_path) as handle:
|
||||
for line in handle:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
os.environ.setdefault(key.strip(), value.strip())
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
_load_env_file()
|
||||
|
||||
import app as priority_app # noqa: E402
|
||||
|
||||
|
||||
class CleanDomainTest(unittest.TestCase):
|
||||
|
||||
def test_plain_domain(self):
|
||||
self.assertEqual(
|
||||
priority_app.clean_domain("example.com"), "example.com"
|
||||
)
|
||||
|
||||
def test_uppercase_and_spaces(self):
|
||||
self.assertEqual(
|
||||
priority_app.clean_domain(" EXAMPLE.COM "), "example.com"
|
||||
)
|
||||
|
||||
def test_www_prefix(self):
|
||||
self.assertEqual(
|
||||
priority_app.clean_domain("www.example.com"), "example.com"
|
||||
)
|
||||
|
||||
def test_full_url(self):
|
||||
self.assertEqual(
|
||||
priority_app.clean_domain("https://www.example.com/pfad?x=1"),
|
||||
"example.com",
|
||||
)
|
||||
|
||||
def test_http_url_with_port(self):
|
||||
self.assertEqual(
|
||||
priority_app.clean_domain("http://example.com:8080/index.html"),
|
||||
"example.com",
|
||||
)
|
||||
|
||||
def test_trailing_dot(self):
|
||||
self.assertEqual(
|
||||
priority_app.clean_domain("example.com."), "example.com"
|
||||
)
|
||||
|
||||
def test_subdomain(self):
|
||||
self.assertEqual(
|
||||
priority_app.clean_domain("docs.example.co.uk"),
|
||||
"docs.example.co.uk",
|
||||
)
|
||||
|
||||
def test_invalid_empty(self):
|
||||
self.assertIsNone(priority_app.clean_domain(""))
|
||||
|
||||
def test_invalid_no_tld(self):
|
||||
self.assertIsNone(priority_app.clean_domain("localhost"))
|
||||
|
||||
def test_invalid_injection(self):
|
||||
self.assertIsNone(
|
||||
priority_app.clean_domain("example.com'; DROP TABLE pages; --")
|
||||
)
|
||||
|
||||
def test_invalid_javascript_scheme(self):
|
||||
self.assertIsNone(
|
||||
priority_app.clean_domain("javascript:alert(1)")
|
||||
)
|
||||
|
||||
|
||||
class QueuePriorityTest(unittest.TestCase):
|
||||
"""
|
||||
Test the queue-jump SQL against a real database connection whose
|
||||
temp tables shadow the production tables.
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.conn = priority_app.db()
|
||||
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
CREATE TEMP TABLE crawl_queue (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
url TEXT NOT NULL UNIQUE,
|
||||
source_url TEXT,
|
||||
discovered_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
priority INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN
|
||||
('pending','processing','done','failed')),
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
last_attempt_at TIMESTAMPTZ
|
||||
)
|
||||
""")
|
||||
cur.execute("""
|
||||
CREATE TEMP TABLE domain_suggestions (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
domain TEXT NOT NULL UNIQUE,
|
||||
requested_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
request_count INTEGER NOT NULL DEFAULT 1,
|
||||
last_status TEXT
|
||||
)
|
||||
""")
|
||||
|
||||
self.conn.commit()
|
||||
|
||||
def tearDown(self):
|
||||
self.conn.rollback()
|
||||
self.conn.close()
|
||||
|
||||
def _rows(self, sql, params=None):
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute(sql, params)
|
||||
return cur.fetchall()
|
||||
|
||||
def test_suggestion_jumps_queue(self):
|
||||
with self.conn.cursor() as cur:
|
||||
# Simulate queue state: normal pending URLs, current max
|
||||
# priority 500 like in production.
|
||||
cur.execute("""
|
||||
INSERT INTO crawl_queue (url, priority, status) VALUES
|
||||
('https://a.example/', 0, 'pending'),
|
||||
('https://b.example/', 0, 'pending'),
|
||||
('https://canva.example/', 500, 'pending')
|
||||
""")
|
||||
self.conn.commit()
|
||||
|
||||
priority_app.suggest_domain(self.conn, "cool.example")
|
||||
|
||||
# The crawler's exact claim order must put the suggestion first.
|
||||
order = self._rows("""
|
||||
SELECT url FROM crawl_queue
|
||||
WHERE status = 'pending'
|
||||
ORDER BY priority DESC, discovered_at ASC, id ASC
|
||||
""")
|
||||
self.assertEqual(order[0][0], "https://cool.example/")
|
||||
|
||||
row = self._rows(
|
||||
"SELECT priority, status FROM crawl_queue "
|
||||
"WHERE url = 'https://cool.example/'"
|
||||
)[0]
|
||||
self.assertEqual(row, (priority_app.SUGGEST_PRIORITY, "pending"))
|
||||
|
||||
# Other rows were not touched.
|
||||
self.assertEqual(
|
||||
self._rows(
|
||||
"SELECT priority FROM crawl_queue "
|
||||
"WHERE url = 'https://canva.example/'"
|
||||
)[0][0],
|
||||
500,
|
||||
)
|
||||
|
||||
# Suggestion bookkeeping was written.
|
||||
self.assertEqual(
|
||||
self._rows("SELECT domain FROM domain_suggestions"),
|
||||
[("cool.example",)],
|
||||
)
|
||||
|
||||
def test_existing_failed_url_is_requeued(self):
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
INSERT INTO crawl_queue (url, priority, status) VALUES
|
||||
('https://old.example/', 0, 'failed'),
|
||||
('https://x.example/', 0, 'pending')
|
||||
""")
|
||||
self.conn.commit()
|
||||
|
||||
priority_app.suggest_domain(self.conn, "old.example")
|
||||
|
||||
row = self._rows(
|
||||
"SELECT priority, status FROM crawl_queue "
|
||||
"WHERE url = 'https://old.example/'"
|
||||
)[0]
|
||||
self.assertEqual(row, (priority_app.SUGGEST_PRIORITY, "pending"))
|
||||
|
||||
def test_processing_row_is_left_alone(self):
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
INSERT INTO crawl_queue (url, priority, status) VALUES
|
||||
('https://busy.example/', 123, 'processing')
|
||||
""")
|
||||
self.conn.commit()
|
||||
|
||||
priority_app.suggest_domain(self.conn, "busy.example")
|
||||
|
||||
# A worker is crawling it right now: do not interfere.
|
||||
row = self._rows(
|
||||
"SELECT priority, status FROM crawl_queue "
|
||||
"WHERE url = 'https://busy.example/'"
|
||||
)[0]
|
||||
self.assertEqual(row, (123, "processing"))
|
||||
|
||||
# But it is still recorded as a suggestion.
|
||||
self.assertEqual(
|
||||
self._rows(
|
||||
"SELECT request_count FROM domain_suggestions "
|
||||
"WHERE domain = 'busy.example'"
|
||||
)[0][0],
|
||||
1,
|
||||
)
|
||||
|
||||
def test_multiple_suggestions_crawl_in_order(self):
|
||||
priority_app.suggest_domain(self.conn, "first.example")
|
||||
priority_app.suggest_domain(self.conn, "second.example")
|
||||
|
||||
order = self._rows("""
|
||||
SELECT url FROM crawl_queue
|
||||
WHERE status = 'pending'
|
||||
ORDER BY priority DESC, discovered_at ASC, id ASC
|
||||
""")
|
||||
|
||||
self.assertEqual(
|
||||
[row[0] for row in order[:2]],
|
||||
["https://first.example/", "https://second.example/"],
|
||||
)
|
||||
|
||||
|
||||
class UrlsAheadTest(unittest.TestCase):
|
||||
"""urls_ahead_of must count only earlier suggestions."""
|
||||
|
||||
def setUp(self):
|
||||
self.conn = priority_app.db()
|
||||
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
CREATE TEMP TABLE crawl_queue (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
url TEXT NOT NULL UNIQUE,
|
||||
source_url TEXT,
|
||||
discovered_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
priority INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
last_attempt_at TIMESTAMPTZ
|
||||
)
|
||||
""")
|
||||
cur.execute("""
|
||||
INSERT INTO crawl_queue (url, priority, status) VALUES
|
||||
('https://earlier.example/', 1000000, 'pending'),
|
||||
('https://me.example/', 1000000, 'pending'),
|
||||
('https://later.example/', 1000000, 'pending'),
|
||||
('https://normal.example/', 0, 'pending'),
|
||||
('https://gone.example/', 1000000, 'done')
|
||||
""")
|
||||
# Force a strict discovered_at ordering.
|
||||
cur.execute("""
|
||||
UPDATE crawl_queue
|
||||
SET discovered_at = now() - (
|
||||
CASE url
|
||||
WHEN 'https://earlier.example/' THEN interval '30 s'
|
||||
WHEN 'https://me.example/' THEN interval '20 s'
|
||||
ELSE interval '10 s'
|
||||
END
|
||||
)
|
||||
WHERE url LIKE 'https://%.example/'
|
||||
""")
|
||||
|
||||
self.conn.commit()
|
||||
|
||||
def tearDown(self):
|
||||
self.conn.rollback()
|
||||
self.conn.close()
|
||||
|
||||
def test_only_earlier_suggestions_count(self):
|
||||
def fake_query(sql, params=None):
|
||||
with self.conn.cursor() as cur:
|
||||
cur.execute(sql, params)
|
||||
rows = cur.fetchall()
|
||||
cols = [d.name for d in cur.description]
|
||||
return [dict(zip(cols, row)) for row in rows]
|
||||
|
||||
with mock.patch.object(priority_app, "query", fake_query):
|
||||
ahead = priority_app.urls_ahead_of("me.example")
|
||||
|
||||
# Only 'earlier.example' is pending, same priority, older.
|
||||
self.assertEqual(ahead, 1)
|
||||
|
||||
|
||||
class ApiTest(unittest.TestCase):
|
||||
"""HTTP-layer tests with mocked database access."""
|
||||
|
||||
def setUp(self):
|
||||
priority_app.app.config["TESTING"] = True
|
||||
self.client = priority_app.app.test_client()
|
||||
|
||||
def test_index_serves_html(self):
|
||||
response = self.client.get("/")
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertIn(b"Vorschlagsverfeinerer", response.data)
|
||||
self.assertIn(b"domainInput", response.data)
|
||||
|
||||
def test_suggest_valid_domain(self):
|
||||
with mock.patch.object(priority_app, "db"), \
|
||||
mock.patch.object(priority_app, "suggest_domain"), \
|
||||
mock.patch.object(priority_app, "urls_ahead_of",
|
||||
return_value=2):
|
||||
response = self.client.post(
|
||||
"/api/suggest", json={"domain": "example.com"}
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = response.get_json()
|
||||
self.assertTrue(data["ok"])
|
||||
self.assertEqual(data["domain"], "example.com")
|
||||
self.assertEqual(data["urls_ahead"], 2)
|
||||
self.assertEqual(data["priority"], priority_app.SUGGEST_PRIORITY)
|
||||
|
||||
def test_suggest_invalid_domain(self):
|
||||
response = self.client.post(
|
||||
"/api/suggest", json={"domain": "not a domain!"}
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 400)
|
||||
self.assertFalse(response.get_json()["ok"])
|
||||
|
||||
def test_suggest_get_variant(self):
|
||||
with mock.patch.object(priority_app, "db"), \
|
||||
mock.patch.object(priority_app, "suggest_domain"), \
|
||||
mock.patch.object(priority_app, "urls_ahead_of",
|
||||
side_effect=RuntimeError):
|
||||
response = self.client.get("/api/suggest?domain=example.org")
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = response.get_json()
|
||||
self.assertTrue(data["ok"])
|
||||
self.assertEqual(data["urls_ahead"], None)
|
||||
|
||||
def test_suggest_creates_missing_table_and_retries(self):
|
||||
"""
|
||||
First run without domain_suggestions: the insert fails with
|
||||
42P01, the table gets created, and the suggestion succeeds.
|
||||
"""
|
||||
calls = {"suggest": 0}
|
||||
|
||||
class FakeUndefinedTable(Exception):
|
||||
pgcode = "42P01"
|
||||
|
||||
def fake_db():
|
||||
return mock.MagicMock()
|
||||
|
||||
def fake_suggest(conn, domain):
|
||||
calls["suggest"] += 1
|
||||
if calls["suggest"] == 1:
|
||||
raise FakeUndefinedTable(
|
||||
"relation \"domain_suggestions\" does not exist"
|
||||
)
|
||||
|
||||
with mock.patch.object(priority_app, "db", side_effect=fake_db), \
|
||||
mock.patch.object(priority_app, "suggest_domain",
|
||||
side_effect=fake_suggest), \
|
||||
mock.patch.object(priority_app,
|
||||
"ensure_suggestions_table_standalone") as \
|
||||
ensure_mock, \
|
||||
mock.patch.object(priority_app, "urls_ahead_of", return_value=0):
|
||||
response = self.client.post(
|
||||
"/api/suggest", json={"domain": "example.com"}
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertTrue(response.get_json()["ok"])
|
||||
self.assertEqual(calls["suggest"], 2)
|
||||
ensure_mock.assert_called_once()
|
||||
|
||||
def test_suggest_db_error_returns_503(self):
|
||||
with mock.patch.object(priority_app, "db",
|
||||
side_effect=RuntimeError("db down")):
|
||||
response = self.client.post(
|
||||
"/api/suggest", json={"domain": "example.com"}
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 503)
|
||||
self.assertFalse(response.get_json()["ok"])
|
||||
|
||||
def test_queue_endpoint_serves_snapshot(self):
|
||||
snapshot = {"pending": 3, "processing": 1, "boosted": 2}
|
||||
with mock.patch.object(priority_app, "_queue_lock"):
|
||||
priority_app._queue_snapshot = dict(snapshot)
|
||||
response = self.client.get("/api/queue")
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = response.get_json()
|
||||
self.assertTrue(data["online"])
|
||||
self.assertEqual(data["pending"], 3)
|
||||
self.assertEqual(data["processing"], 1)
|
||||
self.assertEqual(data["boosted"], 2)
|
||||
|
||||
def test_queue_endpoint_fallback_counts_boosted_only(self):
|
||||
old = priority_app._queue_snapshot
|
||||
priority_app._queue_snapshot = None
|
||||
try:
|
||||
with mock.patch.object(
|
||||
priority_app, "query",
|
||||
return_value=[{"boosted": 7}],
|
||||
):
|
||||
response = self.client.get("/api/queue")
|
||||
finally:
|
||||
priority_app._queue_snapshot = old
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
data = response.get_json()
|
||||
self.assertTrue(data["online"])
|
||||
self.assertIsNone(data["pending"])
|
||||
self.assertEqual(data["boosted"], 7)
|
||||
|
||||
def test_queue_endpoint_db_down(self):
|
||||
old = priority_app._queue_snapshot
|
||||
priority_app._queue_snapshot = None
|
||||
try:
|
||||
with mock.patch.object(priority_app, "query",
|
||||
side_effect=RuntimeError("no db")):
|
||||
response = self.client.get("/api/queue")
|
||||
finally:
|
||||
priority_app._queue_snapshot = old
|
||||
|
||||
self.assertEqual(response.status_code, 503)
|
||||
self.assertFalse(response.get_json()["online"])
|
||||
|
||||
def test_suggestions_endpoint_empty(self):
|
||||
with mock.patch.object(priority_app, "query",
|
||||
side_effect=RuntimeError("no table")):
|
||||
response = self.client.get("/api/suggestions")
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(response.get_json(), [])
|
||||
|
||||
def test_health_ok(self):
|
||||
with mock.patch.object(priority_app, "query", return_value=[]):
|
||||
response = self.client.get("/api/health")
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(response.get_json()["status"], "ok")
|
||||
|
||||
def test_health_db_down(self):
|
||||
with mock.patch.object(priority_app, "query",
|
||||
side_effect=RuntimeError("down")):
|
||||
response = self.client.get("/api/health")
|
||||
|
||||
self.assertEqual(response.status_code, 503)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
Reference in New Issue
Block a user