475 lines
16 KiB
Python
475 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Simple local test suite for the Vorschlagsverfeinerer.
|
|
|
|
DB-backed tests use temp tables shadowing the real ones, so the
|
|
production crawl_queue is never touched.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
|
|
def _load_env_file():
|
|
"""
|
|
Load ../.env like the systemd EnvironmentFile does, so the tests
|
|
also run outside systemd (local manual runs). Must happen before
|
|
app.py is imported, which reads the DB variables at import time.
|
|
"""
|
|
env_path = os.path.join(
|
|
os.path.dirname(os.path.abspath(__file__)),
|
|
os.pardir,
|
|
".env",
|
|
)
|
|
|
|
try:
|
|
with open(env_path) as handle:
|
|
for line in handle:
|
|
line = line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, _, value = line.partition("=")
|
|
os.environ.setdefault(key.strip(), value.strip())
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
_load_env_file()
|
|
|
|
import app as priority_app # noqa: E402
|
|
|
|
|
|
class CleanDomainTest(unittest.TestCase):
|
|
|
|
def test_plain_domain(self):
|
|
self.assertEqual(
|
|
priority_app.clean_domain("example.com"), "example.com"
|
|
)
|
|
|
|
def test_uppercase_and_spaces(self):
|
|
self.assertEqual(
|
|
priority_app.clean_domain(" EXAMPLE.COM "), "example.com"
|
|
)
|
|
|
|
def test_www_prefix(self):
|
|
self.assertEqual(
|
|
priority_app.clean_domain("www.example.com"), "example.com"
|
|
)
|
|
|
|
def test_full_url(self):
|
|
self.assertEqual(
|
|
priority_app.clean_domain("https://www.example.com/pfad?x=1"),
|
|
"example.com",
|
|
)
|
|
|
|
def test_http_url_with_port(self):
|
|
self.assertEqual(
|
|
priority_app.clean_domain("http://example.com:8080/index.html"),
|
|
"example.com",
|
|
)
|
|
|
|
def test_trailing_dot(self):
|
|
self.assertEqual(
|
|
priority_app.clean_domain("example.com."), "example.com"
|
|
)
|
|
|
|
def test_subdomain(self):
|
|
self.assertEqual(
|
|
priority_app.clean_domain("docs.example.co.uk"),
|
|
"docs.example.co.uk",
|
|
)
|
|
|
|
def test_invalid_empty(self):
|
|
self.assertIsNone(priority_app.clean_domain(""))
|
|
|
|
def test_invalid_no_tld(self):
|
|
self.assertIsNone(priority_app.clean_domain("localhost"))
|
|
|
|
def test_invalid_injection(self):
|
|
self.assertIsNone(
|
|
priority_app.clean_domain("example.com'; DROP TABLE pages; --")
|
|
)
|
|
|
|
def test_invalid_javascript_scheme(self):
|
|
self.assertIsNone(
|
|
priority_app.clean_domain("javascript:alert(1)")
|
|
)
|
|
|
|
|
|
class QueuePriorityTest(unittest.TestCase):
|
|
"""
|
|
Test the queue-jump SQL against a real database connection whose
|
|
temp tables shadow the production tables.
|
|
"""
|
|
|
|
def setUp(self):
|
|
self.conn = priority_app.db()
|
|
|
|
with self.conn.cursor() as cur:
|
|
cur.execute("""
|
|
CREATE TEMP TABLE crawl_queue (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
url TEXT NOT NULL UNIQUE,
|
|
source_url TEXT,
|
|
discovered_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
priority INTEGER NOT NULL DEFAULT 0,
|
|
status TEXT NOT NULL DEFAULT 'pending'
|
|
CHECK (status IN
|
|
('pending','processing','done','failed')),
|
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
last_attempt_at TIMESTAMPTZ
|
|
)
|
|
""")
|
|
cur.execute("""
|
|
CREATE TEMP TABLE domain_suggestions (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
domain TEXT NOT NULL UNIQUE,
|
|
requested_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
request_count INTEGER NOT NULL DEFAULT 1,
|
|
last_status TEXT
|
|
)
|
|
""")
|
|
|
|
self.conn.commit()
|
|
|
|
def tearDown(self):
|
|
self.conn.rollback()
|
|
self.conn.close()
|
|
|
|
def _rows(self, sql, params=None):
|
|
with self.conn.cursor() as cur:
|
|
cur.execute(sql, params)
|
|
return cur.fetchall()
|
|
|
|
def test_suggestion_jumps_queue(self):
|
|
with self.conn.cursor() as cur:
|
|
# Simulate queue state: normal pending URLs, current max
|
|
# priority 500 like in production.
|
|
cur.execute("""
|
|
INSERT INTO crawl_queue (url, priority, status) VALUES
|
|
('https://a.example/', 0, 'pending'),
|
|
('https://b.example/', 0, 'pending'),
|
|
('https://canva.example/', 500, 'pending')
|
|
""")
|
|
self.conn.commit()
|
|
|
|
priority_app.suggest_domain(self.conn, "cool.example")
|
|
|
|
# The crawler's exact claim order must put the suggestion first.
|
|
order = self._rows("""
|
|
SELECT url FROM crawl_queue
|
|
WHERE status = 'pending'
|
|
ORDER BY priority DESC, discovered_at ASC, id ASC
|
|
""")
|
|
self.assertEqual(order[0][0], "https://cool.example/")
|
|
|
|
row = self._rows(
|
|
"SELECT priority, status FROM crawl_queue "
|
|
"WHERE url = 'https://cool.example/'"
|
|
)[0]
|
|
self.assertEqual(row, (priority_app.SUGGEST_PRIORITY, "pending"))
|
|
|
|
# Other rows were not touched.
|
|
self.assertEqual(
|
|
self._rows(
|
|
"SELECT priority FROM crawl_queue "
|
|
"WHERE url = 'https://canva.example/'"
|
|
)[0][0],
|
|
500,
|
|
)
|
|
|
|
# Suggestion bookkeeping was written.
|
|
self.assertEqual(
|
|
self._rows("SELECT domain FROM domain_suggestions"),
|
|
[("cool.example",)],
|
|
)
|
|
|
|
def test_existing_failed_url_is_requeued(self):
|
|
with self.conn.cursor() as cur:
|
|
cur.execute("""
|
|
INSERT INTO crawl_queue (url, priority, status) VALUES
|
|
('https://old.example/', 0, 'failed'),
|
|
('https://x.example/', 0, 'pending')
|
|
""")
|
|
self.conn.commit()
|
|
|
|
priority_app.suggest_domain(self.conn, "old.example")
|
|
|
|
row = self._rows(
|
|
"SELECT priority, status FROM crawl_queue "
|
|
"WHERE url = 'https://old.example/'"
|
|
)[0]
|
|
self.assertEqual(row, (priority_app.SUGGEST_PRIORITY, "pending"))
|
|
|
|
def test_processing_row_is_left_alone(self):
|
|
with self.conn.cursor() as cur:
|
|
cur.execute("""
|
|
INSERT INTO crawl_queue (url, priority, status) VALUES
|
|
('https://busy.example/', 123, 'processing')
|
|
""")
|
|
self.conn.commit()
|
|
|
|
priority_app.suggest_domain(self.conn, "busy.example")
|
|
|
|
# A worker is crawling it right now: do not interfere.
|
|
row = self._rows(
|
|
"SELECT priority, status FROM crawl_queue "
|
|
"WHERE url = 'https://busy.example/'"
|
|
)[0]
|
|
self.assertEqual(row, (123, "processing"))
|
|
|
|
# But it is still recorded as a suggestion.
|
|
self.assertEqual(
|
|
self._rows(
|
|
"SELECT request_count FROM domain_suggestions "
|
|
"WHERE domain = 'busy.example'"
|
|
)[0][0],
|
|
1,
|
|
)
|
|
|
|
def test_multiple_suggestions_crawl_in_order(self):
|
|
priority_app.suggest_domain(self.conn, "first.example")
|
|
priority_app.suggest_domain(self.conn, "second.example")
|
|
|
|
order = self._rows("""
|
|
SELECT url FROM crawl_queue
|
|
WHERE status = 'pending'
|
|
ORDER BY priority DESC, discovered_at ASC, id ASC
|
|
""")
|
|
|
|
self.assertEqual(
|
|
[row[0] for row in order[:2]],
|
|
["https://first.example/", "https://second.example/"],
|
|
)
|
|
|
|
|
|
class UrlsAheadTest(unittest.TestCase):
|
|
"""urls_ahead_of must count only earlier suggestions."""
|
|
|
|
def setUp(self):
|
|
self.conn = priority_app.db()
|
|
|
|
with self.conn.cursor() as cur:
|
|
cur.execute("""
|
|
CREATE TEMP TABLE crawl_queue (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
url TEXT NOT NULL UNIQUE,
|
|
source_url TEXT,
|
|
discovered_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
priority INTEGER NOT NULL DEFAULT 0,
|
|
status TEXT NOT NULL DEFAULT 'pending',
|
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
last_attempt_at TIMESTAMPTZ
|
|
)
|
|
""")
|
|
cur.execute("""
|
|
INSERT INTO crawl_queue (url, priority, status) VALUES
|
|
('https://earlier.example/', 1000000, 'pending'),
|
|
('https://me.example/', 1000000, 'pending'),
|
|
('https://later.example/', 1000000, 'pending'),
|
|
('https://normal.example/', 0, 'pending'),
|
|
('https://gone.example/', 1000000, 'done')
|
|
""")
|
|
# Force a strict discovered_at ordering.
|
|
cur.execute("""
|
|
UPDATE crawl_queue
|
|
SET discovered_at = now() - (
|
|
CASE url
|
|
WHEN 'https://earlier.example/' THEN interval '30 s'
|
|
WHEN 'https://me.example/' THEN interval '20 s'
|
|
ELSE interval '10 s'
|
|
END
|
|
)
|
|
WHERE url LIKE 'https://%.example/'
|
|
""")
|
|
|
|
self.conn.commit()
|
|
|
|
def tearDown(self):
|
|
self.conn.rollback()
|
|
self.conn.close()
|
|
|
|
def test_only_earlier_suggestions_count(self):
|
|
def fake_query(sql, params=None):
|
|
with self.conn.cursor() as cur:
|
|
cur.execute(sql, params)
|
|
rows = cur.fetchall()
|
|
cols = [d.name for d in cur.description]
|
|
return [dict(zip(cols, row)) for row in rows]
|
|
|
|
with mock.patch.object(priority_app, "query", fake_query):
|
|
ahead = priority_app.urls_ahead_of("me.example")
|
|
|
|
# Only 'earlier.example' is pending, same priority, older.
|
|
self.assertEqual(ahead, 1)
|
|
|
|
|
|
class ApiTest(unittest.TestCase):
|
|
"""HTTP-layer tests with mocked database access."""
|
|
|
|
def setUp(self):
|
|
priority_app.app.config["TESTING"] = True
|
|
self.client = priority_app.app.test_client()
|
|
|
|
def test_index_serves_html(self):
|
|
response = self.client.get("/")
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertIn(b"Vorschlagsverfeinerer", response.data)
|
|
self.assertIn(b"domainInput", response.data)
|
|
|
|
def test_suggest_valid_domain(self):
|
|
with mock.patch.object(priority_app, "db"), \
|
|
mock.patch.object(priority_app, "suggest_domain"), \
|
|
mock.patch.object(priority_app, "urls_ahead_of",
|
|
return_value=2):
|
|
response = self.client.post(
|
|
"/api/suggest", json={"domain": "example.com"}
|
|
)
|
|
|
|
self.assertEqual(response.status_code, 200)
|
|
data = response.get_json()
|
|
self.assertTrue(data["ok"])
|
|
self.assertEqual(data["domain"], "example.com")
|
|
self.assertEqual(data["urls_ahead"], 2)
|
|
self.assertEqual(data["priority"], priority_app.SUGGEST_PRIORITY)
|
|
|
|
def test_suggest_invalid_domain(self):
|
|
response = self.client.post(
|
|
"/api/suggest", json={"domain": "not a domain!"}
|
|
)
|
|
|
|
self.assertEqual(response.status_code, 400)
|
|
self.assertFalse(response.get_json()["ok"])
|
|
|
|
def test_suggest_get_variant(self):
|
|
with mock.patch.object(priority_app, "db"), \
|
|
mock.patch.object(priority_app, "suggest_domain"), \
|
|
mock.patch.object(priority_app, "urls_ahead_of",
|
|
side_effect=RuntimeError):
|
|
response = self.client.get("/api/suggest?domain=example.org")
|
|
|
|
self.assertEqual(response.status_code, 200)
|
|
data = response.get_json()
|
|
self.assertTrue(data["ok"])
|
|
self.assertEqual(data["urls_ahead"], None)
|
|
|
|
def test_suggest_creates_missing_table_and_retries(self):
|
|
"""
|
|
First run without domain_suggestions: the insert fails with
|
|
42P01, the table gets created, and the suggestion succeeds.
|
|
"""
|
|
calls = {"suggest": 0}
|
|
|
|
class FakeUndefinedTable(Exception):
|
|
pgcode = "42P01"
|
|
|
|
def fake_db():
|
|
return mock.MagicMock()
|
|
|
|
def fake_suggest(conn, domain):
|
|
calls["suggest"] += 1
|
|
if calls["suggest"] == 1:
|
|
raise FakeUndefinedTable(
|
|
"relation \"domain_suggestions\" does not exist"
|
|
)
|
|
|
|
with mock.patch.object(priority_app, "db", side_effect=fake_db), \
|
|
mock.patch.object(priority_app, "suggest_domain",
|
|
side_effect=fake_suggest), \
|
|
mock.patch.object(priority_app,
|
|
"ensure_suggestions_table_standalone") as \
|
|
ensure_mock, \
|
|
mock.patch.object(priority_app, "urls_ahead_of", return_value=0):
|
|
response = self.client.post(
|
|
"/api/suggest", json={"domain": "example.com"}
|
|
)
|
|
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertTrue(response.get_json()["ok"])
|
|
self.assertEqual(calls["suggest"], 2)
|
|
ensure_mock.assert_called_once()
|
|
|
|
def test_suggest_db_error_returns_503(self):
|
|
with mock.patch.object(priority_app, "db",
|
|
side_effect=RuntimeError("db down")):
|
|
response = self.client.post(
|
|
"/api/suggest", json={"domain": "example.com"}
|
|
)
|
|
|
|
self.assertEqual(response.status_code, 503)
|
|
self.assertFalse(response.get_json()["ok"])
|
|
|
|
def test_queue_endpoint_serves_snapshot(self):
|
|
snapshot = {"pending": 3, "processing": 1, "boosted": 2}
|
|
with mock.patch.object(priority_app, "_queue_lock"):
|
|
priority_app._queue_snapshot = dict(snapshot)
|
|
response = self.client.get("/api/queue")
|
|
|
|
self.assertEqual(response.status_code, 200)
|
|
data = response.get_json()
|
|
self.assertTrue(data["online"])
|
|
self.assertEqual(data["pending"], 3)
|
|
self.assertEqual(data["processing"], 1)
|
|
self.assertEqual(data["boosted"], 2)
|
|
|
|
def test_queue_endpoint_fallback_counts_boosted_only(self):
|
|
old = priority_app._queue_snapshot
|
|
priority_app._queue_snapshot = None
|
|
try:
|
|
with mock.patch.object(
|
|
priority_app, "query",
|
|
return_value=[{"boosted": 7}],
|
|
):
|
|
response = self.client.get("/api/queue")
|
|
finally:
|
|
priority_app._queue_snapshot = old
|
|
|
|
self.assertEqual(response.status_code, 200)
|
|
data = response.get_json()
|
|
self.assertTrue(data["online"])
|
|
self.assertIsNone(data["pending"])
|
|
self.assertEqual(data["boosted"], 7)
|
|
|
|
def test_queue_endpoint_db_down(self):
|
|
old = priority_app._queue_snapshot
|
|
priority_app._queue_snapshot = None
|
|
try:
|
|
with mock.patch.object(priority_app, "query",
|
|
side_effect=RuntimeError("no db")):
|
|
response = self.client.get("/api/queue")
|
|
finally:
|
|
priority_app._queue_snapshot = old
|
|
|
|
self.assertEqual(response.status_code, 503)
|
|
self.assertFalse(response.get_json()["online"])
|
|
|
|
def test_suggestions_endpoint_empty(self):
|
|
with mock.patch.object(priority_app, "query",
|
|
side_effect=RuntimeError("no table")):
|
|
response = self.client.get("/api/suggestions")
|
|
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertEqual(response.get_json(), [])
|
|
|
|
def test_health_ok(self):
|
|
with mock.patch.object(priority_app, "query", return_value=[]):
|
|
response = self.client.get("/api/health")
|
|
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertEqual(response.get_json()["status"], "ok")
|
|
|
|
def test_health_db_down(self):
|
|
with mock.patch.object(priority_app, "query",
|
|
side_effect=RuntimeError("down")):
|
|
response = self.client.get("/api/health")
|
|
|
|
self.assertEqual(response.status_code, 503)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|