#!/usr/bin/env python3
"""
Local test suite for the crawler (no database required).

Only the pure, DB-free parts are tested: the HostScheduler (per-host
delays, block backoff, Retry-After), the transient-error classification
and the queue-claim decision logic (blocked hosts are skipped, another
domain is picked) against an in-memory queue substitute.
"""

import os
import sys
import time
import unittest
from email import utils as email_utils
from unittest import mock

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))


def _load_env_file():
    """
    Load .env like the systemd EnvironmentFile does, so the tests also
    run outside systemd. Must happen before crawler.py is imported,
    which reads the JONASSEARCH_* variables at import time.
    """
    env_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".env")

    try:
        with open(env_path) as handle:
            for line in handle:
                line = line.strip()
                if not line or line.startswith("#") or "=" not in line:
                    continue
                key, _, value = line.partition("=")
                os.environ.setdefault(key.strip(), value.strip())
    except OSError:
        pass


_load_env_file()

import crawler  # noqa: E402


class StubRobots:
    """robots.txt stub: constant delay, no network."""

    def __init__(self, delay=3.0):
        self.delay_value = delay

    def delay(self, url):
        return self.delay_value


class FakeState:
    """In-memory crawl_queue substitute for claim/defer/requeue."""

    def __init__(self, rows=None):
        self.rows = rows if rows is not None else []

    def add(self, **kwargs):
        row = {
            "id": len(self.rows) + 1,
            "url": "",
            "source": None,
            "status": "pending",
            "priority": 0,
            "attempts": 0,
        }
        row.update(kwargs)
        self.rows.append(row)
        return row["id"]


def _url_host(url):
    """Mirror the SQL host expression used by the claim query."""
    rest = url.split("://", 1)[1] if "://" in url else url
    return rest.split("/", 1)[0].lower()


class FakeCursor:
    """Just enough SQL surface for the crawler's queue functions."""

    def __init__(self, state):
        self._state = state
        self._rows = []

    def execute(self, query, params=None):
        query = " ".join(query.split())
        params = params or ()

        if "SELECT 1 FROM crawl_queue" in query:
            self._rows = [
                (1,) for row in self._state.rows
                if row["status"] == "pending"
            ][:1]
        elif "SELECT id, url, source_url" in query:
            batch = params[-1]
            exclude = set(params[0]) if len(params) == 3 else set()
            excluded_hosts = set(params[1]) if len(params) == 3 else set()
            candidates = [
                row for row in self._state.rows
                if row["status"] == "pending"
                and row["id"] not in exclude
                and _url_host(row["url"]) not in excluded_hosts
            ]
            candidates.sort(key=lambda row: (-row["priority"], row["id"]))
            self._rows = [
                (row["id"], row["url"], row["source"])
                for row in candidates[:batch]
            ]
        elif "SET status = 'processing'" in query:
            (queue_id,) = params
            self._rows = []
            for row in self._state.rows:
                if row["id"] == queue_id and row["status"] == "pending":
                    row["status"] = "processing"
                    row["attempts"] += 1
                    self._rows = [(row["id"], row["url"], row["source"])]
                    break
        elif "GREATEST(attempts - 1, 0)" in query:
            (queue_id,) = params
            for row in self._state.rows:
                if row["id"] == queue_id:
                    row["status"] = "pending"
                    row["attempts"] = max(row["attempts"] - 1, 0)
        elif "WHEN attempts >=" in query:
            max_attempts, queue_id = params
            self._rows = []
            for row in self._state.rows:
                if row["id"] == queue_id:
                    row["status"] = (
                        "failed" if row["attempts"] >= max_attempts
                        else "pending"
                    )
                    self._rows = [(row["status"],)]
        else:
            raise AssertionError("unexpected query: %s" % query[:120])

    def fetchone(self):
        return self._rows[0] if self._rows else None

    def fetchall(self):
        return self._rows

    def __enter__(self):
        return self

    def __exit__(self, *_exc):
        return False


class FakeConnection:
    def __init__(self, state):
        self.state = state

    def cursor(self):
        return FakeCursor(self.state)

    def commit(self):
        pass

    def rollback(self):
        pass


class HostSchedulerTest(unittest.TestCase):

    def test_first_request_is_ready(self):
        scheduler = crawler.HostScheduler()
        self.assertEqual(
            scheduler.wait_time("https://a.example/x", 3.0), 0.0
        )

    def test_delay_is_honored_after_request(self):
        scheduler = crawler.HostScheduler()
        scheduler.wait("https://a.example/x", 0.0)
        remaining = scheduler.wait_time("https://a.example/y", 3.0)
        self.assertGreater(remaining, 2.5)
        self.assertLessEqual(remaining, 3.0)

    def test_short_wait_is_slept_inline(self):
        scheduler = crawler.HostScheduler()
        scheduler.wait("https://a.example/x", 0.0)
        with mock.patch("time.sleep") as sleep:
            self.assertTrue(scheduler.wait("https://a.example/y", 3.0))
        sleep.assert_called_once()

    def test_long_wait_is_deferred_without_touching_schedule(self):
        scheduler = crawler.HostScheduler()
        scheduler.wait("https://a.example/x", 0.0)
        recorded = scheduler.last_request["a.example"]
        with mock.patch("time.sleep") as sleep:
            self.assertFalse(scheduler.wait("https://a.example/y", 60.0))
        sleep.assert_not_called()
        self.assertEqual(scheduler.last_request["a.example"], recorded)

    def test_block_backoff_doubles(self):
        scheduler = crawler.HostScheduler()
        url = "https://a.example/x"
        scheduler.note_http_error(url, 429)
        first = scheduler.wait_time("https://a.example/y", 3.0)
        scheduler.note_http_error(url, 429)
        second = scheduler.wait_time("https://a.example/y", 3.0)
        self.assertGreater(first, crawler.HOST_BLOCK_RULES[429][0] * 0.9)
        self.assertGreater(second, first * 1.5)

    def test_retry_after_is_honored(self):
        scheduler = crawler.HostScheduler()
        scheduler.note_http_error(
            "https://a.example/x", 429, retry_after=1200
        )
        remaining = scheduler.wait_time("https://a.example/y", 3.0)
        self.assertGreater(remaining, 1000)
        self.assertLessEqual(remaining, crawler.HOST_BLOCK_HARD_CAP)

    def test_block_never_exceeds_hard_cap(self):
        scheduler = crawler.HostScheduler()
        url = "https://a.example/x"
        for _ in range(40):
            scheduler.note_http_error(url, 429)
        remaining = scheduler.wait_time(url, 3.0)
        self.assertLessEqual(remaining, crawler.HOST_BLOCK_HARD_CAP + 1)

    def test_success_resets_block_counter(self):
        scheduler = crawler.HostScheduler()
        url = "https://a.example/x"
        scheduler.note_http_error(url, 429)
        scheduler.note_http_error(url, 429)
        scheduler.note_success(url)
        scheduler.note_http_error(url, 429)
        remaining = scheduler.wait_time("https://a.example/y", 3.0)
        self.assertLessEqual(
            remaining, crawler.HOST_BLOCK_RULES[429][0] + 1
        )

    def test_non_block_status_is_ignored(self):
        scheduler = crawler.HostScheduler()
        scheduler.note_http_error("https://a.example/x", 404)
        self.assertEqual(
            scheduler.wait_time("https://a.example/y", 3.0), 0.0
        )

    def test_network_error_backs_off(self):
        scheduler = crawler.HostScheduler()
        scheduler.note_network_error("https://a.example/x")
        remaining = scheduler.wait_time("https://a.example/y", 3.0)
        self.assertGreater(remaining, 20)

    def test_unrelated_host_is_not_blocked(self):
        scheduler = crawler.HostScheduler()
        scheduler.note_http_error("https://a.example/x", 429)
        self.assertEqual(
            scheduler.wait_time("https://b.example/y", 3.0), 0.0
        )

    def test_busy_hosts_reports_blocked_and_rate_limited_hosts(self):
        scheduler = crawler.HostScheduler()
        scheduler.blocked_until["blocked.example"] = time.monotonic() + 900
        scheduler.wait("https://rate.example/x", 60.0)

        busy = scheduler.busy_hosts()
        self.assertGreater(busy["blocked.example"], 800)
        self.assertGreater(busy["rate.example"], 50)

    def test_busy_hosts_prunes_expired_entries(self):
        scheduler = crawler.HostScheduler()
        scheduler.blocked_until["a.example"] = time.monotonic() - 1
        scheduler.last_request["b.example"] = time.monotonic() - 100
        scheduler.host_delay["b.example"] = 3.0

        self.assertEqual(scheduler.busy_hosts(), {})
        self.assertNotIn("a.example", scheduler.blocked_until)
        self.assertNotIn("b.example", scheduler.last_request)

    def test_never_contacted_host_is_ready_even_with_large_delay(self):
        # monotonic() has an arbitrary origin: "never contacted" must not
        # be treated as last_request = 0, which would block a fresh host
        # for the whole robots.txt crawl-delay on a freshly booted host.
        scheduler = crawler.HostScheduler()
        self.assertEqual(
            scheduler.wait_time("https://slow.example/x", 86400.0, now=10.0),
            0.0,
        )


class ErrorClassificationTest(unittest.TestCase):

    def test_network_errors_are_transient(self):
        self.assertTrue(crawler.is_transient_error("network:timeout"))
        self.assertTrue(crawler.is_transient_error("network:[Errno -2] x"))

    def test_rate_limit_and_server_errors_are_transient(self):
        for status in (408, 425, 429, 500, 502, 503, 504):
            self.assertTrue(crawler.is_transient_error("http_%d" % status))

    def test_terminal_errors_are_not_transient(self):
        for error in (
            "http_404", "http_410", "robots", "too_large",
            "not_html", "error:boom", "host_busy", None, "",
        ):
            self.assertFalse(crawler.is_transient_error(error))

    def test_retry_after_seconds_digit(self):
        self.assertEqual(
            crawler._retry_after_seconds({"Retry-After": "120"}), 120.0
        )

    def test_retry_after_http_date(self):
        headers = {
            "Retry-After": email_utils.formatdate(
                time.time() + 120, usegmt=True
            )
        }
        value = crawler._retry_after_seconds(headers)
        self.assertGreater(value, 100)
        self.assertLess(value, 140)

    def test_retry_after_missing_or_garbage(self):
        self.assertIsNone(crawler._retry_after_seconds({}))
        self.assertIsNone(
            crawler._retry_after_seconds({"Retry-After": "soon"})
        )
        self.assertIsNone(crawler._retry_after_seconds(None))


class PictureExtractionTest(unittest.TestCase):

    BASE = "https://example.com/articles/page.html"

    def _pictures(self, html):
        return crawler.parse_html(html, self.BASE)["pictures"]

    def test_img_src_relative_is_absolute_and_fragment_dropped(self):
        self.assertEqual(
            self._pictures('<img src="/images/a.jpg#frag">'),
            ["https://example.com/images/a.jpg"],
        )

    def test_protocol_relative_external_image_and_dedupe(self):
        pictures = self._pictures(
            '<img src="//cdn.example.net/pic.png">'
            '<img src="https://cdn.example.net/pic.png">'
        )
        self.assertEqual(
            pictures, ["https://cdn.example.net/pic.png"]
        )

    def test_srcset_and_picture_sources(self):
        pictures = self._pictures(
            '<img src="/fallback.jpg" '
            'srcset="/small.jpg 1x, /large.jpg 2x">'
            '<picture><source srcset="/b.webp 1x, /b2.webp 2x"></picture>'
        )
        self.assertEqual(pictures, [
            "https://example.com/fallback.jpg",
            "https://example.com/small.jpg",
            "https://example.com/large.jpg",
            "https://example.com/b.webp",
            "https://example.com/b2.webp",
        ])

    def test_srcset_data_url_does_not_create_bogus_candidates(self):
        pictures = self._pictures(
            '<img srcset="data:image/png;base64,AAAA 1x, /real.jpg 2x">'
        )
        self.assertEqual(pictures, ["https://example.com/real.jpg"])

    def test_og_twitter_and_image_src(self):
        pictures = self._pictures(
            '<meta property="og:image" content="https://example.com/og.png">'
            '<meta name="twitter:image" content="/tw.png">'
            '<link rel="image_src" href="/link.png">'
        )
        self.assertEqual(pictures, [
            "https://example.com/og.png",
            "https://example.com/tw.png",
            "https://example.com/link.png",
        ])

    def test_lazy_loading_attributes(self):
        pictures = self._pictures(
            '<img data-src="//cdn.example.org/lazy.webp" '
            'data-lazy-src="/lazy2.jpg" data-original="/orig.jpg">'
        )
        self.assertEqual(pictures, [
            "https://cdn.example.org/lazy.webp",
            "https://example.com/lazy2.jpg",
            "https://example.com/orig.jpg",
        ])

    def test_invalid_and_unsupported_sources_are_ignored(self):
        pictures = self._pictures(
            '<img src="data:image/png;base64,AAAA">'
            '<img src="javascript:alert(1)">'
            '<img src="ftp://example.com/x.png">'
            '<img src="">'
            '<img src="   ">'
            '<img>'
        )
        self.assertEqual(pictures, [])

    def test_non_public_image_hosts_are_ignored(self):
        pictures = self._pictures(
            '<img src="http://localhost/a.png">'
            '<img src="http://127.0.0.1/b.png">'
        )
        self.assertEqual(pictures, [])

    def test_page_without_images(self):
        self.assertEqual(
            self._pictures("<html><body><p>keine Bilder</p></body></html>"),
            [],
        )

    def test_existing_fields_still_parsed(self):
        parsed = crawler.parse_html(
            '<title>Seite</title><h1>Überschrift</h1>'
            '<a href="/x">x</a><img src="/i.png">',
            self.BASE,
        )
        self.assertEqual(parsed["title"], "Seite")
        self.assertEqual(parsed["headings"], ["Überschrift"])
        self.assertEqual(parsed["links"], ["https://example.com/x"])
        self.assertEqual(parsed["pictures"], ["https://example.com/i.png"])


class SavePagePicturesTest(unittest.TestCase):

    @staticmethod
    def _parsed(pictures=None):
        parsed = {
            "title": "Titel",
            "headings": [],
            "content": "Inhalt",
            "links": [],
            "domain": "example.com",
            "tld": ".com",
        }
        if pictures is not None:
            parsed["pictures"] = pictures
        return parsed

    def _save(self, parsed):
        conn = mock.MagicMock()
        cur = conn.cursor.return_value.__enter__.return_value
        crawler.save_page(
            conn, "https://example.com/", None, 200, "text/html",
            parsed, 12,
        )
        sql, params = cur.execute.call_args[0]
        return sql, params

    def test_pictures_are_written_and_deduplicated(self):
        sql, params = self._save(self._parsed([
            "https://example.com/a.png",
            "https://example.com/a.png",
            "https://example.com/b.png",
        ]))
        self.assertIn("pictures", sql)
        self.assertEqual(params[5], [
            "https://example.com/a.png",
            "https://example.com/b.png",
        ])

    def test_missing_pictures_key_writes_empty_array(self):
        sql, params = self._save(self._parsed())
        self.assertIn("pictures = EXCLUDED.pictures", sql)
        self.assertEqual(params[5], [])


class QueueClaimTest(unittest.TestCase):

    def _state(self):
        state = FakeState()
        state.add(url="https://blocked.example/a", priority=500)
        state.add(url="https://other.example/b", priority=100)
        state.add(url="https://blocked.example/c", priority=400)
        return state

    def _claim(self, state, scheduler=None):
        scheduler = scheduler or crawler.HostScheduler()
        return crawler.claim_next_queue_item(
            FakeConnection(state), scheduler, StubRobots()
        )

    def test_highest_priority_ready_host_wins(self):
        item, wait_hint = self._claim(self._state())
        self.assertEqual(item[1], "https://blocked.example/a")
        self.assertIsNone(wait_hint)

    def test_blocked_host_is_skipped_for_other_domain(self):
        scheduler = crawler.HostScheduler()
        scheduler.blocked_until["blocked.example"] = time.monotonic() + 900
        item, wait_hint = self._claim(self._state(), scheduler)
        self.assertEqual(item[1], "https://other.example/b")
        # The blocked host is filtered out by the query itself, so the
        # claim never had to look at it; no wait hint is needed.
        self.assertIsNone(wait_hint)

    def test_rate_limited_host_is_skipped(self):
        scheduler = crawler.HostScheduler()
        # Was just crawled with the 3s per-host delay of StubRobots.
        scheduler.wait("https://blocked.example/a", 3.0)
        item, _wait_hint = self._claim(self._state(), scheduler)
        self.assertEqual(item[1], "https://other.example/b")

    def test_only_blocked_hosts_left_reports_wait_hint(self):
        state = FakeState()
        state.add(url="https://blocked.example/a", priority=500)
        scheduler = crawler.HostScheduler()
        scheduler.blocked_until["blocked.example"] = time.monotonic() + 900
        item, wait_hint = self._claim(state, scheduler)
        self.assertIsNone(item)
        self.assertGreater(wait_hint, 800)

    def test_empty_queue(self):
        item, wait_hint = self._claim(FakeState())
        self.assertIsNone(item)
        self.assertIsNone(wait_hint)

    def test_claim_marks_processing_and_counts_attempt(self):
        state = self._state()
        self._claim(state)
        self.assertEqual(state.rows[0]["status"], "processing")
        self.assertEqual(state.rows[0]["attempts"], 1)
        self.assertEqual(state.rows[1]["status"], "pending")

    def test_claim_finds_ready_host_behind_blocked_wall(self):
        state = FakeState()
        for index in range(30):
            state.add(
                url="https://blocked.example/%d" % index,
                priority=300 - index,
            )
        state.add(url="https://ready.example/x", priority=10)
        scheduler = crawler.HostScheduler()
        scheduler.blocked_until["blocked.example"] = time.monotonic() + 900
        item, _wait_hint = self._claim(state, scheduler)
        self.assertEqual(item[1], "https://ready.example/x")

    def test_blocked_wall_larger_than_scan_window_does_not_starve_queue(self):
        # Regression: 30000+ hosts were queued, but every worker kept
        # reporting "All candidate hosts busy or blocked". A blocked
        # host with more pending URLs than the scan window sits at the
        # head of the priority order forever, hiding the rest of the
        # queue. The busy host must be excluded in SQL, not just skipped
        # within the small candidate window.
        state = FakeState()
        wall = crawler.QUEUE_CLAIM_SCAN_LIMIT + 50
        for index in range(wall):
            state.add(
                url="https://blocked.example/%d" % index,
                priority=500 - index % 500,
            )
        state.add(url="https://ready.example/x", priority=10)

        scheduler = crawler.HostScheduler()
        scheduler.blocked_until["blocked.example"] = time.monotonic() + 900

        item, wait_hint = self._claim(state, scheduler)
        self.assertEqual(item[1], "https://ready.example/x")
        self.assertIsNone(wait_hint)

    def test_busy_hosts_are_excluded_from_the_claim_query(self):
        state = self._state()
        scheduler = crawler.HostScheduler()
        scheduler.blocked_until["blocked.example"] = time.monotonic() + 900
        self.assertEqual(
            set(scheduler.busy_hosts()), {"blocked.example"}
        )
        item, _wait_hint = self._claim(state, scheduler)
        self.assertEqual(item[1], "https://other.example/b")

    def test_defer_does_not_burn_an_attempt(self):
        state = FakeState()
        queue_id = state.add(
            url="https://slow.example/a",
            status="processing",
            attempts=1,
        )
        crawler.defer_queue_item(FakeConnection(state), queue_id)
        row = state.rows[0]
        self.assertEqual(row["status"], "pending")
        self.assertEqual(row["attempts"], 0)

    def test_requeue_retries_below_attempt_cap(self):
        state = FakeState()
        queue_id = state.add(
            url="https://x.example/a",
            status="processing",
            attempts=1,
        )
        status = crawler.requeue_queue_item(FakeConnection(state), queue_id)
        self.assertEqual(status, "pending")
        self.assertEqual(state.rows[0]["status"], "pending")

    def test_requeue_fails_after_attempt_cap(self):
        state = FakeState()
        queue_id = state.add(
            url="https://x.example/a",
            status="processing",
            attempts=crawler.QUEUE_MAX_ATTEMPTS,
        )
        status = crawler.requeue_queue_item(FakeConnection(state), queue_id)
        self.assertEqual(status, "failed")
        self.assertEqual(state.rows[0]["status"], "failed")


class HostFilterTest(unittest.TestCase):

    def test_localhost_and_private_suffixes_are_blocked(self):
        for host in (
            "localhost", "sub.localhost", "localhost.localdomain",
            "myserver.local", "app.internal",
        ):
            self.assertTrue(
                crawler.is_non_crawlable_host(host), host
            )

    def test_ip_literals_are_blocked(self):
        for host in (
            "127.0.0.1", "10.0.0.5", "192.168.1.10", "172.16.0.1",
            "8.8.8.8", "::1", "fe80::1", "2001:db8::1",
            "::ffff:127.0.0.1",
        ):
            self.assertTrue(
                crawler.is_non_crawlable_host(host), host
            )

    def test_single_label_and_empty_names_are_blocked(self):
        for host in ("mymachine", "", "   ", "."):
            self.assertTrue(
                crawler.is_non_crawlable_host(host), repr(host)
            )

    def test_real_domains_are_allowed(self):
        for host in (
            "example.com", "www.example.com",
            "sub.domain.example.co.uk", "xn--bcher-kva.example",
            "example.com.",
        ):
            self.assertFalse(
                crawler.is_non_crawlable_host(host), host
            )

    def test_normalize_url_rejects_non_public_targets(self):
        for url in (
            "http://localhost/x",
            "http://127.0.0.1/x",
            "http://[::1]:8080/x",
            "http://10.1.2.3/a?b=c",
            "http://intranet.local/x",
            "http://host/x",
        ):
            self.assertIsNone(crawler.normalize_url(url), url)

    def test_normalize_url_keeps_real_domains(self):
        self.assertEqual(
            crawler.normalize_url("http://www.example.com/x?y=1"),
            "http://www.example.com/x?y=1",
        )

    def test_is_non_crawlable_url_handles_malformed_urls(self):
        self.assertTrue(crawler.is_non_crawlable_url("http://localhost/x"))
        self.assertTrue(crawler.is_non_crawlable_url("http://[::1]/"))
        self.assertFalse(crawler.is_non_crawlable_url("https://example.com/"))

    def test_malformed_url_is_treated_as_non_crawlable(self):
        # urlparse raises ValueError on malformed IPv6 brackets; the
        # filter must not crash on it.
        self.assertTrue(
            crawler.is_non_crawlable_url("http://[::1")
        )


if __name__ == "__main__":
    unittest.main()