add pictures

test_crawler.py

@@ -73,6 +73,12 @@ class FakeState:
         return row["id"]
 
 
+def _url_host(url):
+    """Mirror the SQL host expression used by the claim query."""
+    rest = url.split("://", 1)[1] if "://" in url else url
+    return rest.split("/", 1)[0].lower()
+
+
 class FakeCursor:
     """Just enough SQL surface for the crawler's queue functions."""
 
@@ -84,12 +90,20 @@ class FakeCursor:
         query = " ".join(query.split())
         params = params or ()
 
-        if "SELECT id, url, source_url" in query:
+        if "SELECT 1 FROM crawl_queue" in query:
+            self._rows = [
+                (1,) for row in self._state.rows
+                if row["status"] == "pending"
+            ][:1]
+        elif "SELECT id, url, source_url" in query:
             batch = params[-1]
-            exclude = set(params[0]) if len(params) == 2 else set()
+            exclude = set(params[0]) if len(params) == 3 else set()
+            excluded_hosts = set(params[1]) if len(params) == 3 else set()
             candidates = [
                 row for row in self._state.rows
-                if row["status"] == "pending" and row["id"] not in exclude
+                if row["status"] == "pending"
+                and row["id"] not in exclude
+                and _url_host(row["url"]) not in excluded_hosts
             ]
             candidates.sort(key=lambda row: (-row["priority"], row["id"]))
             self._rows = [
@@ -241,6 +255,35 @@ class HostSchedulerTest(unittest.TestCase):
             scheduler.wait_time("https://b.example/y", 3.0), 0.0
         )
 
+    def test_busy_hosts_reports_blocked_and_rate_limited_hosts(self):
+        scheduler = crawler.HostScheduler()
+        scheduler.blocked_until["blocked.example"] = time.monotonic() + 900
+        scheduler.wait("https://rate.example/x", 60.0)
+
+        busy = scheduler.busy_hosts()
+        self.assertGreater(busy["blocked.example"], 800)
+        self.assertGreater(busy["rate.example"], 50)
+
+    def test_busy_hosts_prunes_expired_entries(self):
+        scheduler = crawler.HostScheduler()
+        scheduler.blocked_until["a.example"] = time.monotonic() - 1
+        scheduler.last_request["b.example"] = time.monotonic() - 100
+        scheduler.host_delay["b.example"] = 3.0
+
+        self.assertEqual(scheduler.busy_hosts(), {})
+        self.assertNotIn("a.example", scheduler.blocked_until)
+        self.assertNotIn("b.example", scheduler.last_request)
+
+    def test_never_contacted_host_is_ready_even_with_large_delay(self):
+        # monotonic() has an arbitrary origin: "never contacted" must not
+        # be treated as last_request = 0, which would block a fresh host
+        # for the whole robots.txt crawl-delay on a freshly booted host.
+        scheduler = crawler.HostScheduler()
+        self.assertEqual(
+            scheduler.wait_time("https://slow.example/x", 86400.0, now=10.0),
+            0.0,
+        )
+
 
 class ErrorClassificationTest(unittest.TestCase):
 
@@ -282,6 +325,151 @@ class ErrorClassificationTest(unittest.TestCase):
         self.assertIsNone(crawler._retry_after_seconds(None))
 
 
+class PictureExtractionTest(unittest.TestCase):
+
+    BASE = "https://example.com/articles/page.html"
+
+    def _pictures(self, html):
+        return crawler.parse_html(html, self.BASE)["pictures"]
+
+    def test_img_src_relative_is_absolute_and_fragment_dropped(self):
+        self.assertEqual(
+            self._pictures('<img src="/images/a.jpg#frag">'),
+            ["https://example.com/images/a.jpg"],
+        )
+
+    def test_protocol_relative_external_image_and_dedupe(self):
+        pictures = self._pictures(
+            '<img src="//cdn.example.net/pic.png">'
+            '<img src="https://cdn.example.net/pic.png">'
+        )
+        self.assertEqual(
+            pictures, ["https://cdn.example.net/pic.png"]
+        )
+
+    def test_srcset_and_picture_sources(self):
+        pictures = self._pictures(
+            '<img src="/fallback.jpg" '
+            'srcset="/small.jpg 1x, /large.jpg 2x">'
+            '<picture><source srcset="/b.webp 1x, /b2.webp 2x"></picture>'
+        )
+        self.assertEqual(pictures, [
+            "https://example.com/fallback.jpg",
+            "https://example.com/small.jpg",
+            "https://example.com/large.jpg",
+            "https://example.com/b.webp",
+            "https://example.com/b2.webp",
+        ])
+
+    def test_srcset_data_url_does_not_create_bogus_candidates(self):
+        pictures = self._pictures(
+            '<img srcset="data:image/png;base64,AAAA 1x, /real.jpg 2x">'
+        )
+        self.assertEqual(pictures, ["https://example.com/real.jpg"])
+
+    def test_og_twitter_and_image_src(self):
+        pictures = self._pictures(
+            '<meta property="og:image" content="https://example.com/og.png">'
+            '<meta name="twitter:image" content="/tw.png">'
+            '<link rel="image_src" href="/link.png">'
+        )
+        self.assertEqual(pictures, [
+            "https://example.com/og.png",
+            "https://example.com/tw.png",
+            "https://example.com/link.png",
+        ])
+
+    def test_lazy_loading_attributes(self):
+        pictures = self._pictures(
+            '<img data-src="//cdn.example.org/lazy.webp" '
+            'data-lazy-src="/lazy2.jpg" data-original="/orig.jpg">'
+        )
+        self.assertEqual(pictures, [
+            "https://cdn.example.org/lazy.webp",
+            "https://example.com/lazy2.jpg",
+            "https://example.com/orig.jpg",
+        ])
+
+    def test_invalid_and_unsupported_sources_are_ignored(self):
+        pictures = self._pictures(
+            '<img src="data:image/png;base64,AAAA">'
+            '<img src="javascript:alert(1)">'
+            '<img src="ftp://example.com/x.png">'
+            '<img src="">'
+            '<img src="   ">'
+            '<img>'
+        )
+        self.assertEqual(pictures, [])
+
+    def test_non_public_image_hosts_are_ignored(self):
+        pictures = self._pictures(
+            '<img src="http://localhost/a.png">'
+            '<img src="http://127.0.0.1/b.png">'
+        )
+        self.assertEqual(pictures, [])
+
+    def test_page_without_images(self):
+        self.assertEqual(
+            self._pictures("<html><body><p>keine Bilder</p></body></html>"),
+            [],
+        )
+
+    def test_existing_fields_still_parsed(self):
+        parsed = crawler.parse_html(
+            '<title>Seite</title><h1>Überschrift</h1>'
+            '<a href="/x">x</a><img src="/i.png">',
+            self.BASE,
+        )
+        self.assertEqual(parsed["title"], "Seite")
+        self.assertEqual(parsed["headings"], ["Überschrift"])
+        self.assertEqual(parsed["links"], ["https://example.com/x"])
+        self.assertEqual(parsed["pictures"], ["https://example.com/i.png"])
+
+
+class SavePagePicturesTest(unittest.TestCase):
+
+    @staticmethod
+    def _parsed(pictures=None):
+        parsed = {
+            "title": "Titel",
+            "headings": [],
+            "content": "Inhalt",
+            "links": [],
+            "domain": "example.com",
+            "tld": ".com",
+        }
+        if pictures is not None:
+            parsed["pictures"] = pictures
+        return parsed
+
+    def _save(self, parsed):
+        conn = mock.MagicMock()
+        cur = conn.cursor.return_value.__enter__.return_value
+        crawler.save_page(
+            conn, "https://example.com/", None, 200, "text/html",
+            parsed, 12,
+        )
+        sql, params = cur.execute.call_args[0]
+        return sql, params
+
+    def test_pictures_are_written_and_deduplicated(self):
+        sql, params = self._save(self._parsed([
+            "https://example.com/a.png",
+            "https://example.com/a.png",
+            "https://example.com/b.png",
+        ]))
+        self.assertIn("pictures", sql)
+        self.assertEqual(params[5], [
+            "https://example.com/a.png",
+            "https://example.com/b.png",
+        ])
+
+    def test_missing_pictures_key_writes_empty_array(self):
+        sql, params = self._save(self._parsed())
+        self.assertIn("pictures = EXCLUDED.pictures", sql)
+        self.assertEqual(params[5], [])
+
+
 class QueueClaimTest(unittest.TestCase):
 
     def _state(self):
@@ -307,11 +495,14 @@ class QueueClaimTest(unittest.TestCase):
         scheduler.blocked_until["blocked.example"] = time.monotonic() + 900
         item, wait_hint = self._claim(self._state(), scheduler)
         self.assertEqual(item[1], "https://other.example/b")
-        self.assertGreater(wait_hint, 800)
+        # The blocked host is filtered out by the query itself, so the
+        # claim never had to look at it; no wait hint is needed.
+        self.assertIsNone(wait_hint)
 
     def test_rate_limited_host_is_skipped(self):
         scheduler = crawler.HostScheduler()
-        scheduler.wait("https://blocked.example/a", 0.0)
+        # Was just crawled with the 3s per-host delay of StubRobots.
+        scheduler.wait("https://blocked.example/a", 3.0)
         item, _wait_hint = self._claim(self._state(), scheduler)
         self.assertEqual(item[1], "https://other.example/b")
 
@@ -349,6 +540,39 @@ class QueueClaimTest(unittest.TestCase):
         item, _wait_hint = self._claim(state, scheduler)
         self.assertEqual(item[1], "https://ready.example/x")
 
+    def test_blocked_wall_larger_than_scan_window_does_not_starve_queue(self):
+        # Regression: 30000+ hosts were queued, but every worker kept
+        # reporting "All candidate hosts busy or blocked". A blocked
+        # host with more pending URLs than the scan window sits at the
+        # head of the priority order forever, hiding the rest of the
+        # queue. The busy host must be excluded in SQL, not just skipped
+        # within the small candidate window.
+        state = FakeState()
+        wall = crawler.QUEUE_CLAIM_SCAN_LIMIT + 50
+        for index in range(wall):
+            state.add(
+                url="https://blocked.example/%d" % index,
+                priority=500 - index % 500,
+            )
+        state.add(url="https://ready.example/x", priority=10)
+
+        scheduler = crawler.HostScheduler()
+        scheduler.blocked_until["blocked.example"] = time.monotonic() + 900
+
+        item, wait_hint = self._claim(state, scheduler)
+        self.assertEqual(item[1], "https://ready.example/x")
+        self.assertIsNone(wait_hint)
+
+    def test_busy_hosts_are_excluded_from_the_claim_query(self):
+        state = self._state()
+        scheduler = crawler.HostScheduler()
+        scheduler.blocked_until["blocked.example"] = time.monotonic() + 900
+        self.assertEqual(
+            set(scheduler.busy_hosts()), {"blocked.example"}
+        )
+        item, _wait_hint = self._claim(state, scheduler)
+        self.assertEqual(item[1], "https://other.example/b")
+
     def test_defer_does_not_burn_an_attempt(self):
         state = FakeState()
         queue_id = state.add(