Skip over-long URLs before inserting into crawl_queue

Extremely long URLs exceeded the PostgreSQL B-tree index row limit
(~2704 bytes) on crawl_queue.url and crashed workers with
ProgramLimitExceeded. queue_urls() now drops URLs whose UTF-8
representation exceeds MAX_QUEUE_URL_BYTES (2000), logs a warning with
the skipped count and limit, and returns 0 early when nothing remains.
The INSERT with ON CONFLICT (url) DO NOTHING and the DB schema are
unchanged, so other crawler instances stay compatible.

🤖 Generated with Codebuff
Co-Authored-By: Codebuff <noreply@codebuff.com>

crawler.py

@@ -124,6 +124,13 @@ MAX_HEADING_LENGTH = 1000
 MAX_TEXT_LENGTH = 2_000_000
 MAX_LINKS = 10_000
 
+# Maximum size of a URL (UTF-8 encoded) that may be inserted into the
+# crawl queue. PostgreSQL B-tree index entries are limited to about 2704
+# bytes, so extremely long URLs would abort the INSERT with
+# ProgramLimitExceeded and crash the worker. Filtering them out before
+# the INSERT keeps the crawl running without touching the DB schema.
+MAX_QUEUE_URL_BYTES = 2000
+
 
 # ============================================================
 # LOGGING / SHUTDOWN
@@ -251,7 +258,25 @@ def queue_urls(conn, links, source_url):
     safe_links = clean_text_list(links)
     safe_source = clean_text(source_url)
 
-    rows = [(url, safe_source, 0, "pending") for url in safe_links]
+    # Skip URLs that are too large for the UNIQUE B-tree index on
+    # crawl_queue.url (btree entries are limited to ~2704 bytes).
+    filtered_links = [
+        url for url in safe_links
+        if len(url.encode("utf-8")) <= MAX_QUEUE_URL_BYTES
+    ]
+
+    skipped = len(safe_links) - len(filtered_links)
+    if skipped:
+        logging.warning(
+            "[QUEUE] %d URLs skipped because they exceed %d bytes",
+            skipped,
+            MAX_QUEUE_URL_BYTES,
+        )
+
+    if not filtered_links:
+        return 0
+
+    rows = [(url, safe_source, 0, "pending") for url in filtered_links]
 
     with conn.cursor() as cur:
         execute_values(