From 0adf9eb42c3a236ca48e62701f94cafdba77e006 Mon Sep 17 00:00:00 2001 From: jhkaktus Date: Sun, 6 Sep 2026 10:42:19 +0200 Subject: [PATCH] Skip over-long URLs before inserting into crawl_queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extremely long URLs exceeded the PostgreSQL B-tree index row limit (~2704 bytes) on crawl_queue.url and crashed workers with ProgramLimitExceeded. queue_urls() now drops URLs whose UTF-8 representation exceeds MAX_QUEUE_URL_BYTES (2000), logs a warning with the skipped count and limit, and returns 0 early when nothing remains. The INSERT with ON CONFLICT (url) DO NOTHING and the DB schema are unchanged, so other crawler instances stay compatible. 🤖 Generated with Codebuff Co-Authored-By: Codebuff --- crawler.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/crawler.py b/crawler.py index 8b711d7..0996ea9 100644 --- a/crawler.py +++ b/crawler.py @@ -124,6 +124,13 @@ MAX_HEADING_LENGTH = 1000 MAX_TEXT_LENGTH = 2_000_000 MAX_LINKS = 10_000 +# Maximum size of a URL (UTF-8 encoded) that may be inserted into the +# crawl queue. PostgreSQL B-tree index entries are limited to about 2704 +# bytes, so extremely long URLs would abort the INSERT with +# ProgramLimitExceeded and crash the worker. Filtering them out before +# the INSERT keeps the crawl running without touching the DB schema. +MAX_QUEUE_URL_BYTES = 2000 + # ============================================================ # LOGGING / SHUTDOWN @@ -251,7 +258,25 @@ def queue_urls(conn, links, source_url): safe_links = clean_text_list(links) safe_source = clean_text(source_url) - rows = [(url, safe_source, 0, "pending") for url in safe_links] + # Skip URLs that are too large for the UNIQUE B-tree index on + # crawl_queue.url (btree entries are limited to ~2704 bytes). + filtered_links = [ + url for url in safe_links + if len(url.encode("utf-8")) <= MAX_QUEUE_URL_BYTES + ] + + skipped = len(safe_links) - len(filtered_links) + if skipped: + logging.warning( + "[QUEUE] %d URLs skipped because they exceed %d bytes", + skipped, + MAX_QUEUE_URL_BYTES, + ) + + if not filtered_links: + return 0 + + rows = [(url, safe_source, 0, "pending") for url in filtered_links] with conn.cursor() as cur: execute_values(