diff --git a/crawler.py b/crawler.py index 8b711d7..0996ea9 100644 --- a/crawler.py +++ b/crawler.py @@ -124,6 +124,13 @@ MAX_HEADING_LENGTH = 1000 MAX_TEXT_LENGTH = 2_000_000 MAX_LINKS = 10_000 +# Maximum size of a URL (UTF-8 encoded) that may be inserted into the +# crawl queue. PostgreSQL B-tree index entries are limited to about 2704 +# bytes, so extremely long URLs would abort the INSERT with +# ProgramLimitExceeded and crash the worker. Filtering them out before +# the INSERT keeps the crawl running without touching the DB schema. +MAX_QUEUE_URL_BYTES = 2000 + # ============================================================ # LOGGING / SHUTDOWN @@ -251,7 +258,25 @@ def queue_urls(conn, links, source_url): safe_links = clean_text_list(links) safe_source = clean_text(source_url) - rows = [(url, safe_source, 0, "pending") for url in safe_links] + # Skip URLs that are too large for the UNIQUE B-tree index on + # crawl_queue.url (btree entries are limited to ~2704 bytes). + filtered_links = [ + url for url in safe_links + if len(url.encode("utf-8")) <= MAX_QUEUE_URL_BYTES + ] + + skipped = len(safe_links) - len(filtered_links) + if skipped: + logging.warning( + "[QUEUE] %d URLs skipped because they exceed %d bytes", + skipped, + MAX_QUEUE_URL_BYTES, + ) + + if not filtered_links: + return 0 + + rows = [(url, safe_source, 0, "pending") for url in filtered_links] with conn.cursor() as cur: execute_values(