fix limits

crawler.py

@@ -192,6 +192,20 @@ BLOCKED_PREFIXES = (
     "file:",
 )
 
+# Pages with no search-index value that only burn bandwidth, CPU and
+# index space: pkg.go.dev's reverse-dependency tabs are megabyte-sized
+# link lists (10k+ links of pure package names). These URLs are
+# skipped before fetching; the queue item is marked done so it is not
+# retried endlessly.
+SKIP_URL_PATTERNS = tuple(
+    re.compile(pattern)
+    for pattern in (
+        # Query part (not fragment), then the tab value, then only a
+        # fragment or the end may follow.
+        r"^https?://pkg\.go\.dev/[^#]*[?&]tab=importedby(#|$)",
+    )
+)
+
 # Keep generated pages reasonably useful for search.
 MAX_TITLE_LENGTH = 1000
 MAX_HEADINGS = 200
@@ -199,6 +213,15 @@ MAX_HEADING_LENGTH = 1000
 MAX_TEXT_LENGTH = 2_000_000
 MAX_LINKS = 10_000
 
+# Cap for the text that feeds the full-text index. PostgreSQL
+# hard-limits a single tsvector value to 1,048,575 bytes (not
+# configurable), and the trigger pages_search_vector_update() builds
+# the vector from the content column. Content near MAX_TEXT_LENGTH
+# would overflow that limit and abort the whole INSERT with
+# ProgramLimitExceeded. 200,000 characters are always safe: even with
+# 4-byte UTF-8 characters this stays well below 1,048,575 bytes.
+CONTENT_INDEX_MAX_CHARS = 200_000
+
 # Maximum size of a URL (UTF-8 encoded) that may be inserted into the
 # crawl queue. PostgreSQL B-tree index entries are limited to about 2704
 # bytes, so extremely long URLs would abort the INSERT with
@@ -519,6 +542,11 @@ def safe_commit(conn, operation=None):
        raise UpdateRequired: the caller's writes are discarded
        cleanly, nothing is committed half-done.
     2. Otherwise run the writes (operation) and commit them.
+    3. If the operation hits ProgramLimitExceeded (e.g. a tsvector
+       overflow - PostgreSQL hard-limits one tsvector value to
+       1,048,575 bytes), roll the aborted transaction back and
+       re-raise, so the caller can skip the work item cleanly
+       instead of dying on a poisoned connection.
 
     Every crawler commit that does real work goes through here
     (page saves, queue inserts, queue claim/finish, stats), so a
@@ -535,7 +563,22 @@ def safe_commit(conn, operation=None):
             "new deployment generation active; update required"
         )
 
-    result = operation() if operation is not None else None
+    try:
+        result = operation() if operation is not None else None
+    except psycopg2.errors.ProgramLimitExceeded:
+        # The failed transaction has poisoned the connection ("current
+        # transaction is aborted, commands ignored until end of
+        # transaction block"); roll it back so the connection stays
+        # usable, then let the caller decide (worker: skip the page,
+        # queue_urls: retry the batch). Without the rollback the very
+        # next statement on this connection would fail as well.
+        try:
+            conn.rollback()
+        except Exception:
+            logging.exception(
+                "Rollback after ProgramLimitExceeded failed."
+            )
+        raise
     conn.commit()
     return result
 
@@ -1396,6 +1439,15 @@ class RateLimiter:
         self.last_request[host] = time.monotonic()
 
 
+def is_skipped_url(url):
+    """
+    True for URLs whose pages bring no search-index value but burn
+    significant crawl resources (bandwidth, parsing time, index
+    size). See SKIP_URL_PATTERNS.
+    """
+    return any(pattern.match(url) for pattern in SKIP_URL_PATTERNS)
+
+
 def fetch(url, limiter, robots):
     parsed = urlparse(url)
 
@@ -1565,6 +1617,25 @@ def save_page(conn, url, source_url, status, content_type, parsed, crawl_ms):
         "tld": clean_text(parsed["tld"]),
     }
 
+    # Cap the text that feeds the full-text index. The DB trigger
+    # pages_search_vector_update() builds a tsvector from the content
+    # column and PostgreSQL hard-limits one tsvector value to
+    # 1,048,575 bytes (not configurable). Pages near MAX_TEXT_LENGTH
+    # overflow it and abort the whole INSERT (ProgramLimitExceeded),
+    # which used to kill the worker. Capping here keeps oversize
+    # pages saveable (with a truncated index); the trigger-side cap
+    # (fix_pages_search_vector_trigger.sql) adds the same protection
+    # for every other insert path, independent of this code.
+    index_content = safe_parsed["content"][:CONTENT_INDEX_MAX_CHARS]
+    if len(index_content) < len(safe_parsed["content"]):
+        logging.warning(
+            "[SAVE ] %s: content capped for the search index "
+            "(%d -> %d chars).",
+            url,
+            len(safe_parsed["content"]),
+            len(index_content),
+        )
+
     with conn.cursor() as cur:
         cur.execute("""
             INSERT INTO pages (
@@ -1609,7 +1680,7 @@ def save_page(conn, url, source_url, status, content_type, parsed, crawl_ms):
             url,
             safe_parsed["title"],
             safe_parsed["headings"],
-            safe_parsed["content"],
+            index_content,
             safe_parsed["links"],
             safe_parsed["domain"],
             safe_parsed["tld"],
@@ -1685,6 +1756,20 @@ def worker(worker_id):
             logging.info("[CRAWL] %s", url)
             logging.info("[FROM ] %s", source_url or "-")
 
+            # Known no-value pages (e.g. pkg.go.dev "?tab=importedby":
+            # megabyte-sized reverse-dependency link lists) are
+            # skipped before fetching. Marked 'done' so they are not
+            # retried endlessly.
+            if is_skipped_url(url):
+                logging.info("[SKIP ] %s (no search value)", url)
+                safe_commit(
+                    conn,
+                    lambda: finish_queue_item(
+                        conn, queue_id, success=True
+                    ),
+                )
+                continue
+
             started = time.monotonic()
 
             status, html, error = fetch(url, limiter, robots)
@@ -1734,18 +1819,41 @@ def worker(worker_id):
             logging.info("[HEADS ] %d", len(parsed["headings"]))
             logging.info("[LINKS ] %d gefunden", len(parsed["links"]))
 
-            safe_commit(
-                conn,
-                lambda: save_page(
-                    conn=conn,
-                    url=url,
-                    source_url=source_url,
-                    status=status,
-                    content_type="text/html",
-                    parsed=parsed,
-                    crawl_ms=crawl_ms,
-                ),
-            )
+            try:
+                safe_commit(
+                    conn,
+                    lambda: save_page(
+                        conn=conn,
+                        url=url,
+                        source_url=source_url,
+                        status=status,
+                        content_type="text/html",
+                        parsed=parsed,
+                        crawl_ms=crawl_ms,
+                    ),
+                )
+            except psycopg2.errors.ProgramLimitExceeded as exc:
+                # Defense-in-depth: save_page() caps the content before
+                # the insert, so this can only fire if the DB-side
+                # trigger overflows anyway (e.g. because
+                # fix_pages_search_vector_trigger.sql has not been
+                # applied yet). safe_commit() has already rolled the
+                # transaction back; mark the queue item failed so the
+                # page is not retried endlessly, and keep the worker
+                # alive.
+                logging.warning(
+                    "[SAVE ] %s: not indexable, skipped (%s)",
+                    url,
+                    exc,
+                )
+                safe_commit(
+                    conn,
+                    lambda: finish_queue_item(
+                        conn, queue_id, success=False
+                    ),
+                )
+                failed += 1
+                continue
 
             inserted = queue_urls(
                 conn,