From 4315f1fdb3eb9c10e2603c536df01969896e920e Mon Sep 17 00:00:00 2001 From: jhkaktus Date: Sun, 13 Sep 2026 18:01:58 +0200 Subject: [PATCH] fix limits --- README.md | 31 +++++++ crawler.py | 136 +++++++++++++++++++++++++--- fix_pages_search_vector_trigger.sql | 56 ++++++++++++ 3 files changed, 209 insertions(+), 14 deletions(-) create mode 100644 fix_pages_search_vector_trigger.sql diff --git a/README.md b/README.md index b7bdf3c..7eeca71 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,37 @@ das komplette serverjonas search - `launcher.py` – startet/überwacht den Crawler und rollt Updates aus (siehe unten) - `search/` – Suchmaschine darüber (CLI + HTTP-API), siehe [search/README.md](search/README.md) - `web/` – Statistik-Webseite +- `fix_pages_search_vector_trigger.sql` – einmalig laufen lassen: kappelt den Text, + der im DB-Trigger `pages_search_vector_update()` in `to_tsvector()` einfließt + (Postgres hartes Limit: 1.048.575 Bytes pro `tsvector`) + +## Sehr große Seiten / tsvector-Limit + +PostgreSQL begrenzt einen einzelnen `tsvector`-Wert hart auf 1.048.575 Bytes +(nicht konfigurierbar). Der Trigger `pages_search_vector_update()` baut diesen +aus der `content`-Spalte – Seiten mit sehr viel Text (beobachtet: ~2 MB, z. B. +`pkg.go.dev/...?tab=importedby`) haben den Insert dadurch mit +`ProgramLimitExceeded` abgebrochen und alle Worker crashen lassen. + +Dagegen wirken drei Maßnahmen zusammen: + +1. **DB-Trigger (empfohlen, einmalig einspielen):** + `fix_pages_search_vector_trigger.sql` ersetzt die Trigger-Funktion so, dass + nur die ersten 200.000 Zeichen in `to_tsvector()` einfließen. Der volle Text + bleibt in `content` gespeichert, nur der Suchindex ist gekürzt. Vor dem + Einspielen die eigene Funktions-Version prüfen: + `SELECT prosrc FROM pg_proc WHERE proname = 'pages_search_vector_update';` +2. **Im Crawler:** `save_page()` kappelt den Inhalt vor dem Insert zusätzlich + auf `CONTENT_INDEX_MAX_CHARS` (200.000 Zeichen) – wirkt sofort, auch ohne + Trigger-Fix. Überlange Seiten werden dann normal gespeichert. +3. **Fehlerbehandlung:** Sollte der Insert trotzdem an einem Datenbank-Limit + scheitern, fängt der Worker `ProgramLimitExceeded` ab, loggt die URL als + Warnung, rollt die Transaktion zurück, markiert den Queue-Eintrag als + `failed` (kein endloses Wiederholen) und läuft weiter, statt zu sterben. + +Zusätzlich werden bekannt wertlose URLs (`pkg.go.dev/...?tab=importedby` – +Megabyte-große reine Rückverweis-Listen) über `SKIP_URL_PATTERNS` schon vor dem +Fetch übersprungen und als `done` markiert. ## Rolling Updates (Generation-Mechanismus) diff --git a/crawler.py b/crawler.py index 337cd7c..ad415b4 100644 --- a/crawler.py +++ b/crawler.py @@ -192,6 +192,20 @@ BLOCKED_PREFIXES = ( "file:", ) +# Pages with no search-index value that only burn bandwidth, CPU and +# index space: pkg.go.dev's reverse-dependency tabs are megabyte-sized +# link lists (10k+ links of pure package names). These URLs are +# skipped before fetching; the queue item is marked done so it is not +# retried endlessly. +SKIP_URL_PATTERNS = tuple( + re.compile(pattern) + for pattern in ( + # Query part (not fragment), then the tab value, then only a + # fragment or the end may follow. + r"^https?://pkg\.go\.dev/[^#]*[?&]tab=importedby(#|$)", + ) +) + # Keep generated pages reasonably useful for search. MAX_TITLE_LENGTH = 1000 MAX_HEADINGS = 200 @@ -199,6 +213,15 @@ MAX_HEADING_LENGTH = 1000 MAX_TEXT_LENGTH = 2_000_000 MAX_LINKS = 10_000 +# Cap for the text that feeds the full-text index. PostgreSQL +# hard-limits a single tsvector value to 1,048,575 bytes (not +# configurable), and the trigger pages_search_vector_update() builds +# the vector from the content column. Content near MAX_TEXT_LENGTH +# would overflow that limit and abort the whole INSERT with +# ProgramLimitExceeded. 200,000 characters are always safe: even with +# 4-byte UTF-8 characters this stays well below 1,048,575 bytes. +CONTENT_INDEX_MAX_CHARS = 200_000 + # Maximum size of a URL (UTF-8 encoded) that may be inserted into the # crawl queue. PostgreSQL B-tree index entries are limited to about 2704 # bytes, so extremely long URLs would abort the INSERT with @@ -519,6 +542,11 @@ def safe_commit(conn, operation=None): raise UpdateRequired: the caller's writes are discarded cleanly, nothing is committed half-done. 2. Otherwise run the writes (operation) and commit them. + 3. If the operation hits ProgramLimitExceeded (e.g. a tsvector + overflow - PostgreSQL hard-limits one tsvector value to + 1,048,575 bytes), roll the aborted transaction back and + re-raise, so the caller can skip the work item cleanly + instead of dying on a poisoned connection. Every crawler commit that does real work goes through here (page saves, queue inserts, queue claim/finish, stats), so a @@ -535,7 +563,22 @@ def safe_commit(conn, operation=None): "new deployment generation active; update required" ) - result = operation() if operation is not None else None + try: + result = operation() if operation is not None else None + except psycopg2.errors.ProgramLimitExceeded: + # The failed transaction has poisoned the connection ("current + # transaction is aborted, commands ignored until end of + # transaction block"); roll it back so the connection stays + # usable, then let the caller decide (worker: skip the page, + # queue_urls: retry the batch). Without the rollback the very + # next statement on this connection would fail as well. + try: + conn.rollback() + except Exception: + logging.exception( + "Rollback after ProgramLimitExceeded failed." + ) + raise conn.commit() return result @@ -1396,6 +1439,15 @@ class RateLimiter: self.last_request[host] = time.monotonic() +def is_skipped_url(url): + """ + True for URLs whose pages bring no search-index value but burn + significant crawl resources (bandwidth, parsing time, index + size). See SKIP_URL_PATTERNS. + """ + return any(pattern.match(url) for pattern in SKIP_URL_PATTERNS) + + def fetch(url, limiter, robots): parsed = urlparse(url) @@ -1565,6 +1617,25 @@ def save_page(conn, url, source_url, status, content_type, parsed, crawl_ms): "tld": clean_text(parsed["tld"]), } + # Cap the text that feeds the full-text index. The DB trigger + # pages_search_vector_update() builds a tsvector from the content + # column and PostgreSQL hard-limits one tsvector value to + # 1,048,575 bytes (not configurable). Pages near MAX_TEXT_LENGTH + # overflow it and abort the whole INSERT (ProgramLimitExceeded), + # which used to kill the worker. Capping here keeps oversize + # pages saveable (with a truncated index); the trigger-side cap + # (fix_pages_search_vector_trigger.sql) adds the same protection + # for every other insert path, independent of this code. + index_content = safe_parsed["content"][:CONTENT_INDEX_MAX_CHARS] + if len(index_content) < len(safe_parsed["content"]): + logging.warning( + "[SAVE ] %s: content capped for the search index " + "(%d -> %d chars).", + url, + len(safe_parsed["content"]), + len(index_content), + ) + with conn.cursor() as cur: cur.execute(""" INSERT INTO pages ( @@ -1609,7 +1680,7 @@ def save_page(conn, url, source_url, status, content_type, parsed, crawl_ms): url, safe_parsed["title"], safe_parsed["headings"], - safe_parsed["content"], + index_content, safe_parsed["links"], safe_parsed["domain"], safe_parsed["tld"], @@ -1685,6 +1756,20 @@ def worker(worker_id): logging.info("[CRAWL] %s", url) logging.info("[FROM ] %s", source_url or "-") + # Known no-value pages (e.g. pkg.go.dev "?tab=importedby": + # megabyte-sized reverse-dependency link lists) are + # skipped before fetching. Marked 'done' so they are not + # retried endlessly. + if is_skipped_url(url): + logging.info("[SKIP ] %s (no search value)", url) + safe_commit( + conn, + lambda: finish_queue_item( + conn, queue_id, success=True + ), + ) + continue + started = time.monotonic() status, html, error = fetch(url, limiter, robots) @@ -1734,18 +1819,41 @@ def worker(worker_id): logging.info("[HEADS ] %d", len(parsed["headings"])) logging.info("[LINKS ] %d gefunden", len(parsed["links"])) - safe_commit( - conn, - lambda: save_page( - conn=conn, - url=url, - source_url=source_url, - status=status, - content_type="text/html", - parsed=parsed, - crawl_ms=crawl_ms, - ), - ) + try: + safe_commit( + conn, + lambda: save_page( + conn=conn, + url=url, + source_url=source_url, + status=status, + content_type="text/html", + parsed=parsed, + crawl_ms=crawl_ms, + ), + ) + except psycopg2.errors.ProgramLimitExceeded as exc: + # Defense-in-depth: save_page() caps the content before + # the insert, so this can only fire if the DB-side + # trigger overflows anyway (e.g. because + # fix_pages_search_vector_trigger.sql has not been + # applied yet). safe_commit() has already rolled the + # transaction back; mark the queue item failed so the + # page is not retried endlessly, and keep the worker + # alive. + logging.warning( + "[SAVE ] %s: not indexable, skipped (%s)", + url, + exc, + ) + safe_commit( + conn, + lambda: finish_queue_item( + conn, queue_id, success=False + ), + ) + failed += 1 + continue inserted = queue_urls( conn, diff --git a/fix_pages_search_vector_trigger.sql b/fix_pages_search_vector_trigger.sql new file mode 100644 index 0000000..30cc1bf --- /dev/null +++ b/fix_pages_search_vector_trigger.sql @@ -0,0 +1,56 @@ +-- fix_pages_search_vector_trigger.sql +-- +-- Caps the text that feeds the full-text index of the `pages` table. +-- +-- Problem: +-- PostgreSQL hard-limits a single tsvector value to 1,048,575 bytes +-- (not configurable). The trigger function +-- pages_search_vector_update() built the vector from the full +-- content column, so pages with very large text content (observed: +-- ~2 MB, e.g. pkg.go.dev "?tab=importedby" pages) aborted the whole +-- INSERT with ProgramLimitExceeded ("string is too long for +-- tsvector") and crashed all crawler workers. +-- +-- Fix: +-- Feed at most the first 200,000 characters of the content into +-- to_tsvector(). Even at worst-case 4 bytes per UTF-8 character this +-- stays far below the tsvector limit. The full text remains stored +-- in the content column - only the search index is truncated. +-- 200,000 characters cover the meaningful prose of essentially all +-- real pages; the oversized ones are link lists with no index value. +-- +-- Run once on the jonassearch database as its owner, e.g.: +-- psql -h 127.0.0.1 -p 25432 -U jonassearch -d jonassearch \ +-- -f fix_pages_search_vector_trigger.sql +-- +-- Safe to run repeatedly (CREATE OR REPLACE) and safe to run while the +-- crawler is running: existing rows keep their vectors, only newly +-- INSERTed/UPDATEd rows use the capped version. + +CREATE OR REPLACE FUNCTION pages_search_vector_update() +RETURNS trigger AS $$ +BEGIN + -- Cap the indexed text: a tsvector value may hold at most + -- 1,048,575 bytes. left(content, 200000) can never exceed that + -- (200000 chars * max 4 bytes UTF-8 = 800,000 bytes). + NEW.search_vector := + to_tsvector('simple', left(coalesce(NEW.content, ''), 200000)); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Notes for the DBA: +-- * If your existing function uses a different text search config +-- (e.g. 'german' or 'english') or indexes additional columns +-- (title, headings), port the left(..., 200000) cap into YOUR +-- function body instead of blindly replacing it - check first with: +-- SELECT prosrc FROM pg_proc WHERE proname = +-- 'pages_search_vector_update'; +-- * The trigger itself (CREATE TRIGGER pages_search_vector_update +-- BEFORE INSERT OR UPDATE ...) is unchanged; only the function body +-- above is replaced. +-- * Optional backfill of already-stored rows (this runs UPDATEs over +-- the whole table; run during low traffic): +-- UPDATE pages SET content = content +-- WHERE search_vector IS NULL +-- OR octet_length(search_vector::text) > 1048575;