fix limits

fix_pages_search_vector_trigger.sql

@@ -0,0 +1,56 @@
+-- fix_pages_search_vector_trigger.sql
+--
+-- Caps the text that feeds the full-text index of the `pages` table.
+--
+-- Problem:
+--   PostgreSQL hard-limits a single tsvector value to 1,048,575 bytes
+--   (not configurable). The trigger function
+--   pages_search_vector_update() built the vector from the full
+--   content column, so pages with very large text content (observed:
+--   ~2 MB, e.g. pkg.go.dev "?tab=importedby" pages) aborted the whole
+--   INSERT with ProgramLimitExceeded ("string is too long for
+--   tsvector") and crashed all crawler workers.
+--
+-- Fix:
+--   Feed at most the first 200,000 characters of the content into
+--   to_tsvector(). Even at worst-case 4 bytes per UTF-8 character this
+--   stays far below the tsvector limit. The full text remains stored
+--   in the content column - only the search index is truncated.
+--   200,000 characters cover the meaningful prose of essentially all
+--   real pages; the oversized ones are link lists with no index value.
+--
+-- Run once on the jonassearch database as its owner, e.g.:
+--   psql -h 127.0.0.1 -p 25432 -U jonassearch -d jonassearch \
+--       -f fix_pages_search_vector_trigger.sql
+--
+-- Safe to run repeatedly (CREATE OR REPLACE) and safe to run while the
+-- crawler is running: existing rows keep their vectors, only newly
+-- INSERTed/UPDATEd rows use the capped version.
+
+CREATE OR REPLACE FUNCTION pages_search_vector_update()
+RETURNS trigger AS $$
+BEGIN
+    -- Cap the indexed text: a tsvector value may hold at most
+    -- 1,048,575 bytes. left(content, 200000) can never exceed that
+    -- (200000 chars * max 4 bytes UTF-8 = 800,000 bytes).
+    NEW.search_vector :=
+        to_tsvector('simple', left(coalesce(NEW.content, ''), 200000));
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Notes for the DBA:
+-- * If your existing function uses a different text search config
+--   (e.g. 'german' or 'english') or indexes additional columns
+--   (title, headings), port the left(..., 200000) cap into YOUR
+--   function body instead of blindly replacing it - check first with:
+--     SELECT prosrc FROM pg_proc WHERE proname =
+--         'pages_search_vector_update';
+-- * The trigger itself (CREATE TRIGGER pages_search_vector_update
+--   BEFORE INSERT OR UPDATE ...) is unchanged; only the function body
+--   above is replaced.
+-- * Optional backfill of already-stored rows (this runs UPDATEs over
+--   the whole table; run during low traffic):
+--     UPDATE pages SET content = content
+--       WHERE search_vector IS NULL
+--          OR octet_length(search_vector::text) > 1048575;