57 lines
2.5 KiB
PL/PgSQL
57 lines
2.5 KiB
PL/PgSQL
-- fix_pages_search_vector_trigger.sql
|
|
--
|
|
-- Caps the text that feeds the full-text index of the `pages` table.
|
|
--
|
|
-- Problem:
|
|
-- PostgreSQL hard-limits a single tsvector value to 1,048,575 bytes
|
|
-- (not configurable). The trigger function
|
|
-- pages_search_vector_update() built the vector from the full
|
|
-- content column, so pages with very large text content (observed:
|
|
-- ~2 MB, e.g. pkg.go.dev "?tab=importedby" pages) aborted the whole
|
|
-- INSERT with ProgramLimitExceeded ("string is too long for
|
|
-- tsvector") and crashed all crawler workers.
|
|
--
|
|
-- Fix:
|
|
-- Feed at most the first 200,000 characters of the content into
|
|
-- to_tsvector(). Even at worst-case 4 bytes per UTF-8 character this
|
|
-- stays far below the tsvector limit. The full text remains stored
|
|
-- in the content column - only the search index is truncated.
|
|
-- 200,000 characters cover the meaningful prose of essentially all
|
|
-- real pages; the oversized ones are link lists with no index value.
|
|
--
|
|
-- Run once on the jonassearch database as its owner, e.g.:
|
|
-- psql -h 127.0.0.1 -p 25432 -U jonassearch -d jonassearch \
|
|
-- -f fix_pages_search_vector_trigger.sql
|
|
--
|
|
-- Safe to run repeatedly (CREATE OR REPLACE) and safe to run while the
|
|
-- crawler is running: existing rows keep their vectors, only newly
|
|
-- INSERTed/UPDATEd rows use the capped version.
|
|
|
|
CREATE OR REPLACE FUNCTION pages_search_vector_update()
|
|
RETURNS trigger AS $$
|
|
BEGIN
|
|
-- Cap the indexed text: a tsvector value may hold at most
|
|
-- 1,048,575 bytes. left(content, 200000) can never exceed that
|
|
-- (200000 chars * max 4 bytes UTF-8 = 800,000 bytes).
|
|
NEW.search_vector :=
|
|
to_tsvector('simple', left(coalesce(NEW.content, ''), 200000));
|
|
RETURN NEW;
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
|
|
-- Notes for the DBA:
|
|
-- * If your existing function uses a different text search config
|
|
-- (e.g. 'german' or 'english') or indexes additional columns
|
|
-- (title, headings), port the left(..., 200000) cap into YOUR
|
|
-- function body instead of blindly replacing it - check first with:
|
|
-- SELECT prosrc FROM pg_proc WHERE proname =
|
|
-- 'pages_search_vector_update';
|
|
-- * The trigger itself (CREATE TRIGGER pages_search_vector_update
|
|
-- BEFORE INSERT OR UPDATE ...) is unchanged; only the function body
|
|
-- above is replaced.
|
|
-- * Optional backfill of already-stored rows (this runs UPDATEs over
|
|
-- the whole table; run during low traffic):
|
|
-- UPDATE pages SET content = content
|
|
-- WHERE search_vector IS NULL
|
|
-- OR octet_length(search_vector::text) > 1048575;
|