1058 lines
36 KiB
Python
1058 lines
36 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
JonasSearch Suchmaschine – Kern (engine.py)
|
||
|
||
Die eigentliche Suchlogik über der PostgreSQL-Datenbank `jonassearch`.
|
||
|
||
Retrieval:
|
||
Die Tabelle `public.pages` hält pro Webseite u. a. url, title,
|
||
headings (text[]), content, domain, tld, http_status, content_type,
|
||
discovered_at/crawled_at sowie die per DB-Trigger gepflegte Spalte
|
||
`search_vector` (tsvector, Titel = Gewicht A, Überschriften = B,
|
||
Inhalt = C, Textsuch-Konfiguration 'simple') und den GIN-Index
|
||
`idx_pages_search`. Kandidaten werden per `@@` über diesen Index
|
||
ermittelt.
|
||
|
||
Query-Syntax:
|
||
Die Nutzersuche läuft über `websearch_to_tsquery` – damit sind
|
||
Anführungszeichen für exakte Phrasen, `OR` und `-ausschluss`
|
||
möglich, ohne dass eine fehlerhafte Syntax den Request abschießt.
|
||
|
||
Ranking (mehrstufig, in SQL):
|
||
score = ts_rank_cd(Gewichte, search_vector, tsquery, Norm) [A/B/C]
|
||
* (1 + Frischegewicht * exp(-Alter_in_Tagen / Halbwertszeit))
|
||
|
||
Die ts_rank_cd-Normalisierung (Bitmaske 8 = Rang / 1 + log
|
||
Dokumentlänge) verhindert, dass riesige Seiten kurze, präzise
|
||
Treffer dominieren. Aktualität basiert auf COALESCE(crawled_at,
|
||
discovered_at); Dokumentqualität/HTTP-Metadaten fließen über die
|
||
Standardfilter ein (nur http_status = 200, nur HTML-Inhalt).
|
||
Link-basierte Autoritätssignale sind bewusst noch nicht Teil der
|
||
Formel.
|
||
|
||
Bei sehr häufigen Begriffen (Zeitlimit des vollen Rankings) wird
|
||
automatisch auf ein begrenztes Kandidatenfenster ausgewichen
|
||
(MAX_RANK_CANDIDATES, Antwortmarke ranking_capped).
|
||
|
||
Schutz der Produktivdatenbank:
|
||
- Jede Operation baut eine eigene Verbindung mit
|
||
`statement_timeout` (Default 10 s) auf – keine Request kann die
|
||
Datenbank blockieren.
|
||
- Der Gesamt-Trefferzähler läuft mit kürzerem Timeout (4 s) und ist
|
||
"best effort": bei Überschreitung wird `total = None` geliefert,
|
||
die Ergebnisse selbst bleiben davon unberührt.
|
||
- `content` (TOAST-lastig, bis 200 KB indexiert) wird nur für die
|
||
Treffer der angeforderten Seite aus der DB gelesen (Snippets via
|
||
`ts_headline`), nicht für die komplette Kandidatenmenge.
|
||
"""
|
||
|
||
import logging
|
||
import os
|
||
import re
|
||
import time
|
||
from datetime import datetime, timedelta, timezone
|
||
|
||
import psycopg2
|
||
from psycopg2.extras import RealDictCursor
|
||
|
||
log = logging.getLogger("jonassearch.search")
|
||
log.addHandler(logging.NullHandler())
|
||
|
||
|
||
# ============================================================
|
||
# FEHLERKLASSEN
|
||
# ============================================================
|
||
|
||
class SearchEngineError(Exception):
|
||
"""Basisfehler der Suchmaschine."""
|
||
|
||
|
||
class QueryError(SearchEngineError, ValueError):
|
||
"""Ungültige Suchanfrage / ungültige Parameter (-> HTTP 400)."""
|
||
|
||
|
||
class SearchTimeout(SearchEngineError, TimeoutError):
|
||
"""Suchanfrage hat das Statement-Timeout überschritten (-> HTTP 504)."""
|
||
|
||
|
||
# ============================================================
|
||
# KONFIGURATION (Environment, wie crawler.py / priorityweb)
|
||
# ============================================================
|
||
|
||
# Wichtig: die Query-Konfiguration muss zur Konfiguration passen, mit der
|
||
# der DB-Trigger den Suchvektor gebaut hat (laut DB-Doku: 'simple').
|
||
TS_CONFIG = os.getenv("JONASSEARCH_TS_CONFIG", "simple").strip() or "simple"
|
||
|
||
# tsvector-Spalte erzwingen (optional, sonst automatische Erkennung).
|
||
VECTOR_COLUMN_ENV = os.getenv("JONASSEARCH_VECTOR_COLUMN", "").strip()
|
||
|
||
# Statement-Timeouts (Schutz der ~10-GB-Produktivdatenbank).
|
||
STATEMENT_TIMEOUT_MS = int(os.getenv("JONASSEARCH_STATEMENT_TIMEOUT_MS", "10000"))
|
||
COUNT_TIMEOUT_MS = int(os.getenv("JONASSEARCH_COUNT_TIMEOUT_MS", "4000"))
|
||
|
||
# Paginierung
|
||
DEFAULT_PER_PAGE = int(os.getenv("JONASSEARCH_RESULTS_PER_PAGE", "10"))
|
||
MAX_PER_PAGE = int(os.getenv("JONASSEARCH_MAX_RESULTS_PER_PAGE", "50"))
|
||
|
||
# Ranking: Frische-Signal (Aktualitätsbonus auf den ts_rank_cd).
|
||
FRESHNESS_WEIGHT = float(os.getenv("JONASSEARCH_FRESHNESS_WEIGHT", "0.25"))
|
||
FRESHNESS_HALF_LIFE_DAYS = float(
|
||
os.getenv("JONASSEARCH_FRESHNESS_HALF_LIFE_DAYS", "180")
|
||
)
|
||
|
||
# ts_rank_cd-Normalisierung (Bitmaske): 8 = Rang / (1 + log(Dokumentlänge))
|
||
# – verhindert, dass riesige Seiten automatisch jeden kurzen, präzisen
|
||
# Treffer dominieren. 0 = keine Normalisierung.
|
||
RANK_NORMALIZATION = int(os.getenv("JONASSEARCH_RANK_NORMALIZATION", "8"))
|
||
|
||
# Bei sehr häufigen Begriffen (z. B. Einzelwort mit zigtausend Treffern)
|
||
# wäre ts_rank_cd über alle Kandidaten zu langsam. Passiert das, wird
|
||
# automatisch ein zweiter Versuch über die ersten MAX_RANK_CANDIDATES
|
||
# Kandidaten (Indexreihenfolge) gefahren – garantiert schnelle Antwort,
|
||
# Ranking dann innerhalb dieses Fensters (Antwortflag ranking_capped).
|
||
MAX_RANK_CANDIDATES = int(os.getenv("JONASSEARCH_MAX_RANK_CANDIDATES", "5000"))
|
||
|
||
# Notfall-Stufe: selbst das Kandidatenfenster ist zu groß (extrem häufige
|
||
# Begriffe). Dann wird in Etappen immer weiter reduziert, bis die Antwort
|
||
# innerhalb des Zeitlimits kommt. Der Nutzer bekommt trotzdem Ergebnis-
|
||
# seiten statt einer Fehlermeldung – bei reduzierter Trefferqualität
|
||
# (Antwortflag ranking_degraded).
|
||
RANK_CAP_RETRIES = int(os.getenv("JONASSEARCH_RANK_CAP_RETRIES", "3"))
|
||
|
||
# Sortierung "relevance" fällt bei Überschreitung aller Ranking-Timeouts
|
||
# auf diese Basisreihenfolge zurück (kein ts_rank_cd nötig).
|
||
FALLBACK_SORT_SQL = (
|
||
"COALESCE(p.crawled_at, p.discovered_at) DESC NULLS LAST, p.id ASC"
|
||
)
|
||
|
||
# ts_rank_cd-Gewichte. Reihenfolge im DB-Trigger: Titel = A,
|
||
# Überschriften = B, Inhalt = C, (D = ungewichteter Rest).
|
||
# Format: "A=1.0,B=0.6,C=0.3,D=0.1"
|
||
DEFAULT_RANK_WEIGHTS = {"A": 1.0, "B": 0.6, "C": 0.3, "D": 0.1}
|
||
|
||
# Snippets: ts_headline liest nur die ersten N Zeichen von `content`
|
||
# (TOAST-Schonung; die indexierten 200 KB kommen praktisch nie in ein
|
||
# Snippet-Fenster).
|
||
HEADLINE_SOURCE_MAX_CHARS = int(
|
||
os.getenv("JONASSEARCH_HEADLINE_SOURCE_CHARS", "100000")
|
||
)
|
||
HEADLINE_OPTIONS = (
|
||
"MaxWords=40, MinWords=15, ShortWord=2, MaxFragments=2, "
|
||
"StartSel=⟦, StopSel=⟧"
|
||
)
|
||
|
||
# Einzelabruf einer Seite
|
||
PAGE_CONTENT_CHARS = int(os.getenv("JONASSEARCH_PAGE_CONTENT_CHARS", "50000"))
|
||
PAGE_MAX_LINKS = 100
|
||
|
||
# Vorschläge (Autocomplete)
|
||
MAX_SUGGEST = 10
|
||
|
||
DB_HOST = os.getenv("JONASSEARCH_DB_HOST", "127.0.0.1")
|
||
DB_PORT = int(os.getenv("JONASSEARCH_DB_PORT", "25432"))
|
||
DB_NAME = os.getenv("JONASSEARCH_DB_NAME", "jonassearch")
|
||
DB_USER = os.getenv("JONASSEARCH_DB_USER", "jonassearch")
|
||
DB_PASSWORD = os.getenv("JONASSEARCH_DB_PASSWORD", "")
|
||
|
||
# Markierungen im ts_headline-Snippet (werden im Frontend/CLI zu
|
||
# Hervorhebungen umgesetzt; bewusst KEIN HTML, damit der Inhalt nie
|
||
# ungefiltert als Markup landet).
|
||
SNIPPET_START = "⟦"
|
||
SNIPPET_STOP = "⟧"
|
||
|
||
# ------------------------------------------------------------
|
||
# Validierung
|
||
# ------------------------------------------------------------
|
||
|
||
# Sehr defensive Domain-Prüfung (wie priorityweb/app.py).
|
||
DOMAIN_RE = re.compile(
|
||
r"^(?=.{1,253}\.)([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+"
|
||
r"[a-z]{2,63}$"
|
||
)
|
||
|
||
# SQL-Identifier für die auto-erkannte tsvector-Spalte.
|
||
IDENT_RE = re.compile(r"^[a-z_][a-z0-9_]{0,62}$")
|
||
|
||
# Relative Zeitspannen wie 24h / 7d / 2w.
|
||
_RELATIVE_RE = re.compile(r"^(\d+)\s*([hdw])$", re.IGNORECASE)
|
||
|
||
_SORT_ALIASES = {
|
||
"relevance": "relevance",
|
||
"score": "relevance",
|
||
"rank": "relevance",
|
||
"relevanz": "relevance",
|
||
"freshness": "freshness",
|
||
"fresh": "freshness",
|
||
"date": "freshness",
|
||
"time": "freshness",
|
||
"aktualitaet": "freshness",
|
||
}
|
||
|
||
|
||
# ============================================================
|
||
# DATENBANK
|
||
# ============================================================
|
||
|
||
def db_connect(statement_timeout_ms=None):
|
||
"""Neue Verbindung (autocommit, read-only Nutzung, Timeout gesetzt)."""
|
||
timeout = (
|
||
STATEMENT_TIMEOUT_MS
|
||
if statement_timeout_ms is None
|
||
else int(statement_timeout_ms)
|
||
)
|
||
conn = psycopg2.connect(
|
||
host=DB_HOST,
|
||
port=DB_PORT,
|
||
dbname=DB_NAME,
|
||
user=DB_USER,
|
||
password=DB_PASSWORD,
|
||
connect_timeout=5,
|
||
application_name="jonassearch-search",
|
||
options=f"-c statement_timeout={timeout}",
|
||
)
|
||
conn.autocommit = True
|
||
return conn
|
||
|
||
|
||
def _set_statement_timeout(cur, ms):
|
||
"""Timeout zur Laufzeit umschalten (int()-gecastet -> keine Injection)."""
|
||
cur.execute(f"SET statement_timeout = {int(ms)}")
|
||
|
||
|
||
# tsvector-Spalte von `pages` (einmalig ermittelt, dann gecacht).
|
||
_vector_column_cache = None
|
||
|
||
|
||
def resolve_vector_column(conn):
|
||
"""
|
||
Liefert den Namen der tsvector-Spalte von public.pages.
|
||
|
||
Priorität: JONASSEARCH_VECTOR_COLUMN > 'search_vector' > erste
|
||
gefundene tsvector-Spalte. Der Name wird gegen IDENT_RE geprüft,
|
||
bevor er in SQL eingebettet wird.
|
||
"""
|
||
global _vector_column_cache
|
||
|
||
if VECTOR_COLUMN_ENV:
|
||
if not IDENT_RE.match(VECTOR_COLUMN_ENV):
|
||
raise SearchEngineError(
|
||
f"Ungültiger Spaltenname in JONASSEARCH_VECTOR_COLUMN: "
|
||
f"{VECTOR_COLUMN_ENV!r}"
|
||
)
|
||
return VECTOR_COLUMN_ENV
|
||
|
||
if _vector_column_cache:
|
||
return _vector_column_cache
|
||
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT column_name
|
||
FROM information_schema.columns
|
||
WHERE table_schema = 'public'
|
||
AND table_name = 'pages'
|
||
AND data_type = 'tsvector'
|
||
ORDER BY (column_name = 'search_vector') DESC, column_name
|
||
LIMIT 1
|
||
"""
|
||
)
|
||
row = cur.fetchone()
|
||
|
||
if not row:
|
||
raise SearchEngineError(
|
||
"Keine tsvector-Spalte in public.pages gefunden. Wurde der "
|
||
"Suchindex (Trigger pages_search_vector_update) angelegt?"
|
||
)
|
||
column = row[0]
|
||
if not IDENT_RE.match(column):
|
||
raise SearchEngineError(
|
||
f"Unerwarteter Spaltenname für den Suchvektor: {column!r}"
|
||
)
|
||
_vector_column_cache = column
|
||
log.debug("tsvector-Spalte erkannt: %s", column)
|
||
return column
|
||
|
||
|
||
# ============================================================
|
||
# PARAMETER-NORMALISIERUNG (rein, ohne DB – gut testbar)
|
||
# ============================================================
|
||
|
||
def _as_list(value):
|
||
"""None/str/list -> Liste von Strings (str wird gesplittet an ',')."""
|
||
if value is None:
|
||
return []
|
||
if isinstance(value, str):
|
||
parts = value.split(",")
|
||
else:
|
||
parts = []
|
||
for item in value:
|
||
if isinstance(item, str):
|
||
parts.extend(item.split(","))
|
||
else:
|
||
parts.append(item)
|
||
return [str(p).strip() for p in parts if str(p).strip()]
|
||
|
||
|
||
def normalize_sites(values):
|
||
"""
|
||
Benutzereingaben -> saubere Domainliste (kleingeschrieben, ohne
|
||
Schema/Pfad/Port/www). Akzeptiert auch URLs und '*.example.com'.
|
||
"""
|
||
out = []
|
||
for raw in _as_list(values):
|
||
v = raw.strip().lower()
|
||
if "://" in v:
|
||
v = v.split("://", 1)[1]
|
||
v = v.split("/", 1)[0]
|
||
v = v.split("?", 1)[0]
|
||
v = v.split("#", 1)[0]
|
||
v = v.split("@", 1)[-1]
|
||
v = v.rsplit(":", 1)[0]
|
||
if v.startswith("*."):
|
||
v = v[2:]
|
||
if v.startswith("www."):
|
||
v = v[4:]
|
||
v = v.rstrip(".")
|
||
if not v:
|
||
continue
|
||
if not DOMAIN_RE.match(v):
|
||
raise QueryError(f"Ungültige Domain-Filterangabe: {raw!r}")
|
||
if v not in out:
|
||
out.append(v)
|
||
return out
|
||
|
||
|
||
def normalize_tlds(values):
|
||
"""
|
||
'de', '.DE', ['de', 'at'] -> ['.de', '.at'].
|
||
|
||
Der Crawler speichert die TLD MIT führendem Punkt (z. B. '.de').
|
||
"""
|
||
out = []
|
||
for raw in _as_list(values):
|
||
v = raw.strip().lower().lstrip(".")
|
||
if not v:
|
||
continue
|
||
if not re.match(r"^[a-z0-9-]{2,63}$", v):
|
||
raise QueryError(f"Ungültige TLD-Filterangabe: {raw!r}")
|
||
dotted = "." + v
|
||
if dotted not in out:
|
||
out.append(dotted)
|
||
return out
|
||
|
||
|
||
def normalize_sort(value):
|
||
v = str(value or "relevance").strip().lower()
|
||
if v not in _SORT_ALIASES:
|
||
raise QueryError(
|
||
f"Ungültige Sortierung: {value!r} "
|
||
f"(erlaubt: relevance | freshness)"
|
||
)
|
||
return _SORT_ALIASES[v]
|
||
|
||
|
||
def parse_time_bound(value, which="after"):
|
||
"""
|
||
Zeilfilter parsen.
|
||
|
||
Erlaubt: ISO-Datum/Zeit ('2026-01-31', '2026-01-31T12:00:00', mit
|
||
oder ohne 'Z'/Offset) sowie relative Angaben '24h', '7d', '2w'
|
||
(Ergebnis ist immer now() - Zeitspanne; 'after=7d' heißt also
|
||
"letzte 7 Tage", 'before=7d' "älter als 7 Tage").
|
||
"""
|
||
if value is None or str(value).strip() == "":
|
||
return None
|
||
|
||
v = str(value).strip()
|
||
|
||
m = _RELATIVE_RE.match(v)
|
||
if m:
|
||
amount, unit = int(m.group(1)), m.group(2).lower()
|
||
if amount <= 0:
|
||
raise QueryError(f"Ungültige Zeitspanne: {value!r}")
|
||
delta = {
|
||
"h": timedelta(hours=amount),
|
||
"d": timedelta(days=amount),
|
||
"w": timedelta(weeks=amount),
|
||
}[unit]
|
||
return datetime.now(timezone.utc) - delta
|
||
|
||
try:
|
||
dt = datetime.fromisoformat(v.replace("Z", "+00:00"))
|
||
except ValueError:
|
||
raise QueryError(f"Ungültiges Datumsfilter ({which}): {value!r}")
|
||
|
||
if dt.tzinfo is None:
|
||
dt = dt.replace(tzinfo=timezone.utc)
|
||
return dt
|
||
|
||
|
||
def parse_rank_weights(spec=None):
|
||
"""
|
||
Parst JONASSEARCH_RANK_WEIGHTS ("A=1.0,B=0.6,C=0.3,D=0.1") und
|
||
liefert das Array in der von ts_rank_cd erwarteten Reihenfolge
|
||
[D, C, B, A].
|
||
"""
|
||
weights = dict(DEFAULT_RANK_WEIGHTS)
|
||
if spec:
|
||
for part in str(spec).split(","):
|
||
part = part.strip()
|
||
if not part:
|
||
continue
|
||
if "=" not in part:
|
||
raise QueryError(
|
||
f"Ungültiges Gewicht {part!r} – Format: A=1.0,B=0.6,C=0.3,D=0.1"
|
||
)
|
||
key, _, val = part.partition("=")
|
||
key = key.strip().upper()
|
||
if key not in ("A", "B", "C", "D"):
|
||
raise QueryError(
|
||
f"Ungültiges Gewichtssignal {key!r} – erlaubt: A, B, C, D"
|
||
)
|
||
try:
|
||
weights[key] = float(val)
|
||
except ValueError:
|
||
raise QueryError(f"Ungültiger Zahlenwert für {key}: {val!r}")
|
||
if not (0.0 <= weights[key] <= 1e6):
|
||
raise QueryError(f"Gewicht {key} außerhalb des zulässigen Bereichs")
|
||
return [weights["D"], weights["C"], weights["B"], weights["A"]]
|
||
|
||
|
||
def prefix_lexeme(query):
|
||
"""
|
||
Letztes 'normales' Wort der Eingabe für die Autocomplete-Präfixsuche
|
||
('hallo welt' -> 'welt:*'). Nicht nutzbare Eingaben -> None.
|
||
"""
|
||
if not query:
|
||
return None
|
||
tokens = re.findall(r"\w+", str(query), re.UNICODE)
|
||
for token in reversed(tokens):
|
||
if len(token) >= 2:
|
||
return token.lower() + ":*"
|
||
return None
|
||
|
||
|
||
RANK_WEIGHTS = parse_rank_weights(os.getenv("JONASSEARCH_RANK_WEIGHTS"))
|
||
|
||
|
||
# ============================================================
|
||
# SQL-BAUSTEINE
|
||
# ============================================================
|
||
|
||
def _filter_clauses(sites, tlds, after, before, include_errors,
|
||
include_non_html):
|
||
"""WHERE-Bausteine + Parameter für die Standardfilter."""
|
||
clauses = []
|
||
params = {}
|
||
|
||
if sites:
|
||
# domain speichert den vollen Hostnamen (z. B. www.example.com):
|
||
# exakte Domain ODER Subdomain.
|
||
clauses.append(
|
||
"(p.domain = ANY(%(sites)s) OR p.domain LIKE ANY(%(site_pats)s))"
|
||
)
|
||
params["sites"] = sites
|
||
params["site_pats"] = ["%." + s for s in sites]
|
||
|
||
if tlds:
|
||
clauses.append("(p.tld = ANY(%(tlds)s))")
|
||
params["tlds"] = tlds
|
||
|
||
if not include_errors:
|
||
# Nur erfolgreich gecrawlte Seiten.
|
||
clauses.append("(p.http_status = 200)")
|
||
|
||
if not include_non_html:
|
||
# Nur HTML-Dokumente (leerer content_type zulassen).
|
||
# Achtung: %% statt % – der Filter-SQL-String wird immer mit
|
||
# Parameter-Dict an psycopg2 übergeben, nackte Prozentzeichen
|
||
# würden als Platzhalter geparst.
|
||
clauses.append(
|
||
"(p.content_type LIKE 'text/html%%' OR p.content_type IS NULL)"
|
||
)
|
||
|
||
if after is not None:
|
||
clauses.append("(COALESCE(p.crawled_at, p.discovered_at) >= %(after)s)")
|
||
params["after"] = after
|
||
|
||
if before is not None:
|
||
clauses.append("(COALESCE(p.crawled_at, p.discovered_at) < %(before)s)")
|
||
params["before"] = before
|
||
|
||
sql = (" AND " + " AND ".join(clauses)) if clauses else ""
|
||
return sql, params
|
||
|
||
|
||
def _search_sql(vector_col, sort, filter_sql, capped=False):
|
||
"""
|
||
Ranking-Query: Kandidaten via GIN-Index, bewertet + sortiert.
|
||
|
||
capped=False: volles Ranking über alle Kandidaten (exakt, aber bei
|
||
sehr häufigen Begriffen langsam).
|
||
|
||
capped=True: Ranking nur über die ersten MAX_RANK_CANDIDATES
|
||
Kandidaten (Subquery-LIMIT in Indexreihenfolge) – bounded latency
|
||
als Fallback nach einem Timeout des vollen Rankings.
|
||
"""
|
||
if sort == "relevance":
|
||
order = "(r.text_rank * f.fresh) DESC, r.text_rank DESC, p.id ASC"
|
||
else:
|
||
order = (
|
||
"COALESCE(p.crawled_at, p.discovered_at) DESC NULLS LAST, p.id ASC"
|
||
)
|
||
|
||
# Frische-Signal: braucht p.crawled_at + p.discovered_at.
|
||
fresh_lateral = """
|
||
CROSS JOIN LATERAL (
|
||
SELECT 1.0 + %(fresh_weight)s * exp(
|
||
- GREATEST(0, EXTRACT(EPOCH FROM (
|
||
now() - COALESCE(p.crawled_at, p.discovered_at)
|
||
)))::double precision
|
||
/ 86400.0 / GREATEST(%(fresh_half_life)s, 0.01)
|
||
) AS fresh
|
||
) f
|
||
"""
|
||
|
||
if capped:
|
||
source = """
|
||
FROM (
|
||
SELECT p.id, p.url, p.title, p.domain, p.tld, p.http_status,
|
||
p.content_type, p.discovered_at,
|
||
COALESCE(p.crawled_at, p.discovered_at) AS crawled_at,
|
||
p.{vector_col} AS vector
|
||
FROM pages p
|
||
CROSS JOIN LATERAL (
|
||
SELECT websearch_to_tsquery(%(ts_config)s, %(query)s) AS tsq
|
||
) q
|
||
WHERE p.{vector_col} @@ q.tsq{filter_sql}
|
||
LIMIT %(rank_cap)s
|
||
) p
|
||
CROSS JOIN LATERAL (
|
||
SELECT ts_rank_cd(%(weights)s::real[], p.vector,
|
||
websearch_to_tsquery(%(ts_config)s, %(query)s),
|
||
%(norm)s::integer) AS text_rank
|
||
) r
|
||
""".format(vector_col=vector_col, filter_sql=filter_sql)
|
||
else:
|
||
source = f"""
|
||
FROM pages p
|
||
CROSS JOIN LATERAL (
|
||
SELECT websearch_to_tsquery(%(ts_config)s, %(query)s) AS tsq
|
||
) q
|
||
CROSS JOIN LATERAL (
|
||
SELECT ts_rank_cd(%(weights)s::real[], p.{vector_col}, q.tsq,
|
||
%(norm)s::integer) AS text_rank
|
||
) r
|
||
""" + fresh_lateral + f"""
|
||
WHERE p.{vector_col} @@ q.tsq{filter_sql}
|
||
"""
|
||
|
||
if capped:
|
||
source += fresh_lateral
|
||
|
||
return f"""
|
||
SELECT
|
||
p.id,
|
||
p.url,
|
||
p.title,
|
||
p.domain,
|
||
p.tld,
|
||
p.http_status,
|
||
p.content_type,
|
||
p.crawled_at,
|
||
r.text_rank,
|
||
f.fresh AS freshness,
|
||
(r.text_rank * f.fresh) AS score
|
||
{source}
|
||
ORDER BY {order}
|
||
LIMIT %(limit)s OFFSET %(offset)s
|
||
"""
|
||
|
||
|
||
def _headline_sql():
|
||
"""Snippets nur für die Treffer der angeforderten Seite."""
|
||
return """
|
||
SELECT p.id,
|
||
ts_headline(%(ts_config)s,
|
||
left(p.content, %(hl_chars)s),
|
||
q.tsq,
|
||
%(hl_opts)s) AS snippet
|
||
FROM pages p
|
||
CROSS JOIN LATERAL (
|
||
SELECT websearch_to_tsquery(%(ts_config)s, %(query)s) AS tsq
|
||
) q
|
||
WHERE p.id = ANY(%(ids)s)
|
||
"""
|
||
|
||
|
||
def _emergency_search_sql(vector_col, filter_sql):
|
||
"""
|
||
Notfall-Query ganz ohne ts_rank_cd: Kandidaten werden in fester
|
||
Basisreihenfolge (neueste zuerst) geliefert. Auch bei extrem häufigen
|
||
Begriffen garantiert schnell, weil kein Ranking über die Treffermenge
|
||
nötig ist.
|
||
"""
|
||
return f"""
|
||
SELECT p.id, p.url, p.title, p.domain, p.tld,
|
||
p.http_status, p.content_type,
|
||
COALESCE(p.crawled_at, p.discovered_at) AS crawled_at,
|
||
0.0 AS text_rank,
|
||
1.0 AS freshness,
|
||
0.0 AS score
|
||
FROM pages p
|
||
CROSS JOIN LATERAL (
|
||
SELECT websearch_to_tsquery(%(ts_config)s, %(query)s) AS tsq
|
||
) q
|
||
WHERE p.{vector_col} @@ q.tsq{filter_sql}
|
||
ORDER BY {FALLBACK_SORT_SQL}
|
||
LIMIT %(limit)s OFFSET %(offset)s
|
||
"""
|
||
|
||
|
||
def _count_sql(vector_col, filter_sql):
|
||
return f"""
|
||
SELECT count(*) AS n
|
||
FROM pages p
|
||
CROSS JOIN LATERAL (
|
||
SELECT websearch_to_tsquery(%(ts_config)s, %(query)s) AS tsq
|
||
) q
|
||
WHERE p.{vector_col} @@ q.tsq{filter_sql}
|
||
"""
|
||
|
||
|
||
def _iso(dt):
|
||
if isinstance(dt, datetime):
|
||
return dt.isoformat()
|
||
return None
|
||
|
||
|
||
def _round(value, digits=6):
|
||
try:
|
||
return round(float(value), digits)
|
||
except (TypeError, ValueError):
|
||
return 0.0
|
||
|
||
|
||
# ============================================================
|
||
# ÖFFENTLICHE API
|
||
# ============================================================
|
||
|
||
def _fallback_search(cur, vector_col, sort, filter_sql, params):
|
||
"""
|
||
Mehrstufiger Fallback, wenn das volle Ranking das Zeitlimit
|
||
überschritten hat. Liefert so gut wie immer Ergebnis-Seiten statt
|
||
eines harten Timeout-Fehlers:
|
||
|
||
1. Kandidatenfenster MAX_RANK_CANDIDATES (gs. 5000):
|
||
normales, aber begrenztes Ranking.
|
||
2. Danach in Etappen verkleinern (z. B. 2000 → 800 → 320 …), bis
|
||
die Antwort innerhalb des Zeitlimits kommt.
|
||
3. Letzte Stufe: ganz ohne ts_rank_cd-Ranking – nur Index-Kandidaten
|
||
in definierter Reihenfolge (z. B. neueste zuerst).
|
||
|
||
Rückgabe: (ranking_capped, ranking_degraded, rows).
|
||
"""
|
||
cap = int(params.get("rank_cap") or MAX_RANK_CANDIDATES)
|
||
capped = True
|
||
degraded = False
|
||
|
||
# Stufe 2: Fenster in Etappen verkleinern.
|
||
for _ in range(max(0, RANK_CAP_RETRIES)):
|
||
try:
|
||
cur.execute(_search_sql(vector_col, sort, filter_sql, capped=True),
|
||
params)
|
||
return capped, degraded, cur.fetchall()
|
||
except psycopg2.errors.QueryCanceled:
|
||
cap = max(50, int(cap * 0.4))
|
||
params["rank_cap"] = cap
|
||
degraded = True
|
||
|
||
# Stufe 3: Notfall ohne Ranking – Kandidaten in Basisreihenfolge.
|
||
try:
|
||
cur.execute(
|
||
_emergency_search_sql(vector_col, filter_sql), params,
|
||
)
|
||
return capped, True, cur.fetchall()
|
||
except psycopg2.errors.QueryCanceled:
|
||
raise SearchTimeout(
|
||
"Die Suche dauert ungewöhnlich lange – probiere es nochmal "
|
||
"oder nutze einen Filter wie site=<domain>, um schneller zu "
|
||
"Ergebnissen zu kommen."
|
||
)
|
||
|
||
|
||
def search(query, *, page=1, per_page=None, sites=None, tlds=None,
|
||
after=None, before=None, sort="relevance",
|
||
include_errors=False, include_non_html=False,
|
||
with_snippets=True):
|
||
"""
|
||
Volltextsuche über `pages`.
|
||
|
||
Rückgabe (dict): query, parsed_query, page, per_page, sort, total
|
||
(None = nicht gezählt/Zeitlimit), has_more, results[], took_ms.
|
||
Jedes Result: id, url, title, domain, tld, http_status, content_type,
|
||
crawled_at, score, text_rank, freshness, snippet (⟦…⟧-Markierungen).
|
||
"""
|
||
q = (query or "").strip()
|
||
if not q:
|
||
raise QueryError("Leere Suchanfrage.")
|
||
|
||
try:
|
||
page = max(1, int(page))
|
||
per_page = DEFAULT_PER_PAGE if per_page is None else int(per_page)
|
||
except (TypeError, ValueError):
|
||
raise QueryError("page/per_page müssen Zahlen sein.")
|
||
per_page = min(max(1, per_page), MAX_PER_PAGE)
|
||
offset = (page - 1) * per_page
|
||
|
||
sort = normalize_sort(sort)
|
||
site_list = normalize_sites(sites)
|
||
tld_list = normalize_tlds(tlds)
|
||
after_dt = parse_time_bound(after, "after")
|
||
before_dt = parse_time_bound(before, "before")
|
||
|
||
started = time.monotonic()
|
||
conn = db_connect()
|
||
try:
|
||
vector_col = resolve_vector_column(conn)
|
||
filter_sql, filter_params = _filter_clauses(
|
||
site_list, tld_list, after_dt, before_dt,
|
||
include_errors, include_non_html,
|
||
)
|
||
params = {
|
||
"ts_config": TS_CONFIG,
|
||
"query": q,
|
||
"weights": RANK_WEIGHTS,
|
||
"norm": RANK_NORMALIZATION,
|
||
"fresh_weight": FRESHNESS_WEIGHT,
|
||
"fresh_half_life": FRESHNESS_HALF_LIFE_DAYS,
|
||
"limit": per_page + 1, # +1 -> exaktes has_more
|
||
"offset": offset,
|
||
"rank_cap": MAX_RANK_CANDIDATES,
|
||
}
|
||
params.update(filter_params)
|
||
|
||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||
# Wie hat PostgreSQL die Anfrage interpretiert? (Transparenz
|
||
# + Erkennung "keine durchsuchbaren Begriffe")
|
||
cur.execute(
|
||
"SELECT websearch_to_tsquery(%s, %s)::text AS tsq",
|
||
(TS_CONFIG, q),
|
||
)
|
||
parsed_tsq = ((cur.fetchone() or {}).get("tsq") or "").strip()
|
||
empty_tsq = not parsed_tsq
|
||
|
||
rows = []
|
||
snippets = {}
|
||
total = None
|
||
ranking_capped = False
|
||
ranking_degraded = False
|
||
|
||
if not empty_tsq:
|
||
_set_statement_timeout(cur, STATEMENT_TIMEOUT_MS)
|
||
try:
|
||
cur.execute(_search_sql(vector_col, sort, filter_sql),
|
||
params)
|
||
rows = cur.fetchall()
|
||
except psycopg2.errors.QueryCanceled:
|
||
ranking_capped, ranking_degraded, rows = _fallback_search(
|
||
cur, vector_col, sort, filter_sql, params,
|
||
)
|
||
log.info(
|
||
"Volles Ranking übersprungen (Timeout); "
|
||
"Fallback: %d Kandidaten%s.",
|
||
params["rank_cap"],
|
||
", degradiert" if ranking_degraded else "",
|
||
)
|
||
|
||
if with_snippets and rows:
|
||
ids = [r["id"] for r in rows[:per_page]]
|
||
cur.execute(_headline_sql(), {
|
||
"ts_config": TS_CONFIG,
|
||
"query": q,
|
||
"hl_chars": HEADLINE_SOURCE_MAX_CHARS,
|
||
"hl_opts": HEADLINE_OPTIONS,
|
||
"ids": ids,
|
||
})
|
||
for row in cur.fetchall():
|
||
snippets[row["id"]] = row["snippet"] or ""
|
||
|
||
# Gesamtzahl nur "best effort": eigenes (kürzeres) Timeout,
|
||
# bei Überschreitung bleibt total=None.
|
||
_set_statement_timeout(cur, COUNT_TIMEOUT_MS)
|
||
try:
|
||
cur.execute(_count_sql(vector_col, filter_sql), params)
|
||
total = int(cur.fetchone()["n"])
|
||
except psycopg2.Error as exc:
|
||
log.warning("Trefferzählung abgebrochen: %s", exc)
|
||
total = None
|
||
|
||
has_more = len(rows) > per_page
|
||
rows = rows[:per_page]
|
||
|
||
results = []
|
||
for r in rows:
|
||
results.append({
|
||
"id": r["id"],
|
||
"url": r["url"],
|
||
"title": (r["title"] or "").strip() or r["url"],
|
||
"domain": r["domain"] or "",
|
||
"tld": r["tld"] or "",
|
||
"http_status": r["http_status"],
|
||
"content_type": r["content_type"],
|
||
"crawled_at": _iso(r["crawled_at"]),
|
||
"score": _round(r["score"]),
|
||
"text_rank": _round(r["text_rank"]),
|
||
"freshness": _round(r["freshness"], 3),
|
||
"snippet": (snippets.get(r["id"]) or "").strip(),
|
||
})
|
||
|
||
return {
|
||
"query": q,
|
||
"parsed_query": parsed_tsq,
|
||
"page": page,
|
||
"per_page": per_page,
|
||
"sort": sort,
|
||
"ranking_capped": ranking_capped,
|
||
"ranking_degraded": ranking_degraded,
|
||
"total": total,
|
||
"has_more": has_more,
|
||
"results": results,
|
||
"took_ms": round((time.monotonic() - started) * 1000, 1),
|
||
}
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def get_page(page_id=None, url=None, *, max_content_chars=None):
|
||
"""
|
||
Einzelne Seite per id oder url. content ist auf
|
||
JONASSEARCH_PAGE_CONTENT_CHARS (Default 50000) gekürzt; links auf
|
||
PAGE_MAX_LINKS. content_truncated zeigt die Kürzung an.
|
||
"""
|
||
if (page_id is None) == (url is None):
|
||
raise QueryError("Genau eins von 'id' oder 'url' angeben.")
|
||
if page_id is not None:
|
||
try:
|
||
page_id = int(page_id)
|
||
except (TypeError, ValueError):
|
||
raise QueryError("id muss eine Zahl sein.")
|
||
if page_id <= 0:
|
||
raise QueryError("id muss positiv sein.")
|
||
if url is not None:
|
||
url = str(url).strip()
|
||
if not url or len(url) > 2048:
|
||
raise QueryError("Ungültige url.")
|
||
|
||
limit_chars = (
|
||
PAGE_CONTENT_CHARS
|
||
if max_content_chars is None
|
||
else max(200, min(int(max_content_chars), 1000000))
|
||
)
|
||
|
||
conn = db_connect()
|
||
try:
|
||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||
if page_id is not None:
|
||
cur.execute(
|
||
"""
|
||
SELECT id, url, title, headings, domain, tld, source_url,
|
||
http_status, content_type, discovered_at,
|
||
crawled_at, crawl_time_ms,
|
||
length(content) AS content_length,
|
||
links,
|
||
left(content, %(max_chars)s) AS content
|
||
FROM pages
|
||
WHERE id = %(id)s
|
||
""",
|
||
{"id": page_id, "max_chars": limit_chars},
|
||
)
|
||
else:
|
||
cur.execute(
|
||
"""
|
||
SELECT id, url, title, headings, domain, tld, source_url,
|
||
http_status, content_type, discovered_at,
|
||
crawled_at, crawl_time_ms,
|
||
length(content) AS content_length,
|
||
links,
|
||
left(content, %(max_chars)s) AS content
|
||
FROM pages
|
||
WHERE url = %(url)s
|
||
""",
|
||
{"url": url, "max_chars": limit_chars},
|
||
)
|
||
row = cur.fetchone()
|
||
finally:
|
||
conn.close()
|
||
|
||
if not row:
|
||
return None
|
||
|
||
content = row["content"] or ""
|
||
content_length = row["content_length"] or 0
|
||
links = list(row["links"] or [])[:PAGE_MAX_LINKS]
|
||
|
||
return {
|
||
"id": row["id"],
|
||
"url": row["url"],
|
||
"title": (row["title"] or "").strip(),
|
||
"headings": list(row["headings"] or []),
|
||
"domain": row["domain"] or "",
|
||
"tld": row["tld"] or "",
|
||
"source_url": row["source_url"],
|
||
"http_status": row["http_status"],
|
||
"content_type": row["content_type"],
|
||
"discovered_at": _iso(row["discovered_at"]),
|
||
"crawled_at": _iso(row["crawled_at"]),
|
||
"crawl_time_ms": row["crawl_time_ms"],
|
||
"content_length": content_length,
|
||
"content_truncated": content_length > len(content),
|
||
"content": content,
|
||
"links": links,
|
||
"links_total": len(row["links"] or []),
|
||
}
|
||
|
||
|
||
def suggest(query, limit=None):
|
||
"""
|
||
Autocomplete-Vorschläge: Präfixsuche auf dem letzten Wort
|
||
(to_tsquery 'begr:*') gegen den Suchvektor, gerankt nach Relevanz.
|
||
"""
|
||
lexeme = prefix_lexeme(query)
|
||
if not lexeme:
|
||
return []
|
||
|
||
try:
|
||
limit = MAX_SUGGEST if limit is None else int(limit)
|
||
except (TypeError, ValueError):
|
||
raise QueryError("limit muss eine Zahl sein.")
|
||
limit = min(max(1, limit), MAX_SUGGEST)
|
||
|
||
conn = db_connect()
|
||
try:
|
||
vector_col = resolve_vector_column(conn)
|
||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||
cur.execute(
|
||
f"""
|
||
SELECT p.title, p.url, p.domain
|
||
FROM pages p
|
||
CROSS JOIN LATERAL (
|
||
SELECT to_tsquery(%(ts_config)s, %(lexeme)s) AS tsq
|
||
) q
|
||
CROSS JOIN LATERAL (
|
||
SELECT ts_rank_cd(p.{vector_col}, q.tsq) AS tr
|
||
) r
|
||
WHERE p.{vector_col} @@ q.tsq
|
||
AND COALESCE(p.title, '') <> ''
|
||
ORDER BY r.tr DESC, p.id ASC
|
||
LIMIT %(limit)s
|
||
""",
|
||
{"ts_config": TS_CONFIG, "lexeme": lexeme, "limit": limit},
|
||
)
|
||
rows = cur.fetchall()
|
||
except psycopg2.errors.QueryCanceled:
|
||
return [] # Vorschläge sind optional – nie zum Fehlerfall machen
|
||
finally:
|
||
conn.close()
|
||
|
||
out, seen = [], set()
|
||
for row in rows:
|
||
title = (row["title"] or "").strip()
|
||
if not title or title.lower() in seen:
|
||
continue
|
||
seen.add(title.lower())
|
||
out.append({
|
||
"title": title,
|
||
"url": row["url"],
|
||
"domain": row["domain"] or "",
|
||
})
|
||
return out
|
||
|
||
|
||
def stats():
|
||
"""
|
||
Günstige Betriebsstatistiken (Schätzungen aus pg_class/pg_stats,
|
||
keine teuren Full-Scans über die ~10-GB-Tabelle).
|
||
"""
|
||
conn = db_connect()
|
||
try:
|
||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||
_set_statement_timeout(cur, STATEMENT_TIMEOUT_MS)
|
||
cur.execute(
|
||
"""
|
||
SELECT
|
||
(SELECT reltuples::bigint
|
||
FROM pg_class
|
||
WHERE oid = 'public.pages'::regclass) AS pages,
|
||
(SELECT reltuples::bigint
|
||
FROM pg_class
|
||
WHERE oid = 'public.crawl_queue'::regclass) AS queue,
|
||
(SELECT max(crawled_at) FROM pages) AS last_crawl,
|
||
pg_size_pretty(pg_database_size(current_database()))
|
||
AS db_size
|
||
"""
|
||
)
|
||
row = dict(cur.fetchone() or {})
|
||
|
||
# Offene Queue-Einträge: best effort mit kurzem Timeout
|
||
# (Index auf (status, priority, discovered_at) vorhanden).
|
||
pending = None
|
||
_set_statement_timeout(cur, COUNT_TIMEOUT_MS)
|
||
try:
|
||
cur.execute(
|
||
"SELECT count(*) AS n FROM crawl_queue "
|
||
"WHERE status = 'pending'"
|
||
)
|
||
pending = int(cur.fetchone()["n"])
|
||
except psycopg2.Error as exc:
|
||
log.warning("Queue-Zählung abgebrochen: %s", exc)
|
||
|
||
# Distinct-Domain-Schätzung aus den Planner-Statistiken.
|
||
domains = None
|
||
try:
|
||
cur.execute(
|
||
"""
|
||
SELECT n_distinct
|
||
FROM pg_stats
|
||
WHERE schemaname = 'public'
|
||
AND tablename = 'pages'
|
||
AND attname = 'domain'
|
||
"""
|
||
)
|
||
nd = cur.fetchone()
|
||
if nd and nd["n_distinct"] is not None:
|
||
nd_val = float(nd["n_distinct"])
|
||
pages = row.get("pages") or 0
|
||
if nd_val >= 0:
|
||
domains = int(nd_val)
|
||
elif pages > 0:
|
||
domains = int(abs(nd_val) * pages)
|
||
except psycopg2.Error as exc:
|
||
log.debug("Domain-Schätzung fehlgeschlagen: %s", exc)
|
||
finally:
|
||
conn.close()
|
||
|
||
return {
|
||
"pages": row.get("pages"),
|
||
"queue": row.get("queue"),
|
||
"queue_pending": pending,
|
||
"domains": domains,
|
||
"last_crawl": _iso(row.get("last_crawl")),
|
||
"db_size": row.get("db_size"),
|
||
}
|
||
|
||
|
||
def health():
|
||
"""Healthcheck für Monitoring/systemd – wirft nie."""
|
||
try:
|
||
conn = db_connect(statement_timeout_ms=3000)
|
||
try:
|
||
with conn.cursor() as cur:
|
||
cur.execute("SELECT 1")
|
||
cur.fetchone()
|
||
finally:
|
||
conn.close()
|
||
return {"ok": True, "database": "up", "ts_config": TS_CONFIG}
|
||
except Exception as exc:
|
||
return {"ok": False, "database": "down", "error": str(exc)[:200]}
|