451 lines
14 KiB
Python
451 lines
14 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
JonasSearch – Suchmaschine (Engine).
|
||
|
||
Volltextsuche über den Index, den crawler.py in PostgreSQL aufgebaut hat
|
||
(Tabelle `pages`).
|
||
|
||
Die Engine nutzt, wenn vorhanden, die vom Trigger gepflegte tsvector-Spalte
|
||
der Tabelle `pages` (Suchvektor) und sucht damit mit PostgreSQL's
|
||
Volltextsuche. Fehlt eine solche Spalte (z. B. weil die Tabelle manuell
|
||
angelegt wurde), wird der Suchvektor zur Laufzeit aus title/headings/content
|
||
berechnet – die Suche funktioniert dann genauso, nur langsamer.
|
||
|
||
Features:
|
||
- Rangfolge über ts_rank (Titel/Inhalt gewichtet nach Index-Spalte)
|
||
- Snippets mit hervorgehobenen Suchbegriffen (ts_headline, <mark>)
|
||
- Präfix-Suche ("raspberr" findet "raspberry")
|
||
- UND-Verknüpfung der Suchbegriffe, Phrasen über Anführungszeichen nicht nötig
|
||
- Paginierung
|
||
- Filter nach Domain und TLD
|
||
|
||
Konfiguration über Umgebungsvariablen (siehe .env.example):
|
||
JONASSEARCH_DB_HOST / PORT / NAME / USER / PASSWORD
|
||
JONASSEARCH_TS_CONFIG Textsuch-Konfiguration (Standard: german)
|
||
JONASSEARCH_VECTOR_COLUMN tsvector-Spalte erzwingen (optional)
|
||
|
||
Beispiel:
|
||
from search.engine import SearchEngine
|
||
engine = SearchEngine()
|
||
result = engine.search("raspberry pi", page=1, per_page=10)
|
||
"""
|
||
|
||
import logging
|
||
import os
|
||
import re
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
from typing import List, Optional, Tuple
|
||
|
||
from dotenv import load_dotenv
|
||
import psycopg2
|
||
|
||
load_dotenv()
|
||
|
||
# ============================================================
|
||
# KONFIGURATION
|
||
# ============================================================
|
||
|
||
DB_HOST = os.getenv("JONASSEARCH_DB_HOST", "127.0.0.1")
|
||
DB_PORT = int(os.getenv("JONASSEARCH_DB_PORT", "25432"))
|
||
DB_NAME = os.getenv("JONASSEARCH_DB_NAME", "jonassearch")
|
||
DB_USER = os.getenv("JONASSEARCH_DB_USER", "jonassearch")
|
||
DB_PASSWORD = os.getenv("JONASSEARCH_DB_PASSWORD", "")
|
||
|
||
# PostgreSQL-Textsuch-Konfiguration (Stemming / Stopwörter).
|
||
TS_CONFIG = os.getenv("JONASSEARCH_TS_CONFIG", "german").strip() or "german"
|
||
|
||
# Optional: tsvector-Spalte der Tabelle `pages` erzwingen.
|
||
VECTOR_COLUMN = os.getenv("JONASSEARCH_VECTOR_COLUMN", "").strip()
|
||
|
||
MAX_PER_PAGE = 50
|
||
MAX_TERMS = 32
|
||
|
||
# Suchbegriffe: nur Wortzeichen (auch Umlaute) zulassen.
|
||
_TERM_RE = re.compile(r"\w+", re.UNICODE)
|
||
|
||
# ts_headline-Optionen: kurze Snippets mit <mark>-Hervorhebung.
|
||
HIGHLIGHT_OPTIONS = (
|
||
"StartSel=<mark>, StopSel=</mark>, "
|
||
"MaxWords=32, MinWords=12, MaxFragments=2, "
|
||
"FragmentDelimiter= …"
|
||
)
|
||
|
||
# eslint-ähnlicher Guard, damit ts_headline-Snippets im Browser sicher sind.
|
||
# Der Dokumenttext wird VOR dem Hervorheben escaped (& und <), damit nur die
|
||
# von ts_headline eingefügten <mark>-Tags im Snippet übrig bleiben.
|
||
_ESCAPE_FOR_HEADLINE = (
|
||
"replace(replace(p.content, E'&', '&'), E'<', '<')"
|
||
)
|
||
|
||
|
||
# ============================================================
|
||
# DATENTYPEN
|
||
# ============================================================
|
||
|
||
@dataclass
|
||
class SearchHit:
|
||
"""Ein einzelnes Suchergebnis (eine indexierte Seite)."""
|
||
|
||
url: str
|
||
title: str
|
||
snippet: str
|
||
domain: str = ""
|
||
tld: str = ""
|
||
crawled_at: Optional[str] = None
|
||
rank: float = 0.0
|
||
position: int = 0
|
||
headings: List[str] = field(default_factory=list)
|
||
|
||
def to_dict(self) -> dict:
|
||
return {
|
||
"url": self.url,
|
||
"title": self.title,
|
||
"snippet": self.snippet,
|
||
"domain": self.domain,
|
||
"tld": self.tld,
|
||
"crawled_at": self.crawled_at,
|
||
"rank": round(self.rank, 6),
|
||
"position": self.position,
|
||
"headings": self.headings,
|
||
}
|
||
|
||
|
||
@dataclass
|
||
class SearchResponse:
|
||
"""Antwort einer Suche inkl. Metadaten."""
|
||
|
||
query: str
|
||
total: int
|
||
page: int
|
||
per_page: int
|
||
pages: int
|
||
took_ms: int
|
||
hits: List[SearchHit] = field(default_factory=list)
|
||
|
||
def to_dict(self) -> dict:
|
||
return {
|
||
"query": self.query,
|
||
"total": self.total,
|
||
"page": self.page,
|
||
"per_page": self.per_page,
|
||
"pages": self.pages,
|
||
"took_ms": self.took_ms,
|
||
"hits": [h.to_dict() for h in self.hits],
|
||
}
|
||
|
||
|
||
# ============================================================
|
||
# HILFSFUNKTIONEN
|
||
# ============================================================
|
||
|
||
def parse_terms(raw_query: str) -> List[str]:
|
||
"""
|
||
Zerlegt eine Roh-Abfrage in einzelne Suchbegriffe.
|
||
|
||
Es werden nur Wortzeichen behalten (auch Umlaute), alles andere
|
||
(Interpunktion, Sonderzeichen) wird als Trenner behandelt.
|
||
"""
|
||
terms = [t.lower() for t in _TERM_RE.findall(raw_query or "")]
|
||
return terms[:MAX_TERMS]
|
||
|
||
|
||
def _shorten(text: Optional[str], max_len: int) -> str:
|
||
"""Kürzt Text an einer Wortgrenze und hängt '…' an."""
|
||
if not text:
|
||
return ""
|
||
if len(text) <= max_len:
|
||
return text
|
||
cut = text.rfind(" ", 0, max_len)
|
||
if cut < max_len // 2:
|
||
cut = max_len
|
||
return text[:cut].rstrip() + " …"
|
||
|
||
|
||
def _find_vector(conn) -> Tuple[Optional[str], Optional[str]]:
|
||
"""
|
||
Findet die Suchvektor-Spalte der Tabelle `pages`.
|
||
|
||
Rückgabe: (column_name, None) wenn eine tsvector-Spalte existiert,
|
||
sonst (None, sql_expression) für eine Laufzeit-Berechnung aus
|
||
title/headings/content.
|
||
"""
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"SELECT column_name, data_type "
|
||
"FROM information_schema.columns "
|
||
"WHERE table_name = 'pages' "
|
||
"AND table_schema = ANY (current_schemas(false))"
|
||
)
|
||
columns = dict(cur.fetchall())
|
||
|
||
if not columns:
|
||
raise RuntimeError(
|
||
"Index-Tabelle 'pages' existiert nicht in der Datenbank."
|
||
)
|
||
|
||
# Explizit konfigurierte Spalte hat Vorrang.
|
||
if VECTOR_COLUMN:
|
||
if columns.get(VECTOR_COLUMN) == "tsvector":
|
||
logging.info("Nutze Index-Spalte '%s'.", VECTOR_COLUMN)
|
||
return VECTOR_COLUMN, None
|
||
logging.warning(
|
||
"Konfigurierte Spalte '%s' ist kein tsvector – "
|
||
"Fallback wird verwendet.",
|
||
VECTOR_COLUMN,
|
||
)
|
||
|
||
# Bevorzugt Spalten, die wie ein Suchvektor heißen, sonst irgendein tsvector.
|
||
preferred = [
|
||
name for name in columns
|
||
if columns[name] == "tsvector"
|
||
and ("vector" in name.lower() or "search" in name.lower())
|
||
]
|
||
if preferred:
|
||
logging.info("Nutze Index-Spalte '%s'.", preferred[0])
|
||
return preferred[0], None
|
||
|
||
for name, data_type in columns.items():
|
||
if data_type == "tsvector":
|
||
logging.info("Nutze Index-Spalte '%s'.", name)
|
||
return name, None
|
||
|
||
# Kein tsvector-Index: Vektor zur Laufzeit aufbauen.
|
||
parts = []
|
||
if "title" in columns:
|
||
parts.append("title")
|
||
if "headings" in columns:
|
||
headings_sql = (
|
||
"array_to_string(headings, ' ')"
|
||
if columns["headings"] == "ARRAY"
|
||
else "headings"
|
||
)
|
||
parts.append(headings_sql)
|
||
if "content" in columns:
|
||
parts.append("content")
|
||
|
||
expression = " || ' ' || ".join(parts) if parts else "''"
|
||
logging.warning(
|
||
"Keine tsvector-Spalte in 'pages' gefunden – "
|
||
"Suchvektor wird zur Laufzeit berechnet (%s).",
|
||
expression,
|
||
)
|
||
return None, expression
|
||
|
||
|
||
# ============================================================
|
||
# SUCHMASCHINE
|
||
# ============================================================
|
||
|
||
class SearchEngine:
|
||
"""
|
||
Volltextsuche über die vom Crawler aufgebaute Tabelle `pages`.
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
*,
|
||
host: str = DB_HOST,
|
||
port: int = DB_PORT,
|
||
dbname: str = DB_NAME,
|
||
user: str = DB_USER,
|
||
password: str = DB_PASSWORD,
|
||
ts_config: Optional[str] = None,
|
||
):
|
||
self.host = host
|
||
self.port = port
|
||
self.dbname = dbname
|
||
self.user = user
|
||
self.password = password
|
||
self.ts_config = (ts_config or TS_CONFIG).strip() or "german"
|
||
# Cache für die erkannte Suchvektor-Definition.
|
||
self._vector_cache: Optional[Tuple[Optional[str], Optional[str]]] = None
|
||
|
||
def _connect(self):
|
||
return psycopg2.connect(
|
||
host=self.host,
|
||
port=self.port,
|
||
dbname=self.dbname,
|
||
user=self.user,
|
||
password=self.password,
|
||
connect_timeout=10,
|
||
application_name="jonassearch-search",
|
||
)
|
||
|
||
def _vector(self, conn) -> Tuple[Optional[str], Optional[str]]:
|
||
if self._vector_cache is None:
|
||
self._vector_cache = _find_vector(conn)
|
||
return self._vector_cache
|
||
|
||
def search(
|
||
self,
|
||
query: str,
|
||
page: int = 1,
|
||
per_page: int = 10,
|
||
domain: Optional[str] = None,
|
||
tld: Optional[str] = None,
|
||
) -> SearchResponse:
|
||
"""
|
||
Führt eine Suche aus.
|
||
|
||
`query` kann beliebig formuliert werden; die Begriffe werden
|
||
UND-verknüpft und mit Präfix-Suche ausgeführt. `domain`/`tld`
|
||
schränken auf eine exakte Domain bzw. TLD ein.
|
||
"""
|
||
started = time.monotonic()
|
||
|
||
try:
|
||
page = max(1, int(page))
|
||
except (TypeError, ValueError):
|
||
page = 1
|
||
try:
|
||
per_page = max(1, min(int(per_page), MAX_PER_PAGE))
|
||
except (TypeError, ValueError):
|
||
per_page = 10
|
||
|
||
terms = parse_terms(query)
|
||
|
||
if not terms:
|
||
return self._empty_response(query, page, per_page, started)
|
||
|
||
query_text = " & ".join(f"{term}:*" for term in terms)
|
||
|
||
conn = self._connect()
|
||
try:
|
||
vector_sql, _ = self._prepare_vector(conn)
|
||
query_sql = "to_tsquery(%(config)s, %(querytext)s)"
|
||
|
||
params = {
|
||
"config": self.ts_config,
|
||
"querytext": query_text,
|
||
"hlopts": HIGHLIGHT_OPTIONS,
|
||
"limit": per_page,
|
||
"offset": (page - 1) * per_page,
|
||
}
|
||
|
||
filters = ""
|
||
if domain:
|
||
filters += " AND p.domain = %(domain)s"
|
||
params["domain"] = domain
|
||
if tld:
|
||
filters += " AND p.tld = %(tld)s"
|
||
params["tld"] = tld
|
||
|
||
count_sql = (
|
||
"SELECT count(*) FROM pages p "
|
||
f"WHERE {vector_sql} @@ {query_sql}{filters}"
|
||
)
|
||
|
||
select_sql = f"""
|
||
SELECT
|
||
p.url,
|
||
p.title,
|
||
p.domain,
|
||
p.tld,
|
||
p.crawled_at,
|
||
p.headings,
|
||
ts_headline(
|
||
%(config)s,
|
||
{_ESCAPE_FOR_HEADLINE},
|
||
{query_sql},
|
||
%(hlopts)s
|
||
) AS snippet,
|
||
ts_rank({vector_sql}, {query_sql}) AS rank
|
||
FROM pages p
|
||
WHERE {vector_sql} @@ {query_sql}{filters}
|
||
ORDER BY rank DESC, p.crawled_at DESC, p.url ASC
|
||
LIMIT %(limit)s OFFSET %(offset)s
|
||
"""
|
||
|
||
with conn.cursor() as cur:
|
||
try:
|
||
cur.execute(count_sql, params)
|
||
except psycopg2.Error as exc:
|
||
# Z. B. wenn die Abfrage nur aus Stopwörtern besteht
|
||
# (to_tsquery liefert dann einen leeren Suchvektor).
|
||
logging.info("Abfrage ohne Treffer-Möglichkeit: %s", exc)
|
||
return self._empty_response(query, page, per_page, started)
|
||
|
||
total = cur.fetchone()[0]
|
||
|
||
cur.execute(select_sql, params)
|
||
rows = cur.fetchall()
|
||
|
||
hits = []
|
||
for index, row in enumerate(rows):
|
||
(
|
||
url,
|
||
title,
|
||
dom,
|
||
tld_value,
|
||
crawled_at,
|
||
headings,
|
||
snippet,
|
||
rank,
|
||
) = row
|
||
|
||
title = title or url
|
||
snippet = snippet or title
|
||
|
||
hits.append(
|
||
SearchHit(
|
||
url=url,
|
||
title=_shorten(title, 300),
|
||
snippet=_shorten(snippet, 600),
|
||
domain=dom or "",
|
||
tld=tld_value or "",
|
||
crawled_at=crawled_at.isoformat()
|
||
if crawled_at is not None
|
||
else None,
|
||
rank=float(rank or 0.0),
|
||
position=(page - 1) * per_page + index + 1,
|
||
headings=[
|
||
_shorten(h, 200) for h in (headings or [])[:5]
|
||
],
|
||
)
|
||
)
|
||
|
||
total = int(total or 0)
|
||
|
||
return SearchResponse(
|
||
query=query,
|
||
total=total,
|
||
page=page,
|
||
per_page=per_page,
|
||
pages=(total + per_page - 1) // per_page if total else 0,
|
||
took_ms=int((time.monotonic() - started) * 1000),
|
||
hits=hits,
|
||
)
|
||
|
||
finally:
|
||
conn.close()
|
||
|
||
def _prepare_vector(self, conn) -> Tuple[str, None]:
|
||
"""
|
||
Liefert das SQL für den Suchvektor:
|
||
- vorhandene tsvector-Spalte, z. B. `search_vector`
|
||
- oder Laufzeit-Ausdruck `to_tsvector(%(config)s, ...)`
|
||
"""
|
||
column, expression = self._vector(conn)
|
||
|
||
if column is not None:
|
||
return column, None
|
||
|
||
return f"to_tsvector(%(config)s, {expression})", None
|
||
|
||
def _empty_response(
|
||
self,
|
||
query: str,
|
||
page: int,
|
||
per_page: int,
|
||
started: float,
|
||
) -> SearchResponse:
|
||
return SearchResponse(
|
||
query=query,
|
||
total=0,
|
||
page=page,
|
||
per_page=per_page,
|
||
pages=0,
|
||
took_ms=int((time.monotonic() - started) * 1000),
|
||
hits=[],
|
||
) |