add search
This commit is contained in:
@@ -0,0 +1,451 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
JonasSearch – Suchmaschine (Engine).
|
||||
|
||||
Volltextsuche über den Index, den crawler.py in PostgreSQL aufgebaut hat
|
||||
(Tabelle `pages`).
|
||||
|
||||
Die Engine nutzt, wenn vorhanden, die vom Trigger gepflegte tsvector-Spalte
|
||||
der Tabelle `pages` (Suchvektor) und sucht damit mit PostgreSQL's
|
||||
Volltextsuche. Fehlt eine solche Spalte (z. B. weil die Tabelle manuell
|
||||
angelegt wurde), wird der Suchvektor zur Laufzeit aus title/headings/content
|
||||
berechnet – die Suche funktioniert dann genauso, nur langsamer.
|
||||
|
||||
Features:
|
||||
- Rangfolge über ts_rank (Titel/Inhalt gewichtet nach Index-Spalte)
|
||||
- Snippets mit hervorgehobenen Suchbegriffen (ts_headline, <mark>)
|
||||
- Präfix-Suche ("raspberr" findet "raspberry")
|
||||
- UND-Verknüpfung der Suchbegriffe, Phrasen über Anführungszeichen nicht nötig
|
||||
- Paginierung
|
||||
- Filter nach Domain und TLD
|
||||
|
||||
Konfiguration über Umgebungsvariablen (siehe .env.example):
|
||||
JONASSEARCH_DB_HOST / PORT / NAME / USER / PASSWORD
|
||||
JONASSEARCH_TS_CONFIG Textsuch-Konfiguration (Standard: german)
|
||||
JONASSEARCH_VECTOR_COLUMN tsvector-Spalte erzwingen (optional)
|
||||
|
||||
Beispiel:
|
||||
from search.engine import SearchEngine
|
||||
engine = SearchEngine()
|
||||
result = engine.search("raspberry pi", page=1, per_page=10)
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from dotenv import load_dotenv
|
||||
import psycopg2
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# ============================================================
|
||||
# KONFIGURATION
|
||||
# ============================================================
|
||||
|
||||
DB_HOST = os.getenv("JONASSEARCH_DB_HOST", "127.0.0.1")
|
||||
DB_PORT = int(os.getenv("JONASSEARCH_DB_PORT", "25432"))
|
||||
DB_NAME = os.getenv("JONASSEARCH_DB_NAME", "jonassearch")
|
||||
DB_USER = os.getenv("JONASSEARCH_DB_USER", "jonassearch")
|
||||
DB_PASSWORD = os.getenv("JONASSEARCH_DB_PASSWORD", "")
|
||||
|
||||
# PostgreSQL-Textsuch-Konfiguration (Stemming / Stopwörter).
|
||||
TS_CONFIG = os.getenv("JONASSEARCH_TS_CONFIG", "german").strip() or "german"
|
||||
|
||||
# Optional: tsvector-Spalte der Tabelle `pages` erzwingen.
|
||||
VECTOR_COLUMN = os.getenv("JONASSEARCH_VECTOR_COLUMN", "").strip()
|
||||
|
||||
MAX_PER_PAGE = 50
|
||||
MAX_TERMS = 32
|
||||
|
||||
# Suchbegriffe: nur Wortzeichen (auch Umlaute) zulassen.
|
||||
_TERM_RE = re.compile(r"\w+", re.UNICODE)
|
||||
|
||||
# ts_headline-Optionen: kurze Snippets mit <mark>-Hervorhebung.
|
||||
HIGHLIGHT_OPTIONS = (
|
||||
"StartSel=<mark>, StopSel=</mark>, "
|
||||
"MaxWords=32, MinWords=12, MaxFragments=2, "
|
||||
"FragmentDelimiter= …"
|
||||
)
|
||||
|
||||
# eslint-ähnlicher Guard, damit ts_headline-Snippets im Browser sicher sind.
|
||||
# Der Dokumenttext wird VOR dem Hervorheben escaped (& und <), damit nur die
|
||||
# von ts_headline eingefügten <mark>-Tags im Snippet übrig bleiben.
|
||||
_ESCAPE_FOR_HEADLINE = (
|
||||
"replace(replace(p.content, E'&', '&'), E'<', '<')"
|
||||
)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# DATENTYPEN
|
||||
# ============================================================
|
||||
|
||||
@dataclass
|
||||
class SearchHit:
|
||||
"""Ein einzelnes Suchergebnis (eine indexierte Seite)."""
|
||||
|
||||
url: str
|
||||
title: str
|
||||
snippet: str
|
||||
domain: str = ""
|
||||
tld: str = ""
|
||||
crawled_at: Optional[str] = None
|
||||
rank: float = 0.0
|
||||
position: int = 0
|
||||
headings: List[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"url": self.url,
|
||||
"title": self.title,
|
||||
"snippet": self.snippet,
|
||||
"domain": self.domain,
|
||||
"tld": self.tld,
|
||||
"crawled_at": self.crawled_at,
|
||||
"rank": round(self.rank, 6),
|
||||
"position": self.position,
|
||||
"headings": self.headings,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResponse:
|
||||
"""Antwort einer Suche inkl. Metadaten."""
|
||||
|
||||
query: str
|
||||
total: int
|
||||
page: int
|
||||
per_page: int
|
||||
pages: int
|
||||
took_ms: int
|
||||
hits: List[SearchHit] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"query": self.query,
|
||||
"total": self.total,
|
||||
"page": self.page,
|
||||
"per_page": self.per_page,
|
||||
"pages": self.pages,
|
||||
"took_ms": self.took_ms,
|
||||
"hits": [h.to_dict() for h in self.hits],
|
||||
}
|
||||
|
||||
|
||||
# ============================================================
|
||||
# HILFSFUNKTIONEN
|
||||
# ============================================================
|
||||
|
||||
def parse_terms(raw_query: str) -> List[str]:
|
||||
"""
|
||||
Zerlegt eine Roh-Abfrage in einzelne Suchbegriffe.
|
||||
|
||||
Es werden nur Wortzeichen behalten (auch Umlaute), alles andere
|
||||
(Interpunktion, Sonderzeichen) wird als Trenner behandelt.
|
||||
"""
|
||||
terms = [t.lower() for t in _TERM_RE.findall(raw_query or "")]
|
||||
return terms[:MAX_TERMS]
|
||||
|
||||
|
||||
def _shorten(text: Optional[str], max_len: int) -> str:
|
||||
"""Kürzt Text an einer Wortgrenze und hängt '…' an."""
|
||||
if not text:
|
||||
return ""
|
||||
if len(text) <= max_len:
|
||||
return text
|
||||
cut = text.rfind(" ", 0, max_len)
|
||||
if cut < max_len // 2:
|
||||
cut = max_len
|
||||
return text[:cut].rstrip() + " …"
|
||||
|
||||
|
||||
def _find_vector(conn) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
Findet die Suchvektor-Spalte der Tabelle `pages`.
|
||||
|
||||
Rückgabe: (column_name, None) wenn eine tsvector-Spalte existiert,
|
||||
sonst (None, sql_expression) für eine Laufzeit-Berechnung aus
|
||||
title/headings/content.
|
||||
"""
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT column_name, data_type "
|
||||
"FROM information_schema.columns "
|
||||
"WHERE table_name = 'pages' "
|
||||
"AND table_schema = ANY (current_schemas(false))"
|
||||
)
|
||||
columns = dict(cur.fetchall())
|
||||
|
||||
if not columns:
|
||||
raise RuntimeError(
|
||||
"Index-Tabelle 'pages' existiert nicht in der Datenbank."
|
||||
)
|
||||
|
||||
# Explizit konfigurierte Spalte hat Vorrang.
|
||||
if VECTOR_COLUMN:
|
||||
if columns.get(VECTOR_COLUMN) == "tsvector":
|
||||
logging.info("Nutze Index-Spalte '%s'.", VECTOR_COLUMN)
|
||||
return VECTOR_COLUMN, None
|
||||
logging.warning(
|
||||
"Konfigurierte Spalte '%s' ist kein tsvector – "
|
||||
"Fallback wird verwendet.",
|
||||
VECTOR_COLUMN,
|
||||
)
|
||||
|
||||
# Bevorzugt Spalten, die wie ein Suchvektor heißen, sonst irgendein tsvector.
|
||||
preferred = [
|
||||
name for name in columns
|
||||
if columns[name] == "tsvector"
|
||||
and ("vector" in name.lower() or "search" in name.lower())
|
||||
]
|
||||
if preferred:
|
||||
logging.info("Nutze Index-Spalte '%s'.", preferred[0])
|
||||
return preferred[0], None
|
||||
|
||||
for name, data_type in columns.items():
|
||||
if data_type == "tsvector":
|
||||
logging.info("Nutze Index-Spalte '%s'.", name)
|
||||
return name, None
|
||||
|
||||
# Kein tsvector-Index: Vektor zur Laufzeit aufbauen.
|
||||
parts = []
|
||||
if "title" in columns:
|
||||
parts.append("title")
|
||||
if "headings" in columns:
|
||||
headings_sql = (
|
||||
"array_to_string(headings, ' ')"
|
||||
if columns["headings"] == "ARRAY"
|
||||
else "headings"
|
||||
)
|
||||
parts.append(headings_sql)
|
||||
if "content" in columns:
|
||||
parts.append("content")
|
||||
|
||||
expression = " || ' ' || ".join(parts) if parts else "''"
|
||||
logging.warning(
|
||||
"Keine tsvector-Spalte in 'pages' gefunden – "
|
||||
"Suchvektor wird zur Laufzeit berechnet (%s).",
|
||||
expression,
|
||||
)
|
||||
return None, expression
|
||||
|
||||
|
||||
# ============================================================
|
||||
# SUCHMASCHINE
|
||||
# ============================================================
|
||||
|
||||
class SearchEngine:
|
||||
"""
|
||||
Volltextsuche über die vom Crawler aufgebaute Tabelle `pages`.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
host: str = DB_HOST,
|
||||
port: int = DB_PORT,
|
||||
dbname: str = DB_NAME,
|
||||
user: str = DB_USER,
|
||||
password: str = DB_PASSWORD,
|
||||
ts_config: Optional[str] = None,
|
||||
):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.dbname = dbname
|
||||
self.user = user
|
||||
self.password = password
|
||||
self.ts_config = (ts_config or TS_CONFIG).strip() or "german"
|
||||
# Cache für die erkannte Suchvektor-Definition.
|
||||
self._vector_cache: Optional[Tuple[Optional[str], Optional[str]]] = None
|
||||
|
||||
def _connect(self):
|
||||
return psycopg2.connect(
|
||||
host=self.host,
|
||||
port=self.port,
|
||||
dbname=self.dbname,
|
||||
user=self.user,
|
||||
password=self.password,
|
||||
connect_timeout=10,
|
||||
application_name="jonassearch-search",
|
||||
)
|
||||
|
||||
def _vector(self, conn) -> Tuple[Optional[str], Optional[str]]:
|
||||
if self._vector_cache is None:
|
||||
self._vector_cache = _find_vector(conn)
|
||||
return self._vector_cache
|
||||
|
||||
def search(
|
||||
self,
|
||||
query: str,
|
||||
page: int = 1,
|
||||
per_page: int = 10,
|
||||
domain: Optional[str] = None,
|
||||
tld: Optional[str] = None,
|
||||
) -> SearchResponse:
|
||||
"""
|
||||
Führt eine Suche aus.
|
||||
|
||||
`query` kann beliebig formuliert werden; die Begriffe werden
|
||||
UND-verknüpft und mit Präfix-Suche ausgeführt. `domain`/`tld`
|
||||
schränken auf eine exakte Domain bzw. TLD ein.
|
||||
"""
|
||||
started = time.monotonic()
|
||||
|
||||
try:
|
||||
page = max(1, int(page))
|
||||
except (TypeError, ValueError):
|
||||
page = 1
|
||||
try:
|
||||
per_page = max(1, min(int(per_page), MAX_PER_PAGE))
|
||||
except (TypeError, ValueError):
|
||||
per_page = 10
|
||||
|
||||
terms = parse_terms(query)
|
||||
|
||||
if not terms:
|
||||
return self._empty_response(query, page, per_page, started)
|
||||
|
||||
query_text = " & ".join(f"{term}:*" for term in terms)
|
||||
|
||||
conn = self._connect()
|
||||
try:
|
||||
vector_sql, _ = self._prepare_vector(conn)
|
||||
query_sql = "to_tsquery(%(config)s, %(querytext)s)"
|
||||
|
||||
params = {
|
||||
"config": self.ts_config,
|
||||
"querytext": query_text,
|
||||
"hlopts": HIGHLIGHT_OPTIONS,
|
||||
"limit": per_page,
|
||||
"offset": (page - 1) * per_page,
|
||||
}
|
||||
|
||||
filters = ""
|
||||
if domain:
|
||||
filters += " AND p.domain = %(domain)s"
|
||||
params["domain"] = domain
|
||||
if tld:
|
||||
filters += " AND p.tld = %(tld)s"
|
||||
params["tld"] = tld
|
||||
|
||||
count_sql = (
|
||||
"SELECT count(*) FROM pages p "
|
||||
f"WHERE {vector_sql} @@ {query_sql}{filters}"
|
||||
)
|
||||
|
||||
select_sql = f"""
|
||||
SELECT
|
||||
p.url,
|
||||
p.title,
|
||||
p.domain,
|
||||
p.tld,
|
||||
p.crawled_at,
|
||||
p.headings,
|
||||
ts_headline(
|
||||
%(config)s,
|
||||
{_ESCAPE_FOR_HEADLINE},
|
||||
{query_sql},
|
||||
%(hlopts)s
|
||||
) AS snippet,
|
||||
ts_rank({vector_sql}, {query_sql}) AS rank
|
||||
FROM pages p
|
||||
WHERE {vector_sql} @@ {query_sql}{filters}
|
||||
ORDER BY rank DESC, p.crawled_at DESC, p.url ASC
|
||||
LIMIT %(limit)s OFFSET %(offset)s
|
||||
"""
|
||||
|
||||
with conn.cursor() as cur:
|
||||
try:
|
||||
cur.execute(count_sql, params)
|
||||
except psycopg2.Error as exc:
|
||||
# Z. B. wenn die Abfrage nur aus Stopwörtern besteht
|
||||
# (to_tsquery liefert dann einen leeren Suchvektor).
|
||||
logging.info("Abfrage ohne Treffer-Möglichkeit: %s", exc)
|
||||
return self._empty_response(query, page, per_page, started)
|
||||
|
||||
total = cur.fetchone()[0]
|
||||
|
||||
cur.execute(select_sql, params)
|
||||
rows = cur.fetchall()
|
||||
|
||||
hits = []
|
||||
for index, row in enumerate(rows):
|
||||
(
|
||||
url,
|
||||
title,
|
||||
dom,
|
||||
tld_value,
|
||||
crawled_at,
|
||||
headings,
|
||||
snippet,
|
||||
rank,
|
||||
) = row
|
||||
|
||||
title = title or url
|
||||
snippet = snippet or title
|
||||
|
||||
hits.append(
|
||||
SearchHit(
|
||||
url=url,
|
||||
title=_shorten(title, 300),
|
||||
snippet=_shorten(snippet, 600),
|
||||
domain=dom or "",
|
||||
tld=tld_value or "",
|
||||
crawled_at=crawled_at.isoformat()
|
||||
if crawled_at is not None
|
||||
else None,
|
||||
rank=float(rank or 0.0),
|
||||
position=(page - 1) * per_page + index + 1,
|
||||
headings=[
|
||||
_shorten(h, 200) for h in (headings or [])[:5]
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
total = int(total or 0)
|
||||
|
||||
return SearchResponse(
|
||||
query=query,
|
||||
total=total,
|
||||
page=page,
|
||||
per_page=per_page,
|
||||
pages=(total + per_page - 1) // per_page if total else 0,
|
||||
took_ms=int((time.monotonic() - started) * 1000),
|
||||
hits=hits,
|
||||
)
|
||||
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _prepare_vector(self, conn) -> Tuple[str, None]:
|
||||
"""
|
||||
Liefert das SQL für den Suchvektor:
|
||||
- vorhandene tsvector-Spalte, z. B. `search_vector`
|
||||
- oder Laufzeit-Ausdruck `to_tsvector(%(config)s, ...)`
|
||||
"""
|
||||
column, expression = self._vector(conn)
|
||||
|
||||
if column is not None:
|
||||
return column, None
|
||||
|
||||
return f"to_tsvector(%(config)s, {expression})", None
|
||||
|
||||
def _empty_response(
|
||||
self,
|
||||
query: str,
|
||||
page: int,
|
||||
per_page: int,
|
||||
started: float,
|
||||
) -> SearchResponse:
|
||||
return SearchResponse(
|
||||
query=query,
|
||||
total=0,
|
||||
page=page,
|
||||
per_page=per_page,
|
||||
pages=0,
|
||||
took_ms=int((time.monotonic() - started) * 1000),
|
||||
hits=[],
|
||||
)
|
||||
Reference in New Issue
Block a user