Files
sjmusicsearch/sjaudiosearch/search/engine.py
T
2026-11-16 16:29:32 +01:00

304 lines
11 KiB
Python

"""Suchmaschine fuer SjAudioSearch.
Ablauf einer Suche:
1. Anfrage normalisieren.
2. Kandidaten ueber FTS5 und den Trigramm-Index sammeln (schnelle Vorauswahl).
3. Sliding-Window-Vergleich mit RapidFuzz auf der Wort-Timeline der Kandidaten.
4. Optionaler, vollstaendiger Fallback-Lauf ueber alle verbleibenden Dateien,
damit auch komplett falsch erkannte Woerter gefunden werden.
5. Treffer entduplizieren, sortieren, begrenzen.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Callable
from rapidfuzz import fuzz
from ..config import SearchConfig
from ..database.repository import Repository
from ..utils.logging import get_logger
from .context import SearchContext, Timeline, build_context, build_timeline
from .fuzzy import (
build_fts_query,
normalize_text,
score_window,
significant_overlap,
trigram_terms,
)
from .ranking import dedupe_overlapping, rank_hits
logger = get_logger("search.engine")
ProgressCallback = Callable[[int, int], None]
class SearchError(RuntimeError):
"""Fehler bei der Suche (z. B. leere Anfrage)."""
@dataclass
class SearchHit:
"""Ein einzelner Suchtreffer."""
audio_id: int
filename: str
path: str
score: float
start_time: float
end_time: float
start_token: int
end_token: int
context: SearchContext
language: str | None = None
model: str | None = None
source: str = "fts"
@property
def matched_text(self) -> str:
return self.context.match
def to_dict(self, context_words: int = 50) -> dict[str, object]:
return {
"audio_id": self.audio_id,
"filename": self.filename,
"path": self.path,
"similarity": round(self.score, 2),
"timestamp": self.start_time,
"timestamp_end": self.end_time,
"timestamp_precision": self.context.precision,
"match": self.context.match,
"context_before": self.context.before,
"context_after": self.context.after,
"context_words": context_words,
"language": self.language,
"model": self.model,
"source": self.source,
}
class SearchEngine:
"""Kombinierte Suche aus FTS5, Trigrammen und Fuzzy-Sliding-Window."""
def __init__(self, repository: Repository, config: SearchConfig):
self.repo = repository
self.config = config
# ------------------------------------------------------------------
def search(
self,
query: str,
*,
limit: int | None = None,
min_score: float | None = None,
context_words: int | None = None,
language: str | None = None,
progress: ProgressCallback | None = None,
) -> list[SearchHit]:
query_normalized = normalize_text(query)
if not query_normalized:
raise SearchError("Die Suchanfrage enthaelt keine verwertbaren Woerter.")
query_tokens = query_normalized.split()
limit = self.config.default_limit if limit is None else int(limit)
min_score = self.config.default_min_score if min_score is None else float(min_score)
context_words = self.config.context_words if context_words is None else int(context_words)
if limit <= 0:
raise SearchError("--limit muss groesser als 0 sein.")
candidate_audio, candidate_source = self._collect_candidates(query_tokens)
hits: list[SearchHit] = []
swept: set[int] = set()
total_candidates = len(candidate_audio)
for index, audio_id in enumerate(candidate_audio, start=1):
swept.add(audio_id)
hits.extend(
self._scan_audio(
audio_id,
query_normalized,
query_tokens,
min_score=min_score,
context_words=context_words,
language=language,
source="fts",
)
)
if progress:
progress(index, total_candidates)
# Wichtig: FTS5 ist nur eine Vorauswahl. Wenn wenige Treffer gefunden
# wurden, werden alle uebrigen Dateien vollstaendig durchsucht.
fallback_used = False
if self.config.exhaustive_fallback and len(hits) < limit:
remaining = [
audio_id
for audio_id in self.repo.iter_audio_ids_with_transcripts(language)
if audio_id not in swept
]
fallback_used = bool(remaining)
for index, audio_id in enumerate(remaining, start=1):
hits.extend(
self._scan_audio(
audio_id,
query_normalized,
query_tokens,
min_score=min_score,
context_words=context_words,
language=language,
source="fallback",
)
)
if progress:
progress(index, len(remaining))
if fallback_used:
logger.info(
"Fallback-Suche aktiv: %d Dateien ohne FTS-Treffer durchsucht", len(remaining)
)
deduped = dedupe_overlapping(hits)
return rank_hits(deduped, limit)
# ------------------------------------------------------------------
def _collect_candidates(self, query_tokens: list[str]) -> tuple[list[int], str]:
"""Sammelt Kandidaten-Audiodateien (FTS5 + Trigramme)."""
limit = max(1, int(self.config.candidate_limit))
matches: list[tuple[int, int]] = []
fts_query = build_fts_query(query_tokens)
if fts_query:
matches.extend(self.repo.fts_candidates(fts_query, limit))
remaining = limit - len(matches)
if remaining > 0:
terms = trigram_terms(query_tokens)
if terms:
matches.extend(self.repo.trigram_candidates(terms, remaining))
counts: dict[int, int] = {}
for audio_id, _segment_id in matches:
counts[audio_id] = counts.get(audio_id, 0) + 1
ordered = sorted(counts, key=lambda audio_id: (-counts[audio_id], audio_id))
return ordered, "fts"
# ------------------------------------------------------------------
def _scan_audio(
self,
audio_id: int,
query_normalized: str,
query_tokens: list[str],
*,
min_score: float,
context_words: int,
language: str | None,
source: str,
) -> list[SearchHit]:
record = self.repo.get_audio(audio_id)
if record is None or record.duplicate_of is not None:
return []
transcript = self.repo.get_transcript(audio_id)
if transcript is None:
return []
if language and (transcript["language"] or "").lower() != language.lower():
return []
segments, words = self.repo.timeline(audio_id)
if not segments:
return []
timeline = build_timeline(
audio_id,
segments,
words,
language=transcript["language"],
model=transcript["model"],
)
if not timeline.tokens:
return []
windows = self._find_windows(timeline, query_normalized, query_tokens, min_score)
if not windows:
return []
hits: list[SearchHit] = []
for score, start_index, end_index in windows:
context = build_context(timeline, start_index, end_index, context_words)
hits.append(
SearchHit(
audio_id=audio_id,
filename=record.filename,
path=record.path,
score=score,
start_time=context.start_time,
end_time=context.end_time,
start_token=start_index,
end_token=end_index,
context=context,
language=transcript["language"],
model=transcript["model"],
source=source,
)
)
hits.sort(key=lambda hit: -hit.score)
return hits[: max(1, int(self.config.max_hits_per_file))]
# ------------------------------------------------------------------
def _find_windows(
self,
timeline: Timeline,
query_normalized: str,
query_tokens: list[str],
min_score: float,
) -> list[tuple[float, int, int]]:
"""Sliding-Window-Suche ueber die Wort-Timeline einer Datei."""
norm_tokens = timeline.normalized_tokens
token_count = len(norm_tokens)
query_len = len(query_tokens)
if token_count == 0:
return []
min_size = max(1, query_len - max(0, self.config.window_before))
max_size = max(min_size, query_len + max(0, self.config.window_after))
overlap_threshold = max(0.0, min(1.0, self.config.min_token_overlap))
# Dateiebene: gibt es ueberhaupt Beruehrungspunkte? Das ist deutlich
# billiger als jede Datei Fenster fuer Fenster zu vergleichen.
file_overlap = significant_overlap(query_tokens, norm_tokens)
if file_overlap < overlap_threshold * 0.5:
partial = fuzz.partial_ratio(query_normalized, timeline.normalized_text)
if partial < 55.0:
return []
windows: list[tuple[float, int, int]] = []
# Wenn auf Dateiebene kaum Wortueberschneidung besteht, kann der Treffer
# nur ueber einen falsch erkannten Ausdruck zustande kommen. Dann wird
# ohne Wort-Vorfilter verglichen (selten und durch die Dateipruefung
# oben bereits eingegrenzt).
use_overlap_filter = file_overlap >= overlap_threshold
for start in range(token_count):
pre_end = min(token_count, start + query_len + 2)
if use_overlap_filter and significant_overlap(query_tokens, norm_tokens[start:pre_end]) < overlap_threshold:
continue
best_score = 0.0
best_size = 0
for size in range(min_size, max_size + 1):
end = start + size
if end > token_count:
break
window_normalized = " ".join(norm_tokens[start:end])
candidate = score_window(query_normalized, window_normalized)
if candidate.score > best_score:
best_score = candidate.score
best_size = size
if best_size and best_score >= min_score:
windows.append((best_score, start, start + best_size))
return windows