"""Suchmaschine fuer SjAudioSearch. Ablauf einer Suche: 1. Anfrage normalisieren. 2. Kandidaten ueber FTS5 und den Trigramm-Index sammeln (schnelle Vorauswahl). 3. Sliding-Window-Vergleich mit RapidFuzz auf der Wort-Timeline der Kandidaten. 4. Optionaler, vollstaendiger Fallback-Lauf ueber alle verbleibenden Dateien, damit auch komplett falsch erkannte Woerter gefunden werden. 5. Treffer entduplizieren, sortieren, begrenzen. """ from __future__ import annotations from dataclasses import dataclass from typing import Callable from rapidfuzz import fuzz from ..config import SearchConfig from ..database.repository import Repository from ..utils.logging import get_logger from .context import SearchContext, Timeline, build_context, build_timeline from .fuzzy import ( build_fts_query, normalize_text, score_window, significant_overlap, trigram_terms, ) from .ranking import dedupe_overlapping, rank_hits logger = get_logger("search.engine") ProgressCallback = Callable[[int, int], None] class SearchError(RuntimeError): """Fehler bei der Suche (z. B. leere Anfrage).""" @dataclass class SearchHit: """Ein einzelner Suchtreffer.""" audio_id: int filename: str path: str score: float start_time: float end_time: float start_token: int end_token: int context: SearchContext language: str | None = None model: str | None = None source: str = "fts" @property def matched_text(self) -> str: return self.context.match def to_dict(self, context_words: int = 50) -> dict[str, object]: return { "audio_id": self.audio_id, "filename": self.filename, "path": self.path, "similarity": round(self.score, 2), "timestamp": self.start_time, "timestamp_end": self.end_time, "timestamp_precision": self.context.precision, "match": self.context.match, "context_before": self.context.before, "context_after": self.context.after, "context_words": context_words, "language": self.language, "model": self.model, "source": self.source, } class SearchEngine: """Kombinierte Suche aus FTS5, Trigrammen und Fuzzy-Sliding-Window.""" def __init__(self, repository: Repository, config: SearchConfig): self.repo = repository self.config = config # ------------------------------------------------------------------ def search( self, query: str, *, limit: int | None = None, min_score: float | None = None, context_words: int | None = None, language: str | None = None, progress: ProgressCallback | None = None, ) -> list[SearchHit]: query_normalized = normalize_text(query) if not query_normalized: raise SearchError("Die Suchanfrage enthaelt keine verwertbaren Woerter.") query_tokens = query_normalized.split() limit = self.config.default_limit if limit is None else int(limit) min_score = self.config.default_min_score if min_score is None else float(min_score) context_words = self.config.context_words if context_words is None else int(context_words) if limit <= 0: raise SearchError("--limit muss groesser als 0 sein.") candidate_audio, candidate_source = self._collect_candidates(query_tokens) hits: list[SearchHit] = [] swept: set[int] = set() total_candidates = len(candidate_audio) for index, audio_id in enumerate(candidate_audio, start=1): swept.add(audio_id) hits.extend( self._scan_audio( audio_id, query_normalized, query_tokens, min_score=min_score, context_words=context_words, language=language, source="fts", ) ) if progress: progress(index, total_candidates) # Wichtig: FTS5 ist nur eine Vorauswahl. Wenn wenige Treffer gefunden # wurden, werden alle uebrigen Dateien vollstaendig durchsucht. fallback_used = False if self.config.exhaustive_fallback and len(hits) < limit: remaining = [ audio_id for audio_id in self.repo.iter_audio_ids_with_transcripts(language) if audio_id not in swept ] fallback_used = bool(remaining) for index, audio_id in enumerate(remaining, start=1): hits.extend( self._scan_audio( audio_id, query_normalized, query_tokens, min_score=min_score, context_words=context_words, language=language, source="fallback", ) ) if progress: progress(index, len(remaining)) if fallback_used: logger.info( "Fallback-Suche aktiv: %d Dateien ohne FTS-Treffer durchsucht", len(remaining) ) deduped = dedupe_overlapping(hits) return rank_hits(deduped, limit) # ------------------------------------------------------------------ def _collect_candidates(self, query_tokens: list[str]) -> tuple[list[int], str]: """Sammelt Kandidaten-Audiodateien (FTS5 + Trigramme).""" limit = max(1, int(self.config.candidate_limit)) matches: list[tuple[int, int]] = [] fts_query = build_fts_query(query_tokens) if fts_query: matches.extend(self.repo.fts_candidates(fts_query, limit)) remaining = limit - len(matches) if remaining > 0: terms = trigram_terms(query_tokens) if terms: matches.extend(self.repo.trigram_candidates(terms, remaining)) counts: dict[int, int] = {} for audio_id, _segment_id in matches: counts[audio_id] = counts.get(audio_id, 0) + 1 ordered = sorted(counts, key=lambda audio_id: (-counts[audio_id], audio_id)) return ordered, "fts" # ------------------------------------------------------------------ def _scan_audio( self, audio_id: int, query_normalized: str, query_tokens: list[str], *, min_score: float, context_words: int, language: str | None, source: str, ) -> list[SearchHit]: record = self.repo.get_audio(audio_id) if record is None or record.duplicate_of is not None: return [] transcript = self.repo.get_transcript(audio_id) if transcript is None: return [] if language and (transcript["language"] or "").lower() != language.lower(): return [] segments, words = self.repo.timeline(audio_id) if not segments: return [] timeline = build_timeline( audio_id, segments, words, language=transcript["language"], model=transcript["model"], ) if not timeline.tokens: return [] windows = self._find_windows(timeline, query_normalized, query_tokens, min_score) if not windows: return [] hits: list[SearchHit] = [] for score, start_index, end_index in windows: context = build_context(timeline, start_index, end_index, context_words) hits.append( SearchHit( audio_id=audio_id, filename=record.filename, path=record.path, score=score, start_time=context.start_time, end_time=context.end_time, start_token=start_index, end_token=end_index, context=context, language=transcript["language"], model=transcript["model"], source=source, ) ) hits.sort(key=lambda hit: -hit.score) return hits[: max(1, int(self.config.max_hits_per_file))] # ------------------------------------------------------------------ def _find_windows( self, timeline: Timeline, query_normalized: str, query_tokens: list[str], min_score: float, ) -> list[tuple[float, int, int]]: """Sliding-Window-Suche ueber die Wort-Timeline einer Datei.""" norm_tokens = timeline.normalized_tokens token_count = len(norm_tokens) query_len = len(query_tokens) if token_count == 0: return [] min_size = max(1, query_len - max(0, self.config.window_before)) max_size = max(min_size, query_len + max(0, self.config.window_after)) overlap_threshold = max(0.0, min(1.0, self.config.min_token_overlap)) # Dateiebene: gibt es ueberhaupt Beruehrungspunkte? Das ist deutlich # billiger als jede Datei Fenster fuer Fenster zu vergleichen. file_overlap = significant_overlap(query_tokens, norm_tokens) if file_overlap < overlap_threshold * 0.5: partial = fuzz.partial_ratio(query_normalized, timeline.normalized_text) if partial < 55.0: return [] windows: list[tuple[float, int, int]] = [] # Wenn auf Dateiebene kaum Wortueberschneidung besteht, kann der Treffer # nur ueber einen falsch erkannten Ausdruck zustande kommen. Dann wird # ohne Wort-Vorfilter verglichen (selten und durch die Dateipruefung # oben bereits eingegrenzt). use_overlap_filter = file_overlap >= overlap_threshold for start in range(token_count): pre_end = min(token_count, start + query_len + 2) if use_overlap_filter and significant_overlap(query_tokens, norm_tokens[start:pre_end]) < overlap_threshold: continue best_score = 0.0 best_size = 0 for size in range(min_size, max_size + 1): end = start + size if end > token_count: break window_normalized = " ".join(norm_tokens[start:end]) candidate = score_window(query_normalized, window_normalized) if candidate.score > best_score: best_score = candidate.score best_size = size if best_size and best_score >= min_score: windows.append((best_score, start, start + best_size)) return windows