304 lines
11 KiB
Python
304 lines
11 KiB
Python
"""Suchmaschine fuer SjAudioSearch.
|
|
|
|
Ablauf einer Suche:
|
|
|
|
1. Anfrage normalisieren.
|
|
2. Kandidaten ueber FTS5 und den Trigramm-Index sammeln (schnelle Vorauswahl).
|
|
3. Sliding-Window-Vergleich mit RapidFuzz auf der Wort-Timeline der Kandidaten.
|
|
4. Optionaler, vollstaendiger Fallback-Lauf ueber alle verbleibenden Dateien,
|
|
damit auch komplett falsch erkannte Woerter gefunden werden.
|
|
5. Treffer entduplizieren, sortieren, begrenzen.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Callable
|
|
|
|
from rapidfuzz import fuzz
|
|
|
|
from ..config import SearchConfig
|
|
from ..database.repository import Repository
|
|
from ..utils.logging import get_logger
|
|
from .context import SearchContext, Timeline, build_context, build_timeline
|
|
from .fuzzy import (
|
|
build_fts_query,
|
|
normalize_text,
|
|
score_window,
|
|
significant_overlap,
|
|
trigram_terms,
|
|
)
|
|
from .ranking import dedupe_overlapping, rank_hits
|
|
|
|
logger = get_logger("search.engine")
|
|
|
|
ProgressCallback = Callable[[int, int], None]
|
|
|
|
|
|
class SearchError(RuntimeError):
|
|
"""Fehler bei der Suche (z. B. leere Anfrage)."""
|
|
|
|
|
|
@dataclass
|
|
class SearchHit:
|
|
"""Ein einzelner Suchtreffer."""
|
|
|
|
audio_id: int
|
|
filename: str
|
|
path: str
|
|
score: float
|
|
start_time: float
|
|
end_time: float
|
|
start_token: int
|
|
end_token: int
|
|
context: SearchContext
|
|
language: str | None = None
|
|
model: str | None = None
|
|
source: str = "fts"
|
|
|
|
@property
|
|
def matched_text(self) -> str:
|
|
return self.context.match
|
|
|
|
def to_dict(self, context_words: int = 50) -> dict[str, object]:
|
|
return {
|
|
"audio_id": self.audio_id,
|
|
"filename": self.filename,
|
|
"path": self.path,
|
|
"similarity": round(self.score, 2),
|
|
"timestamp": self.start_time,
|
|
"timestamp_end": self.end_time,
|
|
"timestamp_precision": self.context.precision,
|
|
"match": self.context.match,
|
|
"context_before": self.context.before,
|
|
"context_after": self.context.after,
|
|
"context_words": context_words,
|
|
"language": self.language,
|
|
"model": self.model,
|
|
"source": self.source,
|
|
}
|
|
|
|
|
|
class SearchEngine:
|
|
"""Kombinierte Suche aus FTS5, Trigrammen und Fuzzy-Sliding-Window."""
|
|
|
|
def __init__(self, repository: Repository, config: SearchConfig):
|
|
self.repo = repository
|
|
self.config = config
|
|
|
|
# ------------------------------------------------------------------
|
|
def search(
|
|
self,
|
|
query: str,
|
|
*,
|
|
limit: int | None = None,
|
|
min_score: float | None = None,
|
|
context_words: int | None = None,
|
|
language: str | None = None,
|
|
progress: ProgressCallback | None = None,
|
|
) -> list[SearchHit]:
|
|
query_normalized = normalize_text(query)
|
|
if not query_normalized:
|
|
raise SearchError("Die Suchanfrage enthaelt keine verwertbaren Woerter.")
|
|
query_tokens = query_normalized.split()
|
|
|
|
limit = self.config.default_limit if limit is None else int(limit)
|
|
min_score = self.config.default_min_score if min_score is None else float(min_score)
|
|
context_words = self.config.context_words if context_words is None else int(context_words)
|
|
if limit <= 0:
|
|
raise SearchError("--limit muss groesser als 0 sein.")
|
|
|
|
candidate_audio, candidate_source = self._collect_candidates(query_tokens)
|
|
hits: list[SearchHit] = []
|
|
swept: set[int] = set()
|
|
|
|
total_candidates = len(candidate_audio)
|
|
for index, audio_id in enumerate(candidate_audio, start=1):
|
|
swept.add(audio_id)
|
|
hits.extend(
|
|
self._scan_audio(
|
|
audio_id,
|
|
query_normalized,
|
|
query_tokens,
|
|
min_score=min_score,
|
|
context_words=context_words,
|
|
language=language,
|
|
source="fts",
|
|
)
|
|
)
|
|
if progress:
|
|
progress(index, total_candidates)
|
|
|
|
# Wichtig: FTS5 ist nur eine Vorauswahl. Wenn wenige Treffer gefunden
|
|
# wurden, werden alle uebrigen Dateien vollstaendig durchsucht.
|
|
fallback_used = False
|
|
if self.config.exhaustive_fallback and len(hits) < limit:
|
|
remaining = [
|
|
audio_id
|
|
for audio_id in self.repo.iter_audio_ids_with_transcripts(language)
|
|
if audio_id not in swept
|
|
]
|
|
fallback_used = bool(remaining)
|
|
for index, audio_id in enumerate(remaining, start=1):
|
|
hits.extend(
|
|
self._scan_audio(
|
|
audio_id,
|
|
query_normalized,
|
|
query_tokens,
|
|
min_score=min_score,
|
|
context_words=context_words,
|
|
language=language,
|
|
source="fallback",
|
|
)
|
|
)
|
|
if progress:
|
|
progress(index, len(remaining))
|
|
|
|
if fallback_used:
|
|
logger.info(
|
|
"Fallback-Suche aktiv: %d Dateien ohne FTS-Treffer durchsucht", len(remaining)
|
|
)
|
|
|
|
deduped = dedupe_overlapping(hits)
|
|
return rank_hits(deduped, limit)
|
|
|
|
# ------------------------------------------------------------------
|
|
def _collect_candidates(self, query_tokens: list[str]) -> tuple[list[int], str]:
|
|
"""Sammelt Kandidaten-Audiodateien (FTS5 + Trigramme)."""
|
|
limit = max(1, int(self.config.candidate_limit))
|
|
matches: list[tuple[int, int]] = []
|
|
|
|
fts_query = build_fts_query(query_tokens)
|
|
if fts_query:
|
|
matches.extend(self.repo.fts_candidates(fts_query, limit))
|
|
|
|
remaining = limit - len(matches)
|
|
if remaining > 0:
|
|
terms = trigram_terms(query_tokens)
|
|
if terms:
|
|
matches.extend(self.repo.trigram_candidates(terms, remaining))
|
|
|
|
counts: dict[int, int] = {}
|
|
for audio_id, _segment_id in matches:
|
|
counts[audio_id] = counts.get(audio_id, 0) + 1
|
|
|
|
ordered = sorted(counts, key=lambda audio_id: (-counts[audio_id], audio_id))
|
|
return ordered, "fts"
|
|
|
|
# ------------------------------------------------------------------
|
|
def _scan_audio(
|
|
self,
|
|
audio_id: int,
|
|
query_normalized: str,
|
|
query_tokens: list[str],
|
|
*,
|
|
min_score: float,
|
|
context_words: int,
|
|
language: str | None,
|
|
source: str,
|
|
) -> list[SearchHit]:
|
|
record = self.repo.get_audio(audio_id)
|
|
if record is None or record.duplicate_of is not None:
|
|
return []
|
|
|
|
transcript = self.repo.get_transcript(audio_id)
|
|
if transcript is None:
|
|
return []
|
|
if language and (transcript["language"] or "").lower() != language.lower():
|
|
return []
|
|
|
|
segments, words = self.repo.timeline(audio_id)
|
|
if not segments:
|
|
return []
|
|
timeline = build_timeline(
|
|
audio_id,
|
|
segments,
|
|
words,
|
|
language=transcript["language"],
|
|
model=transcript["model"],
|
|
)
|
|
if not timeline.tokens:
|
|
return []
|
|
|
|
windows = self._find_windows(timeline, query_normalized, query_tokens, min_score)
|
|
if not windows:
|
|
return []
|
|
|
|
hits: list[SearchHit] = []
|
|
for score, start_index, end_index in windows:
|
|
context = build_context(timeline, start_index, end_index, context_words)
|
|
hits.append(
|
|
SearchHit(
|
|
audio_id=audio_id,
|
|
filename=record.filename,
|
|
path=record.path,
|
|
score=score,
|
|
start_time=context.start_time,
|
|
end_time=context.end_time,
|
|
start_token=start_index,
|
|
end_token=end_index,
|
|
context=context,
|
|
language=transcript["language"],
|
|
model=transcript["model"],
|
|
source=source,
|
|
)
|
|
)
|
|
|
|
hits.sort(key=lambda hit: -hit.score)
|
|
return hits[: max(1, int(self.config.max_hits_per_file))]
|
|
|
|
# ------------------------------------------------------------------
|
|
def _find_windows(
|
|
self,
|
|
timeline: Timeline,
|
|
query_normalized: str,
|
|
query_tokens: list[str],
|
|
min_score: float,
|
|
) -> list[tuple[float, int, int]]:
|
|
"""Sliding-Window-Suche ueber die Wort-Timeline einer Datei."""
|
|
norm_tokens = timeline.normalized_tokens
|
|
token_count = len(norm_tokens)
|
|
query_len = len(query_tokens)
|
|
if token_count == 0:
|
|
return []
|
|
|
|
min_size = max(1, query_len - max(0, self.config.window_before))
|
|
max_size = max(min_size, query_len + max(0, self.config.window_after))
|
|
overlap_threshold = max(0.0, min(1.0, self.config.min_token_overlap))
|
|
|
|
# Dateiebene: gibt es ueberhaupt Beruehrungspunkte? Das ist deutlich
|
|
# billiger als jede Datei Fenster fuer Fenster zu vergleichen.
|
|
file_overlap = significant_overlap(query_tokens, norm_tokens)
|
|
if file_overlap < overlap_threshold * 0.5:
|
|
partial = fuzz.partial_ratio(query_normalized, timeline.normalized_text)
|
|
if partial < 55.0:
|
|
return []
|
|
|
|
windows: list[tuple[float, int, int]] = []
|
|
# Wenn auf Dateiebene kaum Wortueberschneidung besteht, kann der Treffer
|
|
# nur ueber einen falsch erkannten Ausdruck zustande kommen. Dann wird
|
|
# ohne Wort-Vorfilter verglichen (selten und durch die Dateipruefung
|
|
# oben bereits eingegrenzt).
|
|
use_overlap_filter = file_overlap >= overlap_threshold
|
|
for start in range(token_count):
|
|
pre_end = min(token_count, start + query_len + 2)
|
|
if use_overlap_filter and significant_overlap(query_tokens, norm_tokens[start:pre_end]) < overlap_threshold:
|
|
continue
|
|
|
|
best_score = 0.0
|
|
best_size = 0
|
|
for size in range(min_size, max_size + 1):
|
|
end = start + size
|
|
if end > token_count:
|
|
break
|
|
window_normalized = " ".join(norm_tokens[start:end])
|
|
candidate = score_window(query_normalized, window_normalized)
|
|
if candidate.score > best_score:
|
|
best_score = candidate.score
|
|
best_size = size
|
|
|
|
if best_size and best_score >= min_score:
|
|
windows.append((best_score, start, start + best_size))
|
|
|
|
return windows
|