Initial commit
This commit is contained in:
@@ -0,0 +1,303 @@
|
||||
"""Suchmaschine fuer SjAudioSearch.
|
||||
|
||||
Ablauf einer Suche:
|
||||
|
||||
1. Anfrage normalisieren.
|
||||
2. Kandidaten ueber FTS5 und den Trigramm-Index sammeln (schnelle Vorauswahl).
|
||||
3. Sliding-Window-Vergleich mit RapidFuzz auf der Wort-Timeline der Kandidaten.
|
||||
4. Optionaler, vollstaendiger Fallback-Lauf ueber alle verbleibenden Dateien,
|
||||
damit auch komplett falsch erkannte Woerter gefunden werden.
|
||||
5. Treffer entduplizieren, sortieren, begrenzen.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
|
||||
from rapidfuzz import fuzz
|
||||
|
||||
from ..config import SearchConfig
|
||||
from ..database.repository import Repository
|
||||
from ..utils.logging import get_logger
|
||||
from .context import SearchContext, Timeline, build_context, build_timeline
|
||||
from .fuzzy import (
|
||||
build_fts_query,
|
||||
normalize_text,
|
||||
score_window,
|
||||
significant_overlap,
|
||||
trigram_terms,
|
||||
)
|
||||
from .ranking import dedupe_overlapping, rank_hits
|
||||
|
||||
logger = get_logger("search.engine")
|
||||
|
||||
ProgressCallback = Callable[[int, int], None]
|
||||
|
||||
|
||||
class SearchError(RuntimeError):
|
||||
"""Fehler bei der Suche (z. B. leere Anfrage)."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchHit:
|
||||
"""Ein einzelner Suchtreffer."""
|
||||
|
||||
audio_id: int
|
||||
filename: str
|
||||
path: str
|
||||
score: float
|
||||
start_time: float
|
||||
end_time: float
|
||||
start_token: int
|
||||
end_token: int
|
||||
context: SearchContext
|
||||
language: str | None = None
|
||||
model: str | None = None
|
||||
source: str = "fts"
|
||||
|
||||
@property
|
||||
def matched_text(self) -> str:
|
||||
return self.context.match
|
||||
|
||||
def to_dict(self, context_words: int = 50) -> dict[str, object]:
|
||||
return {
|
||||
"audio_id": self.audio_id,
|
||||
"filename": self.filename,
|
||||
"path": self.path,
|
||||
"similarity": round(self.score, 2),
|
||||
"timestamp": self.start_time,
|
||||
"timestamp_end": self.end_time,
|
||||
"timestamp_precision": self.context.precision,
|
||||
"match": self.context.match,
|
||||
"context_before": self.context.before,
|
||||
"context_after": self.context.after,
|
||||
"context_words": context_words,
|
||||
"language": self.language,
|
||||
"model": self.model,
|
||||
"source": self.source,
|
||||
}
|
||||
|
||||
|
||||
class SearchEngine:
|
||||
"""Kombinierte Suche aus FTS5, Trigrammen und Fuzzy-Sliding-Window."""
|
||||
|
||||
def __init__(self, repository: Repository, config: SearchConfig):
|
||||
self.repo = repository
|
||||
self.config = config
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def search(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
limit: int | None = None,
|
||||
min_score: float | None = None,
|
||||
context_words: int | None = None,
|
||||
language: str | None = None,
|
||||
progress: ProgressCallback | None = None,
|
||||
) -> list[SearchHit]:
|
||||
query_normalized = normalize_text(query)
|
||||
if not query_normalized:
|
||||
raise SearchError("Die Suchanfrage enthaelt keine verwertbaren Woerter.")
|
||||
query_tokens = query_normalized.split()
|
||||
|
||||
limit = self.config.default_limit if limit is None else int(limit)
|
||||
min_score = self.config.default_min_score if min_score is None else float(min_score)
|
||||
context_words = self.config.context_words if context_words is None else int(context_words)
|
||||
if limit <= 0:
|
||||
raise SearchError("--limit muss groesser als 0 sein.")
|
||||
|
||||
candidate_audio, candidate_source = self._collect_candidates(query_tokens)
|
||||
hits: list[SearchHit] = []
|
||||
swept: set[int] = set()
|
||||
|
||||
total_candidates = len(candidate_audio)
|
||||
for index, audio_id in enumerate(candidate_audio, start=1):
|
||||
swept.add(audio_id)
|
||||
hits.extend(
|
||||
self._scan_audio(
|
||||
audio_id,
|
||||
query_normalized,
|
||||
query_tokens,
|
||||
min_score=min_score,
|
||||
context_words=context_words,
|
||||
language=language,
|
||||
source="fts",
|
||||
)
|
||||
)
|
||||
if progress:
|
||||
progress(index, total_candidates)
|
||||
|
||||
# Wichtig: FTS5 ist nur eine Vorauswahl. Wenn wenige Treffer gefunden
|
||||
# wurden, werden alle uebrigen Dateien vollstaendig durchsucht.
|
||||
fallback_used = False
|
||||
if self.config.exhaustive_fallback and len(hits) < limit:
|
||||
remaining = [
|
||||
audio_id
|
||||
for audio_id in self.repo.iter_audio_ids_with_transcripts(language)
|
||||
if audio_id not in swept
|
||||
]
|
||||
fallback_used = bool(remaining)
|
||||
for index, audio_id in enumerate(remaining, start=1):
|
||||
hits.extend(
|
||||
self._scan_audio(
|
||||
audio_id,
|
||||
query_normalized,
|
||||
query_tokens,
|
||||
min_score=min_score,
|
||||
context_words=context_words,
|
||||
language=language,
|
||||
source="fallback",
|
||||
)
|
||||
)
|
||||
if progress:
|
||||
progress(index, len(remaining))
|
||||
|
||||
if fallback_used:
|
||||
logger.info(
|
||||
"Fallback-Suche aktiv: %d Dateien ohne FTS-Treffer durchsucht", len(remaining)
|
||||
)
|
||||
|
||||
deduped = dedupe_overlapping(hits)
|
||||
return rank_hits(deduped, limit)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _collect_candidates(self, query_tokens: list[str]) -> tuple[list[int], str]:
|
||||
"""Sammelt Kandidaten-Audiodateien (FTS5 + Trigramme)."""
|
||||
limit = max(1, int(self.config.candidate_limit))
|
||||
matches: list[tuple[int, int]] = []
|
||||
|
||||
fts_query = build_fts_query(query_tokens)
|
||||
if fts_query:
|
||||
matches.extend(self.repo.fts_candidates(fts_query, limit))
|
||||
|
||||
remaining = limit - len(matches)
|
||||
if remaining > 0:
|
||||
terms = trigram_terms(query_tokens)
|
||||
if terms:
|
||||
matches.extend(self.repo.trigram_candidates(terms, remaining))
|
||||
|
||||
counts: dict[int, int] = {}
|
||||
for audio_id, _segment_id in matches:
|
||||
counts[audio_id] = counts.get(audio_id, 0) + 1
|
||||
|
||||
ordered = sorted(counts, key=lambda audio_id: (-counts[audio_id], audio_id))
|
||||
return ordered, "fts"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _scan_audio(
|
||||
self,
|
||||
audio_id: int,
|
||||
query_normalized: str,
|
||||
query_tokens: list[str],
|
||||
*,
|
||||
min_score: float,
|
||||
context_words: int,
|
||||
language: str | None,
|
||||
source: str,
|
||||
) -> list[SearchHit]:
|
||||
record = self.repo.get_audio(audio_id)
|
||||
if record is None or record.duplicate_of is not None:
|
||||
return []
|
||||
|
||||
transcript = self.repo.get_transcript(audio_id)
|
||||
if transcript is None:
|
||||
return []
|
||||
if language and (transcript["language"] or "").lower() != language.lower():
|
||||
return []
|
||||
|
||||
segments, words = self.repo.timeline(audio_id)
|
||||
if not segments:
|
||||
return []
|
||||
timeline = build_timeline(
|
||||
audio_id,
|
||||
segments,
|
||||
words,
|
||||
language=transcript["language"],
|
||||
model=transcript["model"],
|
||||
)
|
||||
if not timeline.tokens:
|
||||
return []
|
||||
|
||||
windows = self._find_windows(timeline, query_normalized, query_tokens, min_score)
|
||||
if not windows:
|
||||
return []
|
||||
|
||||
hits: list[SearchHit] = []
|
||||
for score, start_index, end_index in windows:
|
||||
context = build_context(timeline, start_index, end_index, context_words)
|
||||
hits.append(
|
||||
SearchHit(
|
||||
audio_id=audio_id,
|
||||
filename=record.filename,
|
||||
path=record.path,
|
||||
score=score,
|
||||
start_time=context.start_time,
|
||||
end_time=context.end_time,
|
||||
start_token=start_index,
|
||||
end_token=end_index,
|
||||
context=context,
|
||||
language=transcript["language"],
|
||||
model=transcript["model"],
|
||||
source=source,
|
||||
)
|
||||
)
|
||||
|
||||
hits.sort(key=lambda hit: -hit.score)
|
||||
return hits[: max(1, int(self.config.max_hits_per_file))]
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _find_windows(
|
||||
self,
|
||||
timeline: Timeline,
|
||||
query_normalized: str,
|
||||
query_tokens: list[str],
|
||||
min_score: float,
|
||||
) -> list[tuple[float, int, int]]:
|
||||
"""Sliding-Window-Suche ueber die Wort-Timeline einer Datei."""
|
||||
norm_tokens = timeline.normalized_tokens
|
||||
token_count = len(norm_tokens)
|
||||
query_len = len(query_tokens)
|
||||
if token_count == 0:
|
||||
return []
|
||||
|
||||
min_size = max(1, query_len - max(0, self.config.window_before))
|
||||
max_size = max(min_size, query_len + max(0, self.config.window_after))
|
||||
overlap_threshold = max(0.0, min(1.0, self.config.min_token_overlap))
|
||||
|
||||
# Dateiebene: gibt es ueberhaupt Beruehrungspunkte? Das ist deutlich
|
||||
# billiger als jede Datei Fenster fuer Fenster zu vergleichen.
|
||||
file_overlap = significant_overlap(query_tokens, norm_tokens)
|
||||
if file_overlap < overlap_threshold * 0.5:
|
||||
partial = fuzz.partial_ratio(query_normalized, timeline.normalized_text)
|
||||
if partial < 55.0:
|
||||
return []
|
||||
|
||||
windows: list[tuple[float, int, int]] = []
|
||||
# Wenn auf Dateiebene kaum Wortueberschneidung besteht, kann der Treffer
|
||||
# nur ueber einen falsch erkannten Ausdruck zustande kommen. Dann wird
|
||||
# ohne Wort-Vorfilter verglichen (selten und durch die Dateipruefung
|
||||
# oben bereits eingegrenzt).
|
||||
use_overlap_filter = file_overlap >= overlap_threshold
|
||||
for start in range(token_count):
|
||||
pre_end = min(token_count, start + query_len + 2)
|
||||
if use_overlap_filter and significant_overlap(query_tokens, norm_tokens[start:pre_end]) < overlap_threshold:
|
||||
continue
|
||||
|
||||
best_score = 0.0
|
||||
best_size = 0
|
||||
for size in range(min_size, max_size + 1):
|
||||
end = start + size
|
||||
if end > token_count:
|
||||
break
|
||||
window_normalized = " ".join(norm_tokens[start:end])
|
||||
candidate = score_window(query_normalized, window_normalized)
|
||||
if candidate.score > best_score:
|
||||
best_score = candidate.score
|
||||
best_size = size
|
||||
|
||||
if best_size and best_score >= min_score:
|
||||
windows.append((best_score, start, start + best_size))
|
||||
|
||||
return windows
|
||||
Reference in New Issue
Block a user