Initial commit

This commit is contained in:
2026-11-16 16:29:32 +01:00
commit 97e5c9ef4d
37 changed files with 5932 additions and 0 deletions
+186
View File
@@ -0,0 +1,186 @@
"""Kontextaufbau fuer Suchtreffer.
Aus den gespeicherten Segmenten wird eine durchgehende Wort-Timeline gebaut.
Dadurch gehen an Segmentgrenzen keine Informationen verloren, und der Kontext
kann mitten in einem Segment beginnen und enden.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Sequence
from .fuzzy import normalize_text, tokenize_original
@dataclass
class TokenRef:
"""Ein Wort im Transkript mit Zeitbezug und Herkunft."""
text: str
norm: str
start: float
end: float
segment_id: int
precision: str # "word" oder "segment"
@dataclass
class Timeline:
"""Durchgehende Wortfolge einer Audiodatei."""
audio_id: int
tokens: list[TokenRef] = field(default_factory=list)
language: str | None = None
model: str | None = None
@property
def normalized_tokens(self) -> list[str]:
return [token.norm for token in self.tokens]
@property
def normalized_text(self) -> str:
return " ".join(token.norm for token in self.tokens)
@property
def original_text(self) -> str:
return " ".join(token.text for token in self.tokens)
def __len__(self) -> int:
return len(self.tokens)
def _distribute_times(
segment_start: float, segment_end: float, token_count: int
) -> list[tuple[float, float]]:
"""Verteilt die Segmentzeit gleichmaessig auf die Woerter (Schaetzung)."""
if token_count <= 0:
return []
span = max(0.0, float(segment_end) - float(segment_start))
step = span / token_count if span else 0.0
times: list[tuple[float, float]] = []
for index in range(token_count):
start = float(segment_start) + step * index
end = float(segment_start) + step * (index + 1)
times.append((round(start, 3), round(end, 3)))
return times
def build_timeline(
audio_id: int,
segments: Sequence[Any],
words_by_segment: dict[int, Sequence[Any]] | None = None,
*,
language: str | None = None,
model: str | None = None,
) -> Timeline:
"""Baut die Timeline aus Segmenten (sqlite3.Row oder Objekte mit Attributen)."""
timeline = Timeline(audio_id=audio_id, language=language, model=model)
words_by_segment = words_by_segment or {}
for segment in segments:
if isinstance(segment, dict):
segment_id = int(segment["id"])
start = float(segment["start_time"])
end = float(segment["end_time"])
text = segment["text"]
else:
segment_id = int(segment["id"])
start = float(segment["start_time"])
end = float(segment["end_time"])
text = segment["text"]
original_tokens = tokenize_original(text)
if not original_tokens:
continue
words = list(words_by_segment.get(segment_id, []))
use_words = len(words) == len(original_tokens)
if use_words:
for token, word in zip(original_tokens, words):
word_start = word["start_time"] if not isinstance(word, dict) else word.get("start_time")
word_end = word["end_time"] if not isinstance(word, dict) else word.get("end_time")
if word_start is None or word_end is None:
use_words = False
break
if use_words:
for token, word in zip(original_tokens, words):
word_start = word["start_time"] if not isinstance(word, dict) else word["start_time"]
word_end = word["end_time"] if not isinstance(word, dict) else word["end_time"]
timeline.tokens.append(
TokenRef(
text=token,
norm=normalize_text(token),
start=float(word_start),
end=float(word_end),
segment_id=segment_id,
precision="word",
)
)
else:
times = _distribute_times(start, end, len(original_tokens))
for token, (token_start, token_end) in zip(original_tokens, times):
timeline.tokens.append(
TokenRef(
text=token,
norm=normalize_text(token),
start=token_start,
end=token_end,
segment_id=segment_id,
precision="segment",
)
)
return timeline
@dataclass
class SearchContext:
"""Die drei Kontextabschnitte eines Treffers."""
before: str
match: str
after: str
start_time: float
end_time: float
precision: str
def as_block(self, quote: str = '"') -> str:
"""Formatiert den Kontext als zusammenhaengenden Textblock."""
before = f"\u2026{self.before} " if self.before else ""
after = f" {self.after}\u2026" if self.after else ""
return f"{quote}{before}{self.match}{after}{quote}"
def build_context(
timeline: Timeline, start_index: int, end_index: int, context_words: int = 50
) -> SearchContext:
"""Schneidet Kontext um den Treffer ``[start_index, end_index)`` aus."""
tokens = timeline.tokens
if not tokens:
return SearchContext("", "", "", 0.0, 0.0, "segment")
start_index = max(0, min(start_index, len(tokens)))
end_index = max(start_index + 1, min(end_index, len(tokens)))
before_start = max(0, start_index - max(0, context_words))
after_end = min(len(tokens), end_index + max(0, context_words))
before_tokens = tokens[before_start:start_index]
match_tokens = tokens[start_index:end_index]
after_tokens = tokens[end_index:after_end]
precision = "word" if all(token.precision == "word" for token in match_tokens) else "segment"
start_time = match_tokens[0].start
end_time = match_tokens[-1].end
return SearchContext(
before=" ".join(token.text for token in before_tokens),
match=" ".join(token.text for token in match_tokens),
after=" ".join(token.text for token in after_tokens),
start_time=start_time,
end_time=end_time,
precision=precision,
)
+303
View File
@@ -0,0 +1,303 @@
"""Suchmaschine fuer SjAudioSearch.
Ablauf einer Suche:
1. Anfrage normalisieren.
2. Kandidaten ueber FTS5 und den Trigramm-Index sammeln (schnelle Vorauswahl).
3. Sliding-Window-Vergleich mit RapidFuzz auf der Wort-Timeline der Kandidaten.
4. Optionaler, vollstaendiger Fallback-Lauf ueber alle verbleibenden Dateien,
damit auch komplett falsch erkannte Woerter gefunden werden.
5. Treffer entduplizieren, sortieren, begrenzen.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Callable
from rapidfuzz import fuzz
from ..config import SearchConfig
from ..database.repository import Repository
from ..utils.logging import get_logger
from .context import SearchContext, Timeline, build_context, build_timeline
from .fuzzy import (
build_fts_query,
normalize_text,
score_window,
significant_overlap,
trigram_terms,
)
from .ranking import dedupe_overlapping, rank_hits
logger = get_logger("search.engine")
ProgressCallback = Callable[[int, int], None]
class SearchError(RuntimeError):
"""Fehler bei der Suche (z. B. leere Anfrage)."""
@dataclass
class SearchHit:
"""Ein einzelner Suchtreffer."""
audio_id: int
filename: str
path: str
score: float
start_time: float
end_time: float
start_token: int
end_token: int
context: SearchContext
language: str | None = None
model: str | None = None
source: str = "fts"
@property
def matched_text(self) -> str:
return self.context.match
def to_dict(self, context_words: int = 50) -> dict[str, object]:
return {
"audio_id": self.audio_id,
"filename": self.filename,
"path": self.path,
"similarity": round(self.score, 2),
"timestamp": self.start_time,
"timestamp_end": self.end_time,
"timestamp_precision": self.context.precision,
"match": self.context.match,
"context_before": self.context.before,
"context_after": self.context.after,
"context_words": context_words,
"language": self.language,
"model": self.model,
"source": self.source,
}
class SearchEngine:
"""Kombinierte Suche aus FTS5, Trigrammen und Fuzzy-Sliding-Window."""
def __init__(self, repository: Repository, config: SearchConfig):
self.repo = repository
self.config = config
# ------------------------------------------------------------------
def search(
self,
query: str,
*,
limit: int | None = None,
min_score: float | None = None,
context_words: int | None = None,
language: str | None = None,
progress: ProgressCallback | None = None,
) -> list[SearchHit]:
query_normalized = normalize_text(query)
if not query_normalized:
raise SearchError("Die Suchanfrage enthaelt keine verwertbaren Woerter.")
query_tokens = query_normalized.split()
limit = self.config.default_limit if limit is None else int(limit)
min_score = self.config.default_min_score if min_score is None else float(min_score)
context_words = self.config.context_words if context_words is None else int(context_words)
if limit <= 0:
raise SearchError("--limit muss groesser als 0 sein.")
candidate_audio, candidate_source = self._collect_candidates(query_tokens)
hits: list[SearchHit] = []
swept: set[int] = set()
total_candidates = len(candidate_audio)
for index, audio_id in enumerate(candidate_audio, start=1):
swept.add(audio_id)
hits.extend(
self._scan_audio(
audio_id,
query_normalized,
query_tokens,
min_score=min_score,
context_words=context_words,
language=language,
source="fts",
)
)
if progress:
progress(index, total_candidates)
# Wichtig: FTS5 ist nur eine Vorauswahl. Wenn wenige Treffer gefunden
# wurden, werden alle uebrigen Dateien vollstaendig durchsucht.
fallback_used = False
if self.config.exhaustive_fallback and len(hits) < limit:
remaining = [
audio_id
for audio_id in self.repo.iter_audio_ids_with_transcripts(language)
if audio_id not in swept
]
fallback_used = bool(remaining)
for index, audio_id in enumerate(remaining, start=1):
hits.extend(
self._scan_audio(
audio_id,
query_normalized,
query_tokens,
min_score=min_score,
context_words=context_words,
language=language,
source="fallback",
)
)
if progress:
progress(index, len(remaining))
if fallback_used:
logger.info(
"Fallback-Suche aktiv: %d Dateien ohne FTS-Treffer durchsucht", len(remaining)
)
deduped = dedupe_overlapping(hits)
return rank_hits(deduped, limit)
# ------------------------------------------------------------------
def _collect_candidates(self, query_tokens: list[str]) -> tuple[list[int], str]:
"""Sammelt Kandidaten-Audiodateien (FTS5 + Trigramme)."""
limit = max(1, int(self.config.candidate_limit))
matches: list[tuple[int, int]] = []
fts_query = build_fts_query(query_tokens)
if fts_query:
matches.extend(self.repo.fts_candidates(fts_query, limit))
remaining = limit - len(matches)
if remaining > 0:
terms = trigram_terms(query_tokens)
if terms:
matches.extend(self.repo.trigram_candidates(terms, remaining))
counts: dict[int, int] = {}
for audio_id, _segment_id in matches:
counts[audio_id] = counts.get(audio_id, 0) + 1
ordered = sorted(counts, key=lambda audio_id: (-counts[audio_id], audio_id))
return ordered, "fts"
# ------------------------------------------------------------------
def _scan_audio(
self,
audio_id: int,
query_normalized: str,
query_tokens: list[str],
*,
min_score: float,
context_words: int,
language: str | None,
source: str,
) -> list[SearchHit]:
record = self.repo.get_audio(audio_id)
if record is None or record.duplicate_of is not None:
return []
transcript = self.repo.get_transcript(audio_id)
if transcript is None:
return []
if language and (transcript["language"] or "").lower() != language.lower():
return []
segments, words = self.repo.timeline(audio_id)
if not segments:
return []
timeline = build_timeline(
audio_id,
segments,
words,
language=transcript["language"],
model=transcript["model"],
)
if not timeline.tokens:
return []
windows = self._find_windows(timeline, query_normalized, query_tokens, min_score)
if not windows:
return []
hits: list[SearchHit] = []
for score, start_index, end_index in windows:
context = build_context(timeline, start_index, end_index, context_words)
hits.append(
SearchHit(
audio_id=audio_id,
filename=record.filename,
path=record.path,
score=score,
start_time=context.start_time,
end_time=context.end_time,
start_token=start_index,
end_token=end_index,
context=context,
language=transcript["language"],
model=transcript["model"],
source=source,
)
)
hits.sort(key=lambda hit: -hit.score)
return hits[: max(1, int(self.config.max_hits_per_file))]
# ------------------------------------------------------------------
def _find_windows(
self,
timeline: Timeline,
query_normalized: str,
query_tokens: list[str],
min_score: float,
) -> list[tuple[float, int, int]]:
"""Sliding-Window-Suche ueber die Wort-Timeline einer Datei."""
norm_tokens = timeline.normalized_tokens
token_count = len(norm_tokens)
query_len = len(query_tokens)
if token_count == 0:
return []
min_size = max(1, query_len - max(0, self.config.window_before))
max_size = max(min_size, query_len + max(0, self.config.window_after))
overlap_threshold = max(0.0, min(1.0, self.config.min_token_overlap))
# Dateiebene: gibt es ueberhaupt Beruehrungspunkte? Das ist deutlich
# billiger als jede Datei Fenster fuer Fenster zu vergleichen.
file_overlap = significant_overlap(query_tokens, norm_tokens)
if file_overlap < overlap_threshold * 0.5:
partial = fuzz.partial_ratio(query_normalized, timeline.normalized_text)
if partial < 55.0:
return []
windows: list[tuple[float, int, int]] = []
# Wenn auf Dateiebene kaum Wortueberschneidung besteht, kann der Treffer
# nur ueber einen falsch erkannten Ausdruck zustande kommen. Dann wird
# ohne Wort-Vorfilter verglichen (selten und durch die Dateipruefung
# oben bereits eingegrenzt).
use_overlap_filter = file_overlap >= overlap_threshold
for start in range(token_count):
pre_end = min(token_count, start + query_len + 2)
if use_overlap_filter and significant_overlap(query_tokens, norm_tokens[start:pre_end]) < overlap_threshold:
continue
best_score = 0.0
best_size = 0
for size in range(min_size, max_size + 1):
end = start + size
if end > token_count:
break
window_normalized = " ".join(norm_tokens[start:end])
candidate = score_window(query_normalized, window_normalized)
if candidate.score > best_score:
best_score = candidate.score
best_size = size
if best_size and best_score >= min_score:
windows.append((best_score, start, start + best_size))
return windows
+178
View File
@@ -0,0 +1,178 @@
"""Textnormalisierung und Fuzzy-Vergleich (RapidFuzz).
Wichtig: Die Normalisierung wird ausschliesslich fuer Vergleiche und fuer den
Suchindex verwendet. Der gespeicherte Originaltext bleibt unveraendert.
"""
from __future__ import annotations
import re
import unicodedata
from dataclasses import dataclass
from rapidfuzz import fuzz
# Satzzeichen und Symbole, die beim Vergleich stoeren. Wortzeichen (Buchstaben,
# Ziffern, Unterstrich) bleiben erhalten.
_PUNCT_RE = re.compile(r"[^\w\s]", re.UNICODE)
_WHITESPACE_RE = re.compile(r"\s+", re.UNICODE)
# Sehr haeufige deutsche Fuellwoerter. Sie werden beim Vorfilter nicht als
# belastbare Uebereinstimmung gewertet, damit "der/die/das" allein keine
# Kandidaten erzeugt.
STOPWORDS = frozenset(
{
"der", "die", "das", "den", "dem", "des", "ein", "eine", "einer", "eines",
"einem", "einen", "und", "oder", "aber", "als", "auch", "auf", "aus", "bei",
"bin", "bis", "da", "dann", "dass", "doch", "dort", "du", "durch", "er",
"es", "hat", "ich", "ihm", "ihn", "ihr", "im", "in", "ist", "mit", "nach",
"nicht", "noch", "nun", "nur", "ob", "so", "sich", "sie", "sind", "um",
"und", "vom", "von", "vor", "war", "was", "wenn", "wie", "wir", "wo", "zu",
"zum", "zur", "the", "and", "of", "to", "in", "a", "is",
}
)
def normalize_text(text: str) -> str:
"""Normalisiert Text fuer Vergleiche: NFKC, Kleinschreibung, ohne Satzzeichen."""
if not text:
return ""
normalized = unicodedata.normalize("NFKC", text).casefold()
normalized = _PUNCT_RE.sub(" ", normalized)
return _WHITESPACE_RE.sub(" ", normalized).strip()
def tokenize(text: str) -> list[str]:
"""Normalisierte Wortliste eines Textes."""
normalized = normalize_text(text)
return normalized.split() if normalized else []
def tokenize_original(text: str) -> list[str]:
"""Wortliste mit Originaltext inklusive anhängender Satzzeichen (fuer die Ausgabe)."""
if not text:
return []
return [token for token in _WHITESPACE_RE.split(text.strip()) if token]
def token_norm(token: str) -> str:
"""Normalisiert ein einzelnes Originalwort."""
return normalize_text(token)
def is_stopword(token: str) -> bool:
return token in STOPWORDS
@dataclass(frozen=True)
class WindowScore:
"""Ergebnis eines Fenstervergleichs."""
score: float
exact: bool
def score_window(query_normalized: str, window_normalized: str) -> WindowScore:
"""Vergleicht Suchanfrage und Textfenster und liefert einen Wert 0-100.
Es werden mehrere RapidFuzz-Kennzahlen kombiniert, damit fehlende,
zusaetzliche, vertauschte oder falsch erkannte Woerter nicht sofort zum
Verlust des Treffers fuehren.
"""
if not query_normalized or not window_normalized:
return WindowScore(0.0, False)
if query_normalized == window_normalized:
return WindowScore(100.0, True)
# Hinweis: Eine reine Teilzeichenketten-Pruefung ist hier absichtlich NICHT
# enthalten. Sie wuerde z. B. das Fenster "Leider" fuer die Anfrage
# "Leider zu spaet" mit 100 bewerten und den Treffer auf ein Wort
# zusammenfallen lassen. Teiluebereinstimmungen werden stattdessen ueber
# die gewichteten RapidFuzz-Kennzahlen unten bewertet.
ratio = fuzz.ratio(query_normalized, window_normalized)
token_set = fuzz.token_set_ratio(query_normalized, window_normalized)
weighted = fuzz.WRatio(query_normalized, window_normalized)
partial = fuzz.partial_ratio(query_normalized, window_normalized)
# partial_ratio neigt bei kurzen Fenstern zu hohen Werten, daher nur
# gedämpft einbeziehen.
# token_set_ratio/WRatio/partial_ratio bewerten eine Teilmenge der Anfrage
# mit 100. Ohne Daempfung wuerde ein einzelnes Wort einer mehrwoertrigen
# Anfrage als perfekter Treffer gelten. Daher werden diese Kennzahlen mit
# dem Laengenverhaeltnis gewichtet, wenn das Fenster kuerzer ist.
length_factor = min(1.0, len(window_normalized) / max(1, len(query_normalized)))
score = max(
ratio,
token_set * length_factor,
weighted * length_factor,
partial * 0.92 * length_factor,
)
return WindowScore(min(100.0, round(score, 2)), False)
def token_overlap(query_tokens: list[str], window_tokens: list[str]) -> float:
"""Anteil der Anfragewoerter, die im Fenster vorkommen (0..1)."""
if not query_tokens:
return 0.0
window_set = set(window_tokens)
hits = sum(1 for token in query_tokens if token in window_set)
return hits / len(query_tokens)
def significant_overlap(query_tokens: list[str], window_tokens: list[str]) -> float:
"""Wie :func:`token_overlap`, aber Fuellwoerter zaehlen nur halb."""
if not query_tokens:
return 0.0
window_set = set(window_tokens)
total = 0.0
hits = 0.0
for token in query_tokens:
weight = 0.5 if is_stopword(token) else 1.0
total += weight
if token in window_set:
hits += weight
if total == 0:
return 0.0
return hits / total
def _escape_fts_token(token: str) -> str:
return '"' + token.replace('"', '""') + '"'
def build_fts_query(tokens: list[str], prefix_min_length: int = 4) -> str:
"""Baut eine FTS5-Anfrage aus ODER-verknuepften Woertern.
Praefixsuche fuer laengere Woerter faengt deutsche Flexionsformen und
Zusammensetzungen ab. Die Anfrage ist bewusst weit gefasst: sie dient nur
der Kandidatensuche, die Bewertung erfolgt spaeter per Fuzzy-Vergleich.
"""
parts: list[str] = []
seen: set[str] = set()
for token in tokens:
if len(token) < 2 or token in seen:
continue
seen.add(token)
escaped = _escape_fts_token(token)
if len(token) >= prefix_min_length:
parts.append(f"{escaped}*")
else:
parts.append(escaped)
return " OR ".join(parts)
def trigram_terms(tokens: list[str], max_terms: int = 32) -> list[str]:
"""Sinnvolle Trigramm-Anfragen aus laengeren Anfragewoertern."""
terms: list[str] = []
seen: set[str] = set()
for token in tokens:
if len(token) < 4 or token in seen:
continue
seen.add(token)
terms.append(token)
if len(terms) >= max_terms:
break
return terms
+53
View File
@@ -0,0 +1,53 @@
"""Bewertung und Sortierung von Suchtreffern.
Der Score ist eine Aehnlichkeitsbewertung (0-100) und ausdruecklich keine
statistische Wahrscheinlichkeit.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING: # pragma: no cover
from .engine import SearchHit
def clamp_score(score: float) -> float:
return max(0.0, min(100.0, float(score)))
def dedupe_overlapping(hits: list["SearchHit"], gap_tokens: int = 3) -> list["SearchHit"]:
"""Entfernt Treffer, die denselben Textabschnitt mehrfach abdecken."""
if not hits:
return []
ordered = sorted(hits, key=lambda hit: (-hit.score, hit.start_token))
kept: list["SearchHit"] = []
for hit in ordered:
overlapping = False
for existing in kept:
if existing.audio_id != hit.audio_id:
continue
same_region = (
abs(existing.start_token - hit.start_token) <= gap_tokens
or (hit.start_token < existing.end_token and existing.start_token < hit.end_token)
)
if same_region:
overlapping = True
break
if not overlapping:
kept.append(hit)
return kept
def rank_hits(hits: list["SearchHit"], limit: int) -> list["SearchHit"]:
"""Sortiert Treffer nach Relevanz und begrenzt die Anzahl."""
ordered = sorted(
hits,
key=lambda hit: (
-hit.score,
hit.filename.lower(),
hit.start_time,
),
)
return ordered[: max(0, int(limit))]