Initial commit
This commit is contained in:
@@ -0,0 +1,186 @@
|
||||
"""Kontextaufbau fuer Suchtreffer.
|
||||
|
||||
Aus den gespeicherten Segmenten wird eine durchgehende Wort-Timeline gebaut.
|
||||
Dadurch gehen an Segmentgrenzen keine Informationen verloren, und der Kontext
|
||||
kann mitten in einem Segment beginnen und enden.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Sequence
|
||||
|
||||
from .fuzzy import normalize_text, tokenize_original
|
||||
|
||||
|
||||
@dataclass
|
||||
class TokenRef:
|
||||
"""Ein Wort im Transkript mit Zeitbezug und Herkunft."""
|
||||
|
||||
text: str
|
||||
norm: str
|
||||
start: float
|
||||
end: float
|
||||
segment_id: int
|
||||
precision: str # "word" oder "segment"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Timeline:
|
||||
"""Durchgehende Wortfolge einer Audiodatei."""
|
||||
|
||||
audio_id: int
|
||||
tokens: list[TokenRef] = field(default_factory=list)
|
||||
language: str | None = None
|
||||
model: str | None = None
|
||||
|
||||
@property
|
||||
def normalized_tokens(self) -> list[str]:
|
||||
return [token.norm for token in self.tokens]
|
||||
|
||||
@property
|
||||
def normalized_text(self) -> str:
|
||||
return " ".join(token.norm for token in self.tokens)
|
||||
|
||||
@property
|
||||
def original_text(self) -> str:
|
||||
return " ".join(token.text for token in self.tokens)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.tokens)
|
||||
|
||||
|
||||
def _distribute_times(
|
||||
segment_start: float, segment_end: float, token_count: int
|
||||
) -> list[tuple[float, float]]:
|
||||
"""Verteilt die Segmentzeit gleichmaessig auf die Woerter (Schaetzung)."""
|
||||
if token_count <= 0:
|
||||
return []
|
||||
span = max(0.0, float(segment_end) - float(segment_start))
|
||||
step = span / token_count if span else 0.0
|
||||
times: list[tuple[float, float]] = []
|
||||
for index in range(token_count):
|
||||
start = float(segment_start) + step * index
|
||||
end = float(segment_start) + step * (index + 1)
|
||||
times.append((round(start, 3), round(end, 3)))
|
||||
return times
|
||||
|
||||
|
||||
def build_timeline(
|
||||
audio_id: int,
|
||||
segments: Sequence[Any],
|
||||
words_by_segment: dict[int, Sequence[Any]] | None = None,
|
||||
*,
|
||||
language: str | None = None,
|
||||
model: str | None = None,
|
||||
) -> Timeline:
|
||||
"""Baut die Timeline aus Segmenten (sqlite3.Row oder Objekte mit Attributen)."""
|
||||
timeline = Timeline(audio_id=audio_id, language=language, model=model)
|
||||
words_by_segment = words_by_segment or {}
|
||||
|
||||
for segment in segments:
|
||||
if isinstance(segment, dict):
|
||||
segment_id = int(segment["id"])
|
||||
start = float(segment["start_time"])
|
||||
end = float(segment["end_time"])
|
||||
text = segment["text"]
|
||||
else:
|
||||
segment_id = int(segment["id"])
|
||||
start = float(segment["start_time"])
|
||||
end = float(segment["end_time"])
|
||||
text = segment["text"]
|
||||
|
||||
original_tokens = tokenize_original(text)
|
||||
if not original_tokens:
|
||||
continue
|
||||
|
||||
words = list(words_by_segment.get(segment_id, []))
|
||||
use_words = len(words) == len(original_tokens)
|
||||
|
||||
if use_words:
|
||||
for token, word in zip(original_tokens, words):
|
||||
word_start = word["start_time"] if not isinstance(word, dict) else word.get("start_time")
|
||||
word_end = word["end_time"] if not isinstance(word, dict) else word.get("end_time")
|
||||
if word_start is None or word_end is None:
|
||||
use_words = False
|
||||
break
|
||||
|
||||
if use_words:
|
||||
for token, word in zip(original_tokens, words):
|
||||
word_start = word["start_time"] if not isinstance(word, dict) else word["start_time"]
|
||||
word_end = word["end_time"] if not isinstance(word, dict) else word["end_time"]
|
||||
timeline.tokens.append(
|
||||
TokenRef(
|
||||
text=token,
|
||||
norm=normalize_text(token),
|
||||
start=float(word_start),
|
||||
end=float(word_end),
|
||||
segment_id=segment_id,
|
||||
precision="word",
|
||||
)
|
||||
)
|
||||
else:
|
||||
times = _distribute_times(start, end, len(original_tokens))
|
||||
for token, (token_start, token_end) in zip(original_tokens, times):
|
||||
timeline.tokens.append(
|
||||
TokenRef(
|
||||
text=token,
|
||||
norm=normalize_text(token),
|
||||
start=token_start,
|
||||
end=token_end,
|
||||
segment_id=segment_id,
|
||||
precision="segment",
|
||||
)
|
||||
)
|
||||
|
||||
return timeline
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchContext:
|
||||
"""Die drei Kontextabschnitte eines Treffers."""
|
||||
|
||||
before: str
|
||||
match: str
|
||||
after: str
|
||||
start_time: float
|
||||
end_time: float
|
||||
precision: str
|
||||
|
||||
def as_block(self, quote: str = '"') -> str:
|
||||
"""Formatiert den Kontext als zusammenhaengenden Textblock."""
|
||||
before = f"\u2026{self.before} " if self.before else ""
|
||||
after = f" {self.after}\u2026" if self.after else ""
|
||||
return f"{quote}{before}{self.match}{after}{quote}"
|
||||
|
||||
|
||||
def build_context(
|
||||
timeline: Timeline, start_index: int, end_index: int, context_words: int = 50
|
||||
) -> SearchContext:
|
||||
"""Schneidet Kontext um den Treffer ``[start_index, end_index)`` aus."""
|
||||
tokens = timeline.tokens
|
||||
if not tokens:
|
||||
return SearchContext("", "", "", 0.0, 0.0, "segment")
|
||||
|
||||
start_index = max(0, min(start_index, len(tokens)))
|
||||
end_index = max(start_index + 1, min(end_index, len(tokens)))
|
||||
|
||||
before_start = max(0, start_index - max(0, context_words))
|
||||
after_end = min(len(tokens), end_index + max(0, context_words))
|
||||
|
||||
before_tokens = tokens[before_start:start_index]
|
||||
match_tokens = tokens[start_index:end_index]
|
||||
after_tokens = tokens[end_index:after_end]
|
||||
|
||||
precision = "word" if all(token.precision == "word" for token in match_tokens) else "segment"
|
||||
start_time = match_tokens[0].start
|
||||
end_time = match_tokens[-1].end
|
||||
|
||||
return SearchContext(
|
||||
before=" ".join(token.text for token in before_tokens),
|
||||
match=" ".join(token.text for token in match_tokens),
|
||||
after=" ".join(token.text for token in after_tokens),
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
precision=precision,
|
||||
)
|
||||
@@ -0,0 +1,303 @@
|
||||
"""Suchmaschine fuer SjAudioSearch.
|
||||
|
||||
Ablauf einer Suche:
|
||||
|
||||
1. Anfrage normalisieren.
|
||||
2. Kandidaten ueber FTS5 und den Trigramm-Index sammeln (schnelle Vorauswahl).
|
||||
3. Sliding-Window-Vergleich mit RapidFuzz auf der Wort-Timeline der Kandidaten.
|
||||
4. Optionaler, vollstaendiger Fallback-Lauf ueber alle verbleibenden Dateien,
|
||||
damit auch komplett falsch erkannte Woerter gefunden werden.
|
||||
5. Treffer entduplizieren, sortieren, begrenzen.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
|
||||
from rapidfuzz import fuzz
|
||||
|
||||
from ..config import SearchConfig
|
||||
from ..database.repository import Repository
|
||||
from ..utils.logging import get_logger
|
||||
from .context import SearchContext, Timeline, build_context, build_timeline
|
||||
from .fuzzy import (
|
||||
build_fts_query,
|
||||
normalize_text,
|
||||
score_window,
|
||||
significant_overlap,
|
||||
trigram_terms,
|
||||
)
|
||||
from .ranking import dedupe_overlapping, rank_hits
|
||||
|
||||
logger = get_logger("search.engine")
|
||||
|
||||
ProgressCallback = Callable[[int, int], None]
|
||||
|
||||
|
||||
class SearchError(RuntimeError):
|
||||
"""Fehler bei der Suche (z. B. leere Anfrage)."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchHit:
|
||||
"""Ein einzelner Suchtreffer."""
|
||||
|
||||
audio_id: int
|
||||
filename: str
|
||||
path: str
|
||||
score: float
|
||||
start_time: float
|
||||
end_time: float
|
||||
start_token: int
|
||||
end_token: int
|
||||
context: SearchContext
|
||||
language: str | None = None
|
||||
model: str | None = None
|
||||
source: str = "fts"
|
||||
|
||||
@property
|
||||
def matched_text(self) -> str:
|
||||
return self.context.match
|
||||
|
||||
def to_dict(self, context_words: int = 50) -> dict[str, object]:
|
||||
return {
|
||||
"audio_id": self.audio_id,
|
||||
"filename": self.filename,
|
||||
"path": self.path,
|
||||
"similarity": round(self.score, 2),
|
||||
"timestamp": self.start_time,
|
||||
"timestamp_end": self.end_time,
|
||||
"timestamp_precision": self.context.precision,
|
||||
"match": self.context.match,
|
||||
"context_before": self.context.before,
|
||||
"context_after": self.context.after,
|
||||
"context_words": context_words,
|
||||
"language": self.language,
|
||||
"model": self.model,
|
||||
"source": self.source,
|
||||
}
|
||||
|
||||
|
||||
class SearchEngine:
|
||||
"""Kombinierte Suche aus FTS5, Trigrammen und Fuzzy-Sliding-Window."""
|
||||
|
||||
def __init__(self, repository: Repository, config: SearchConfig):
|
||||
self.repo = repository
|
||||
self.config = config
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def search(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
limit: int | None = None,
|
||||
min_score: float | None = None,
|
||||
context_words: int | None = None,
|
||||
language: str | None = None,
|
||||
progress: ProgressCallback | None = None,
|
||||
) -> list[SearchHit]:
|
||||
query_normalized = normalize_text(query)
|
||||
if not query_normalized:
|
||||
raise SearchError("Die Suchanfrage enthaelt keine verwertbaren Woerter.")
|
||||
query_tokens = query_normalized.split()
|
||||
|
||||
limit = self.config.default_limit if limit is None else int(limit)
|
||||
min_score = self.config.default_min_score if min_score is None else float(min_score)
|
||||
context_words = self.config.context_words if context_words is None else int(context_words)
|
||||
if limit <= 0:
|
||||
raise SearchError("--limit muss groesser als 0 sein.")
|
||||
|
||||
candidate_audio, candidate_source = self._collect_candidates(query_tokens)
|
||||
hits: list[SearchHit] = []
|
||||
swept: set[int] = set()
|
||||
|
||||
total_candidates = len(candidate_audio)
|
||||
for index, audio_id in enumerate(candidate_audio, start=1):
|
||||
swept.add(audio_id)
|
||||
hits.extend(
|
||||
self._scan_audio(
|
||||
audio_id,
|
||||
query_normalized,
|
||||
query_tokens,
|
||||
min_score=min_score,
|
||||
context_words=context_words,
|
||||
language=language,
|
||||
source="fts",
|
||||
)
|
||||
)
|
||||
if progress:
|
||||
progress(index, total_candidates)
|
||||
|
||||
# Wichtig: FTS5 ist nur eine Vorauswahl. Wenn wenige Treffer gefunden
|
||||
# wurden, werden alle uebrigen Dateien vollstaendig durchsucht.
|
||||
fallback_used = False
|
||||
if self.config.exhaustive_fallback and len(hits) < limit:
|
||||
remaining = [
|
||||
audio_id
|
||||
for audio_id in self.repo.iter_audio_ids_with_transcripts(language)
|
||||
if audio_id not in swept
|
||||
]
|
||||
fallback_used = bool(remaining)
|
||||
for index, audio_id in enumerate(remaining, start=1):
|
||||
hits.extend(
|
||||
self._scan_audio(
|
||||
audio_id,
|
||||
query_normalized,
|
||||
query_tokens,
|
||||
min_score=min_score,
|
||||
context_words=context_words,
|
||||
language=language,
|
||||
source="fallback",
|
||||
)
|
||||
)
|
||||
if progress:
|
||||
progress(index, len(remaining))
|
||||
|
||||
if fallback_used:
|
||||
logger.info(
|
||||
"Fallback-Suche aktiv: %d Dateien ohne FTS-Treffer durchsucht", len(remaining)
|
||||
)
|
||||
|
||||
deduped = dedupe_overlapping(hits)
|
||||
return rank_hits(deduped, limit)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _collect_candidates(self, query_tokens: list[str]) -> tuple[list[int], str]:
|
||||
"""Sammelt Kandidaten-Audiodateien (FTS5 + Trigramme)."""
|
||||
limit = max(1, int(self.config.candidate_limit))
|
||||
matches: list[tuple[int, int]] = []
|
||||
|
||||
fts_query = build_fts_query(query_tokens)
|
||||
if fts_query:
|
||||
matches.extend(self.repo.fts_candidates(fts_query, limit))
|
||||
|
||||
remaining = limit - len(matches)
|
||||
if remaining > 0:
|
||||
terms = trigram_terms(query_tokens)
|
||||
if terms:
|
||||
matches.extend(self.repo.trigram_candidates(terms, remaining))
|
||||
|
||||
counts: dict[int, int] = {}
|
||||
for audio_id, _segment_id in matches:
|
||||
counts[audio_id] = counts.get(audio_id, 0) + 1
|
||||
|
||||
ordered = sorted(counts, key=lambda audio_id: (-counts[audio_id], audio_id))
|
||||
return ordered, "fts"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _scan_audio(
|
||||
self,
|
||||
audio_id: int,
|
||||
query_normalized: str,
|
||||
query_tokens: list[str],
|
||||
*,
|
||||
min_score: float,
|
||||
context_words: int,
|
||||
language: str | None,
|
||||
source: str,
|
||||
) -> list[SearchHit]:
|
||||
record = self.repo.get_audio(audio_id)
|
||||
if record is None or record.duplicate_of is not None:
|
||||
return []
|
||||
|
||||
transcript = self.repo.get_transcript(audio_id)
|
||||
if transcript is None:
|
||||
return []
|
||||
if language and (transcript["language"] or "").lower() != language.lower():
|
||||
return []
|
||||
|
||||
segments, words = self.repo.timeline(audio_id)
|
||||
if not segments:
|
||||
return []
|
||||
timeline = build_timeline(
|
||||
audio_id,
|
||||
segments,
|
||||
words,
|
||||
language=transcript["language"],
|
||||
model=transcript["model"],
|
||||
)
|
||||
if not timeline.tokens:
|
||||
return []
|
||||
|
||||
windows = self._find_windows(timeline, query_normalized, query_tokens, min_score)
|
||||
if not windows:
|
||||
return []
|
||||
|
||||
hits: list[SearchHit] = []
|
||||
for score, start_index, end_index in windows:
|
||||
context = build_context(timeline, start_index, end_index, context_words)
|
||||
hits.append(
|
||||
SearchHit(
|
||||
audio_id=audio_id,
|
||||
filename=record.filename,
|
||||
path=record.path,
|
||||
score=score,
|
||||
start_time=context.start_time,
|
||||
end_time=context.end_time,
|
||||
start_token=start_index,
|
||||
end_token=end_index,
|
||||
context=context,
|
||||
language=transcript["language"],
|
||||
model=transcript["model"],
|
||||
source=source,
|
||||
)
|
||||
)
|
||||
|
||||
hits.sort(key=lambda hit: -hit.score)
|
||||
return hits[: max(1, int(self.config.max_hits_per_file))]
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _find_windows(
|
||||
self,
|
||||
timeline: Timeline,
|
||||
query_normalized: str,
|
||||
query_tokens: list[str],
|
||||
min_score: float,
|
||||
) -> list[tuple[float, int, int]]:
|
||||
"""Sliding-Window-Suche ueber die Wort-Timeline einer Datei."""
|
||||
norm_tokens = timeline.normalized_tokens
|
||||
token_count = len(norm_tokens)
|
||||
query_len = len(query_tokens)
|
||||
if token_count == 0:
|
||||
return []
|
||||
|
||||
min_size = max(1, query_len - max(0, self.config.window_before))
|
||||
max_size = max(min_size, query_len + max(0, self.config.window_after))
|
||||
overlap_threshold = max(0.0, min(1.0, self.config.min_token_overlap))
|
||||
|
||||
# Dateiebene: gibt es ueberhaupt Beruehrungspunkte? Das ist deutlich
|
||||
# billiger als jede Datei Fenster fuer Fenster zu vergleichen.
|
||||
file_overlap = significant_overlap(query_tokens, norm_tokens)
|
||||
if file_overlap < overlap_threshold * 0.5:
|
||||
partial = fuzz.partial_ratio(query_normalized, timeline.normalized_text)
|
||||
if partial < 55.0:
|
||||
return []
|
||||
|
||||
windows: list[tuple[float, int, int]] = []
|
||||
# Wenn auf Dateiebene kaum Wortueberschneidung besteht, kann der Treffer
|
||||
# nur ueber einen falsch erkannten Ausdruck zustande kommen. Dann wird
|
||||
# ohne Wort-Vorfilter verglichen (selten und durch die Dateipruefung
|
||||
# oben bereits eingegrenzt).
|
||||
use_overlap_filter = file_overlap >= overlap_threshold
|
||||
for start in range(token_count):
|
||||
pre_end = min(token_count, start + query_len + 2)
|
||||
if use_overlap_filter and significant_overlap(query_tokens, norm_tokens[start:pre_end]) < overlap_threshold:
|
||||
continue
|
||||
|
||||
best_score = 0.0
|
||||
best_size = 0
|
||||
for size in range(min_size, max_size + 1):
|
||||
end = start + size
|
||||
if end > token_count:
|
||||
break
|
||||
window_normalized = " ".join(norm_tokens[start:end])
|
||||
candidate = score_window(query_normalized, window_normalized)
|
||||
if candidate.score > best_score:
|
||||
best_score = candidate.score
|
||||
best_size = size
|
||||
|
||||
if best_size and best_score >= min_score:
|
||||
windows.append((best_score, start, start + best_size))
|
||||
|
||||
return windows
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Textnormalisierung und Fuzzy-Vergleich (RapidFuzz).
|
||||
|
||||
Wichtig: Die Normalisierung wird ausschliesslich fuer Vergleiche und fuer den
|
||||
Suchindex verwendet. Der gespeicherte Originaltext bleibt unveraendert.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
from dataclasses import dataclass
|
||||
|
||||
from rapidfuzz import fuzz
|
||||
|
||||
# Satzzeichen und Symbole, die beim Vergleich stoeren. Wortzeichen (Buchstaben,
|
||||
# Ziffern, Unterstrich) bleiben erhalten.
|
||||
_PUNCT_RE = re.compile(r"[^\w\s]", re.UNICODE)
|
||||
_WHITESPACE_RE = re.compile(r"\s+", re.UNICODE)
|
||||
|
||||
# Sehr haeufige deutsche Fuellwoerter. Sie werden beim Vorfilter nicht als
|
||||
# belastbare Uebereinstimmung gewertet, damit "der/die/das" allein keine
|
||||
# Kandidaten erzeugt.
|
||||
STOPWORDS = frozenset(
|
||||
{
|
||||
"der", "die", "das", "den", "dem", "des", "ein", "eine", "einer", "eines",
|
||||
"einem", "einen", "und", "oder", "aber", "als", "auch", "auf", "aus", "bei",
|
||||
"bin", "bis", "da", "dann", "dass", "doch", "dort", "du", "durch", "er",
|
||||
"es", "hat", "ich", "ihm", "ihn", "ihr", "im", "in", "ist", "mit", "nach",
|
||||
"nicht", "noch", "nun", "nur", "ob", "so", "sich", "sie", "sind", "um",
|
||||
"und", "vom", "von", "vor", "war", "was", "wenn", "wie", "wir", "wo", "zu",
|
||||
"zum", "zur", "the", "and", "of", "to", "in", "a", "is",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def normalize_text(text: str) -> str:
|
||||
"""Normalisiert Text fuer Vergleiche: NFKC, Kleinschreibung, ohne Satzzeichen."""
|
||||
if not text:
|
||||
return ""
|
||||
normalized = unicodedata.normalize("NFKC", text).casefold()
|
||||
normalized = _PUNCT_RE.sub(" ", normalized)
|
||||
return _WHITESPACE_RE.sub(" ", normalized).strip()
|
||||
|
||||
|
||||
def tokenize(text: str) -> list[str]:
|
||||
"""Normalisierte Wortliste eines Textes."""
|
||||
normalized = normalize_text(text)
|
||||
return normalized.split() if normalized else []
|
||||
|
||||
|
||||
def tokenize_original(text: str) -> list[str]:
|
||||
"""Wortliste mit Originaltext inklusive anhängender Satzzeichen (fuer die Ausgabe)."""
|
||||
if not text:
|
||||
return []
|
||||
return [token for token in _WHITESPACE_RE.split(text.strip()) if token]
|
||||
|
||||
|
||||
def token_norm(token: str) -> str:
|
||||
"""Normalisiert ein einzelnes Originalwort."""
|
||||
return normalize_text(token)
|
||||
|
||||
|
||||
def is_stopword(token: str) -> bool:
|
||||
return token in STOPWORDS
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WindowScore:
|
||||
"""Ergebnis eines Fenstervergleichs."""
|
||||
|
||||
score: float
|
||||
exact: bool
|
||||
|
||||
|
||||
def score_window(query_normalized: str, window_normalized: str) -> WindowScore:
|
||||
"""Vergleicht Suchanfrage und Textfenster und liefert einen Wert 0-100.
|
||||
|
||||
Es werden mehrere RapidFuzz-Kennzahlen kombiniert, damit fehlende,
|
||||
zusaetzliche, vertauschte oder falsch erkannte Woerter nicht sofort zum
|
||||
Verlust des Treffers fuehren.
|
||||
"""
|
||||
if not query_normalized or not window_normalized:
|
||||
return WindowScore(0.0, False)
|
||||
|
||||
if query_normalized == window_normalized:
|
||||
return WindowScore(100.0, True)
|
||||
|
||||
# Hinweis: Eine reine Teilzeichenketten-Pruefung ist hier absichtlich NICHT
|
||||
# enthalten. Sie wuerde z. B. das Fenster "Leider" fuer die Anfrage
|
||||
# "Leider zu spaet" mit 100 bewerten und den Treffer auf ein Wort
|
||||
# zusammenfallen lassen. Teiluebereinstimmungen werden stattdessen ueber
|
||||
# die gewichteten RapidFuzz-Kennzahlen unten bewertet.
|
||||
|
||||
ratio = fuzz.ratio(query_normalized, window_normalized)
|
||||
token_set = fuzz.token_set_ratio(query_normalized, window_normalized)
|
||||
weighted = fuzz.WRatio(query_normalized, window_normalized)
|
||||
partial = fuzz.partial_ratio(query_normalized, window_normalized)
|
||||
|
||||
# partial_ratio neigt bei kurzen Fenstern zu hohen Werten, daher nur
|
||||
# gedämpft einbeziehen.
|
||||
# token_set_ratio/WRatio/partial_ratio bewerten eine Teilmenge der Anfrage
|
||||
# mit 100. Ohne Daempfung wuerde ein einzelnes Wort einer mehrwoertrigen
|
||||
# Anfrage als perfekter Treffer gelten. Daher werden diese Kennzahlen mit
|
||||
# dem Laengenverhaeltnis gewichtet, wenn das Fenster kuerzer ist.
|
||||
length_factor = min(1.0, len(window_normalized) / max(1, len(query_normalized)))
|
||||
|
||||
score = max(
|
||||
ratio,
|
||||
token_set * length_factor,
|
||||
weighted * length_factor,
|
||||
partial * 0.92 * length_factor,
|
||||
)
|
||||
return WindowScore(min(100.0, round(score, 2)), False)
|
||||
|
||||
|
||||
def token_overlap(query_tokens: list[str], window_tokens: list[str]) -> float:
|
||||
"""Anteil der Anfragewoerter, die im Fenster vorkommen (0..1)."""
|
||||
if not query_tokens:
|
||||
return 0.0
|
||||
window_set = set(window_tokens)
|
||||
hits = sum(1 for token in query_tokens if token in window_set)
|
||||
return hits / len(query_tokens)
|
||||
|
||||
|
||||
def significant_overlap(query_tokens: list[str], window_tokens: list[str]) -> float:
|
||||
"""Wie :func:`token_overlap`, aber Fuellwoerter zaehlen nur halb."""
|
||||
if not query_tokens:
|
||||
return 0.0
|
||||
window_set = set(window_tokens)
|
||||
total = 0.0
|
||||
hits = 0.0
|
||||
for token in query_tokens:
|
||||
weight = 0.5 if is_stopword(token) else 1.0
|
||||
total += weight
|
||||
if token in window_set:
|
||||
hits += weight
|
||||
if total == 0:
|
||||
return 0.0
|
||||
return hits / total
|
||||
|
||||
|
||||
def _escape_fts_token(token: str) -> str:
|
||||
return '"' + token.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def build_fts_query(tokens: list[str], prefix_min_length: int = 4) -> str:
|
||||
"""Baut eine FTS5-Anfrage aus ODER-verknuepften Woertern.
|
||||
|
||||
Praefixsuche fuer laengere Woerter faengt deutsche Flexionsformen und
|
||||
Zusammensetzungen ab. Die Anfrage ist bewusst weit gefasst: sie dient nur
|
||||
der Kandidatensuche, die Bewertung erfolgt spaeter per Fuzzy-Vergleich.
|
||||
"""
|
||||
parts: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for token in tokens:
|
||||
if len(token) < 2 or token in seen:
|
||||
continue
|
||||
seen.add(token)
|
||||
escaped = _escape_fts_token(token)
|
||||
if len(token) >= prefix_min_length:
|
||||
parts.append(f"{escaped}*")
|
||||
else:
|
||||
parts.append(escaped)
|
||||
return " OR ".join(parts)
|
||||
|
||||
|
||||
def trigram_terms(tokens: list[str], max_terms: int = 32) -> list[str]:
|
||||
"""Sinnvolle Trigramm-Anfragen aus laengeren Anfragewoertern."""
|
||||
terms: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for token in tokens:
|
||||
if len(token) < 4 or token in seen:
|
||||
continue
|
||||
seen.add(token)
|
||||
terms.append(token)
|
||||
if len(terms) >= max_terms:
|
||||
break
|
||||
return terms
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Bewertung und Sortierung von Suchtreffern.
|
||||
|
||||
Der Score ist eine Aehnlichkeitsbewertung (0-100) und ausdruecklich keine
|
||||
statistische Wahrscheinlichkeit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from .engine import SearchHit
|
||||
|
||||
|
||||
def clamp_score(score: float) -> float:
|
||||
return max(0.0, min(100.0, float(score)))
|
||||
|
||||
|
||||
def dedupe_overlapping(hits: list["SearchHit"], gap_tokens: int = 3) -> list["SearchHit"]:
|
||||
"""Entfernt Treffer, die denselben Textabschnitt mehrfach abdecken."""
|
||||
if not hits:
|
||||
return []
|
||||
|
||||
ordered = sorted(hits, key=lambda hit: (-hit.score, hit.start_token))
|
||||
kept: list["SearchHit"] = []
|
||||
for hit in ordered:
|
||||
overlapping = False
|
||||
for existing in kept:
|
||||
if existing.audio_id != hit.audio_id:
|
||||
continue
|
||||
same_region = (
|
||||
abs(existing.start_token - hit.start_token) <= gap_tokens
|
||||
or (hit.start_token < existing.end_token and existing.start_token < hit.end_token)
|
||||
)
|
||||
if same_region:
|
||||
overlapping = True
|
||||
break
|
||||
if not overlapping:
|
||||
kept.append(hit)
|
||||
return kept
|
||||
|
||||
|
||||
def rank_hits(hits: list["SearchHit"], limit: int) -> list["SearchHit"]:
|
||||
"""Sortiert Treffer nach Relevanz und begrenzt die Anzahl."""
|
||||
ordered = sorted(
|
||||
hits,
|
||||
key=lambda hit: (
|
||||
-hit.score,
|
||||
hit.filename.lower(),
|
||||
hit.start_time,
|
||||
),
|
||||
)
|
||||
return ordered[: max(0, int(limit))]
|
||||
Reference in New Issue
Block a user