Initial commit
This commit is contained in:
@@ -0,0 +1,192 @@
|
||||
"""Tests fuer die Suchmaschine (Fuzzy, Sliding Window, Fallback)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from sjaudiosearch.search.engine import SearchEngine, SearchError
|
||||
from sjaudiosearch.search.fuzzy import build_fts_query, normalize_text, score_window
|
||||
|
||||
|
||||
def _engine(repo, search_config) -> SearchEngine:
|
||||
return SearchEngine(repo, search_config)
|
||||
|
||||
|
||||
def test_fuzzy_search_finds_slightly_different_sentence(repo, add_audio, store_transcript, search_config):
|
||||
record = add_audio("/tmp/hoerspiel_042.mp3", duration=3000.0)
|
||||
store_transcript(
|
||||
record.id,
|
||||
[("Der Professor sah sich um. Der Schlüssel befindet sich in dem alten Haus. "
|
||||
"Niemand hatte das bisher bemerkt.", 42 * 60 + 10.0, 42 * 60 + 20.0)],
|
||||
duration=3000.0,
|
||||
)
|
||||
|
||||
hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus")
|
||||
|
||||
assert len(hits) == 1
|
||||
hit = hits[0]
|
||||
assert hit.filename == "hoerspiel_042.mp3"
|
||||
assert hit.path == record.path
|
||||
assert hit.score >= 85
|
||||
assert 42 * 60 <= hit.start_time <= 42 * 60 + 40
|
||||
assert "Schlüssel" in hit.matched_text
|
||||
assert hit.context.before
|
||||
|
||||
|
||||
def test_exact_phrase_scores_100(repo, add_audio, store_transcript, search_config):
|
||||
record = add_audio("/tmp/exakt.mp3")
|
||||
store_transcript(record.id, ["Der Schlüssel liegt im alten Haus."])
|
||||
|
||||
hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus")
|
||||
|
||||
assert hits
|
||||
assert hits[0].score == pytest.approx(100.0, abs=0.1)
|
||||
|
||||
|
||||
def test_search_across_segment_boundaries(repo, add_audio, store_transcript, search_config):
|
||||
record = add_audio("/tmp/krimi.mp3")
|
||||
store_transcript(
|
||||
record.id,
|
||||
[
|
||||
("Der Professor sah sich um. Der Schlüssel", 10.0, 15.0),
|
||||
("liegt in dem alten Haus. Niemand bemerkte es.", 15.0, 20.0),
|
||||
],
|
||||
)
|
||||
|
||||
hits = _engine(repo, search_config).search("Der Schlüssel liegt in dem alten Haus")
|
||||
|
||||
assert hits
|
||||
hit = hits[0]
|
||||
assert hit.score >= 90
|
||||
assert "Schlüssel" in hit.matched_text
|
||||
assert "Haus" in hit.matched_text
|
||||
# Der Treffer beginnt beim Wort "Schlüssel" innerhalb des ersten Segments.
|
||||
assert 10.0 <= hit.start_time <= 16.0
|
||||
|
||||
|
||||
def test_fallback_finds_word_fts_cannot_match(repo, add_audio, store_transcript, search_config):
|
||||
"""Whisper-Transkripte koennen Woerter komplett falsch schreiben.
|
||||
|
||||
Ohne FTS-Treffer muss der Fallback-Lauf die Datei trotzdem durchsuchen.
|
||||
"""
|
||||
record = add_audio("/tmp/misheard.mp3")
|
||||
store_transcript(record.id, [("Die Sage vom Schluessel im alten Gemäuer.", 0.0, 6.0)])
|
||||
|
||||
# Keine exakte Uebereinstimmung fuer den Suchbegriff im FTS-Index
|
||||
assert repo.fts_candidates(build_fts_query(["schlüssel"]), 10) == []
|
||||
assert repo.trigram_candidates(["schlüssel"], 10) == []
|
||||
|
||||
hits = _engine(repo, search_config).search("Schlüssel")
|
||||
|
||||
assert len(hits) == 1
|
||||
assert hits[0].audio_id == record.id
|
||||
assert hits[0].source == "fallback"
|
||||
assert hits[0].score >= 70
|
||||
|
||||
|
||||
def test_search_returns_nothing_for_unrelated_text(repo, add_audio, store_transcript, search_config):
|
||||
record = add_audio("/tmp/unrelated.mp3")
|
||||
store_transcript(record.id, ["Es war einmal ein kleiner Drache im Wald."])
|
||||
|
||||
hits = _engine(repo, search_config).search("Quantenphysik Vorlesung")
|
||||
assert hits == []
|
||||
|
||||
|
||||
def test_min_score_and_limit(repo, add_audio, store_transcript, search_config):
|
||||
for index in range(5):
|
||||
record = add_audio(f"/tmp/folge{index}.mp3")
|
||||
store_transcript(
|
||||
record.id,
|
||||
[f"Der Schlüssel liegt im alten Haus Nummer {index}.", ],
|
||||
)
|
||||
|
||||
engine = _engine(repo, search_config)
|
||||
all_hits = engine.search("Der Schlüssel liegt im alten Haus", limit=10, min_score=50)
|
||||
assert len(all_hits) >= 1
|
||||
|
||||
limited = engine.search("Der Schlüssel liegt im alten Haus", limit=2, min_score=50)
|
||||
assert len(limited) == 2
|
||||
|
||||
strict = engine.search("Der Schlüssel liegt im alten Haus", limit=10, min_score=99.5)
|
||||
assert len(strict) <= len(all_hits)
|
||||
|
||||
|
||||
def test_results_sorted_by_score(repo, add_audio, store_transcript, search_config):
|
||||
exact = add_audio("/tmp/exact.mp3")
|
||||
store_transcript(exact.id, ["Der Schlüssel liegt im alten Haus."])
|
||||
fuzzy = add_audio("/tmp/fuzzy.mp3")
|
||||
store_transcript(fuzzy.id, ["Der Schlüssel ist wohl im alten Haus zu finden, oder?"])
|
||||
|
||||
hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus", min_score=60)
|
||||
|
||||
scores = [hit.score for hit in hits]
|
||||
assert scores == sorted(scores, reverse=True)
|
||||
assert hits[0].filename == "exact.mp3"
|
||||
|
||||
|
||||
def test_language_filter(repo, add_audio, store_transcript, search_config):
|
||||
german = add_audio("/tmp/de.mp3")
|
||||
store_transcript(german.id, ["Der Schlüssel liegt im alten Haus."], language="de")
|
||||
english = add_audio("/tmp/en.mp3")
|
||||
store_transcript(english.id, ["The key is in the old house."], language="en")
|
||||
|
||||
hits = _engine(repo, search_config).search(
|
||||
"The key is in the old house", language="en", min_score=60
|
||||
)
|
||||
assert [hit.audio_id for hit in hits] == [english.id]
|
||||
|
||||
german_hits = _engine(repo, search_config).search(
|
||||
"Der Schlüssel liegt im alten Haus", language="de", min_score=60
|
||||
)
|
||||
assert [hit.audio_id for hit in german_hits] == [german.id]
|
||||
|
||||
|
||||
def test_duplicate_files_are_not_searched(repo, add_audio, store_transcript, search_config):
|
||||
original = add_audio("/tmp/original.mp3")
|
||||
store_transcript(original.id, ["Der Schlüssel liegt im alten Haus."])
|
||||
duplicate = add_audio("/tmp/kopie.mp3", duplicate_of=original.id)
|
||||
|
||||
hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus")
|
||||
assert {hit.audio_id for hit in hits} == {original.id}
|
||||
|
||||
|
||||
def test_empty_query_is_rejected(repo, search_config):
|
||||
with pytest.raises(SearchError):
|
||||
_engine(repo, search_config).search(" ... ")
|
||||
|
||||
|
||||
def test_word_insertions_and_deletions(repo, add_audio, store_transcript, search_config):
|
||||
record = add_audio("/tmp/insert.mp3")
|
||||
store_transcript(
|
||||
record.id,
|
||||
["Also, der Schlüssel liegt wirklich direkt im alten Haus, nicht wahr?"],
|
||||
)
|
||||
|
||||
hits = _engine(repo, search_config).search("Schlüssel liegt im alten Haus")
|
||||
assert hits
|
||||
assert hits[0].score >= 80
|
||||
|
||||
|
||||
def test_multiword_query_matches_whole_phrase_not_single_word(repo, add_audio, store_transcript, search_config):
|
||||
"""Regression: token_set_ratio darf ein Einzelwort nicht als 100% werten."""
|
||||
record = add_audio("/tmp/kurz.mp3")
|
||||
store_transcript(record.id, [("Leider zu spät.", 0.0, 2.0)])
|
||||
|
||||
hits = _engine(repo, search_config).search("Leider zu spät", limit=5, min_score=70)
|
||||
|
||||
assert hits
|
||||
assert hits[0].context.match == "Leider zu spät."
|
||||
assert hits[0].score == pytest.approx(100.0, abs=0.1)
|
||||
|
||||
|
||||
def test_normalization_and_scoring_helpers():
|
||||
assert normalize_text("Der Schlüssel, liegt im alten Haus!") == "der schlüssel liegt im alten haus"
|
||||
assert score_window("der schlüssel liegt im alten haus", "der schlüssel liegt im alten haus").score == 100.0
|
||||
assert score_window("der schlüssel liegt im alten haus", "").score == 0.0
|
||||
assert build_fts_query(["haus"]).startswith('"haus"')
|
||||
|
||||
|
||||
def test_partial_window_is_not_a_perfect_match():
|
||||
# Ein einzelnes Wort einer mehrwoertrigen Anfrage darf nicht 100 ergeben.
|
||||
assert score_window("leider zu spät", "leider").score < 70
|
||||
assert score_window("der schlüssel liegt im alten haus", "haus").score < 70
|
||||
Reference in New Issue
Block a user