193 lines
7.3 KiB
Python
193 lines
7.3 KiB
Python
"""Tests fuer die Suchmaschine (Fuzzy, Sliding Window, Fallback)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from sjaudiosearch.search.engine import SearchEngine, SearchError
|
|
from sjaudiosearch.search.fuzzy import build_fts_query, normalize_text, score_window
|
|
|
|
|
|
def _engine(repo, search_config) -> SearchEngine:
|
|
return SearchEngine(repo, search_config)
|
|
|
|
|
|
def test_fuzzy_search_finds_slightly_different_sentence(repo, add_audio, store_transcript, search_config):
|
|
record = add_audio("/tmp/hoerspiel_042.mp3", duration=3000.0)
|
|
store_transcript(
|
|
record.id,
|
|
[("Der Professor sah sich um. Der Schlüssel befindet sich in dem alten Haus. "
|
|
"Niemand hatte das bisher bemerkt.", 42 * 60 + 10.0, 42 * 60 + 20.0)],
|
|
duration=3000.0,
|
|
)
|
|
|
|
hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus")
|
|
|
|
assert len(hits) == 1
|
|
hit = hits[0]
|
|
assert hit.filename == "hoerspiel_042.mp3"
|
|
assert hit.path == record.path
|
|
assert hit.score >= 85
|
|
assert 42 * 60 <= hit.start_time <= 42 * 60 + 40
|
|
assert "Schlüssel" in hit.matched_text
|
|
assert hit.context.before
|
|
|
|
|
|
def test_exact_phrase_scores_100(repo, add_audio, store_transcript, search_config):
|
|
record = add_audio("/tmp/exakt.mp3")
|
|
store_transcript(record.id, ["Der Schlüssel liegt im alten Haus."])
|
|
|
|
hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus")
|
|
|
|
assert hits
|
|
assert hits[0].score == pytest.approx(100.0, abs=0.1)
|
|
|
|
|
|
def test_search_across_segment_boundaries(repo, add_audio, store_transcript, search_config):
|
|
record = add_audio("/tmp/krimi.mp3")
|
|
store_transcript(
|
|
record.id,
|
|
[
|
|
("Der Professor sah sich um. Der Schlüssel", 10.0, 15.0),
|
|
("liegt in dem alten Haus. Niemand bemerkte es.", 15.0, 20.0),
|
|
],
|
|
)
|
|
|
|
hits = _engine(repo, search_config).search("Der Schlüssel liegt in dem alten Haus")
|
|
|
|
assert hits
|
|
hit = hits[0]
|
|
assert hit.score >= 90
|
|
assert "Schlüssel" in hit.matched_text
|
|
assert "Haus" in hit.matched_text
|
|
# Der Treffer beginnt beim Wort "Schlüssel" innerhalb des ersten Segments.
|
|
assert 10.0 <= hit.start_time <= 16.0
|
|
|
|
|
|
def test_fallback_finds_word_fts_cannot_match(repo, add_audio, store_transcript, search_config):
|
|
"""Whisper-Transkripte koennen Woerter komplett falsch schreiben.
|
|
|
|
Ohne FTS-Treffer muss der Fallback-Lauf die Datei trotzdem durchsuchen.
|
|
"""
|
|
record = add_audio("/tmp/misheard.mp3")
|
|
store_transcript(record.id, [("Die Sage vom Schluessel im alten Gemäuer.", 0.0, 6.0)])
|
|
|
|
# Keine exakte Uebereinstimmung fuer den Suchbegriff im FTS-Index
|
|
assert repo.fts_candidates(build_fts_query(["schlüssel"]), 10) == []
|
|
assert repo.trigram_candidates(["schlüssel"], 10) == []
|
|
|
|
hits = _engine(repo, search_config).search("Schlüssel")
|
|
|
|
assert len(hits) == 1
|
|
assert hits[0].audio_id == record.id
|
|
assert hits[0].source == "fallback"
|
|
assert hits[0].score >= 70
|
|
|
|
|
|
def test_search_returns_nothing_for_unrelated_text(repo, add_audio, store_transcript, search_config):
|
|
record = add_audio("/tmp/unrelated.mp3")
|
|
store_transcript(record.id, ["Es war einmal ein kleiner Drache im Wald."])
|
|
|
|
hits = _engine(repo, search_config).search("Quantenphysik Vorlesung")
|
|
assert hits == []
|
|
|
|
|
|
def test_min_score_and_limit(repo, add_audio, store_transcript, search_config):
|
|
for index in range(5):
|
|
record = add_audio(f"/tmp/folge{index}.mp3")
|
|
store_transcript(
|
|
record.id,
|
|
[f"Der Schlüssel liegt im alten Haus Nummer {index}.", ],
|
|
)
|
|
|
|
engine = _engine(repo, search_config)
|
|
all_hits = engine.search("Der Schlüssel liegt im alten Haus", limit=10, min_score=50)
|
|
assert len(all_hits) >= 1
|
|
|
|
limited = engine.search("Der Schlüssel liegt im alten Haus", limit=2, min_score=50)
|
|
assert len(limited) == 2
|
|
|
|
strict = engine.search("Der Schlüssel liegt im alten Haus", limit=10, min_score=99.5)
|
|
assert len(strict) <= len(all_hits)
|
|
|
|
|
|
def test_results_sorted_by_score(repo, add_audio, store_transcript, search_config):
|
|
exact = add_audio("/tmp/exact.mp3")
|
|
store_transcript(exact.id, ["Der Schlüssel liegt im alten Haus."])
|
|
fuzzy = add_audio("/tmp/fuzzy.mp3")
|
|
store_transcript(fuzzy.id, ["Der Schlüssel ist wohl im alten Haus zu finden, oder?"])
|
|
|
|
hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus", min_score=60)
|
|
|
|
scores = [hit.score for hit in hits]
|
|
assert scores == sorted(scores, reverse=True)
|
|
assert hits[0].filename == "exact.mp3"
|
|
|
|
|
|
def test_language_filter(repo, add_audio, store_transcript, search_config):
|
|
german = add_audio("/tmp/de.mp3")
|
|
store_transcript(german.id, ["Der Schlüssel liegt im alten Haus."], language="de")
|
|
english = add_audio("/tmp/en.mp3")
|
|
store_transcript(english.id, ["The key is in the old house."], language="en")
|
|
|
|
hits = _engine(repo, search_config).search(
|
|
"The key is in the old house", language="en", min_score=60
|
|
)
|
|
assert [hit.audio_id for hit in hits] == [english.id]
|
|
|
|
german_hits = _engine(repo, search_config).search(
|
|
"Der Schlüssel liegt im alten Haus", language="de", min_score=60
|
|
)
|
|
assert [hit.audio_id for hit in german_hits] == [german.id]
|
|
|
|
|
|
def test_duplicate_files_are_not_searched(repo, add_audio, store_transcript, search_config):
|
|
original = add_audio("/tmp/original.mp3")
|
|
store_transcript(original.id, ["Der Schlüssel liegt im alten Haus."])
|
|
duplicate = add_audio("/tmp/kopie.mp3", duplicate_of=original.id)
|
|
|
|
hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus")
|
|
assert {hit.audio_id for hit in hits} == {original.id}
|
|
|
|
|
|
def test_empty_query_is_rejected(repo, search_config):
|
|
with pytest.raises(SearchError):
|
|
_engine(repo, search_config).search(" ... ")
|
|
|
|
|
|
def test_word_insertions_and_deletions(repo, add_audio, store_transcript, search_config):
|
|
record = add_audio("/tmp/insert.mp3")
|
|
store_transcript(
|
|
record.id,
|
|
["Also, der Schlüssel liegt wirklich direkt im alten Haus, nicht wahr?"],
|
|
)
|
|
|
|
hits = _engine(repo, search_config).search("Schlüssel liegt im alten Haus")
|
|
assert hits
|
|
assert hits[0].score >= 80
|
|
|
|
|
|
def test_multiword_query_matches_whole_phrase_not_single_word(repo, add_audio, store_transcript, search_config):
|
|
"""Regression: token_set_ratio darf ein Einzelwort nicht als 100% werten."""
|
|
record = add_audio("/tmp/kurz.mp3")
|
|
store_transcript(record.id, [("Leider zu spät.", 0.0, 2.0)])
|
|
|
|
hits = _engine(repo, search_config).search("Leider zu spät", limit=5, min_score=70)
|
|
|
|
assert hits
|
|
assert hits[0].context.match == "Leider zu spät."
|
|
assert hits[0].score == pytest.approx(100.0, abs=0.1)
|
|
|
|
|
|
def test_normalization_and_scoring_helpers():
|
|
assert normalize_text("Der Schlüssel, liegt im alten Haus!") == "der schlüssel liegt im alten haus"
|
|
assert score_window("der schlüssel liegt im alten haus", "der schlüssel liegt im alten haus").score == 100.0
|
|
assert score_window("der schlüssel liegt im alten haus", "").score == 0.0
|
|
assert build_fts_query(["haus"]).startswith('"haus"')
|
|
|
|
|
|
def test_partial_window_is_not_a_perfect_match():
|
|
# Ein einzelnes Wort einer mehrwoertrigen Anfrage darf nicht 100 ergeben.
|
|
assert score_window("leider zu spät", "leider").score < 70
|
|
assert score_window("der schlüssel liegt im alten haus", "haus").score < 70
|