Initial commit

This commit is contained in:
2026-11-16 16:29:32 +01:00
commit 97e5c9ef4d
37 changed files with 5932 additions and 0 deletions
+168
View File
@@ -0,0 +1,168 @@
"""Gemeinsame Fixtures fuer die SjAudioSearch-Tests.
Die Tests erzeugen winzige echte WAV-Dateien und verwenden eine gefakte
Transkriptions-Engine. Es wird nie ein Whisper-Modell heruntergeladen.
"""
from __future__ import annotations
import os
import wave
from pathlib import Path
from typing import Callable, Iterable, Sequence
import pytest
from sjaudiosearch.config import ProcessingConfig, SearchConfig
from sjaudiosearch.core.models import AudioFileRecord, ScanCandidate
from sjaudiosearch.database.repository import Repository
from sjaudiosearch.transcription.models import TranscriptResult, TranscriptSegment, WordToken
def write_wav(path: Path, seconds: float = 1.0, sample_rate: int = 8000) -> Path:
"""Schreibt eine stille WAV-Datei (gueltiges Audio, sehr klein)."""
path.parent.mkdir(parents=True, exist_ok=True)
frames = int(sample_rate * seconds)
with wave.open(str(path), "wb") as handle:
handle.setnchannels(1)
handle.setsampwidth(2)
handle.setframerate(sample_rate)
handle.writeframes(b"\x00\x00" * frames)
return path
@pytest.fixture
def repo(tmp_path: Path) -> Iterable[Repository]:
repository = Repository.open(tmp_path / "sjaudiosearch.db")
try:
yield repository
finally:
repository.close()
@pytest.fixture
def processing() -> ProcessingConfig:
return ProcessingConfig(compute_hash="auto")
@pytest.fixture
def search_config() -> SearchConfig:
return SearchConfig(
default_limit=50,
default_min_score=70.0,
context_words=50,
exhaustive_fallback=True,
)
@pytest.fixture
def make_wav(tmp_path: Path) -> Callable[..., Path]:
def factory(name: str, seconds: float = 1.0, sample_rate: int = 8000) -> Path:
path = tmp_path / name
return write_wav(path, seconds=seconds, sample_rate=sample_rate)
return factory
@pytest.fixture
def add_audio(repo: Repository) -> Callable[..., AudioFileRecord]:
def factory(
path: str | Path,
*,
duration: float | None = 60.0,
status: str = "pending",
content_hash: str | None = None,
duplicate_of: int | None = None,
) -> AudioFileRecord:
file_path = Path(path)
if not file_path.exists():
file_path.parent.mkdir(parents=True, exist_ok=True)
file_path.write_bytes(b"\x00")
candidate = ScanCandidate(
path=str(file_path),
filename=file_path.name,
file_size=file_path.stat().st_size,
modified_at=file_path.stat().st_mtime,
duration=duration,
content_hash=content_hash,
)
audio_id = repo.insert_audio(
candidate, status=status, duplicate_of=duplicate_of
)
record = repo.get_audio(audio_id)
assert record is not None
return record
return factory
@pytest.fixture
def store_transcript(repo: Repository) -> Callable[..., int]:
"""Speichert ein Transkript mit optionalen Wort-Zeitstempeln."""
def factory(
audio_id: int,
segments: Sequence[str | tuple[str, float, float]],
*,
language: str = "de",
model: str = "test-model",
with_words: bool = False,
duration: float = 120.0,
) -> int:
built: list[TranscriptSegment] = []
cursor = 0.0
for item in segments:
if isinstance(item, str):
text = item
start = cursor
end = cursor + 5.0
else:
text, start, end = item
cursor = end
words: list[WordToken] = []
if with_words:
tokens = text.split()
step = (end - start) / max(1, len(tokens))
words = [
WordToken(
word=token,
start=start + index * step,
end=start + (index + 1) * step,
)
for index, token in enumerate(tokens)
]
built.append(
TranscriptSegment(
text=text,
start=float(start),
end=float(end),
confidence=0.9,
words=words,
)
)
full_text = " ".join(segment.text for segment in built)
result = TranscriptResult(
segments=built,
full_text=full_text,
language=language,
language_probability=0.99,
duration=duration,
model=model,
)
return repo.save_transcript(
audio_id,
language=result.language,
model=result.model,
full_text=result.full_text,
segments=result.segments,
duration=result.duration,
)
return factory
def set_mtime(path: Path, seconds: float) -> None:
os.utime(path, (seconds, seconds))
+101
View File
@@ -0,0 +1,101 @@
"""Tests fuer Timeline, Kontextfenster und Zeitstempel."""
from __future__ import annotations
import pytest
from sjaudiosearch.search.context import build_context, build_timeline
def _segments(*specs):
return [
{
"id": index + 1,
"start_time": start,
"end_time": end,
"text": text,
"confidence": 0.9,
}
for index, (text, start, end) in enumerate(specs)
]
def test_context_contains_fifty_words_before_and_after():
prefix = " ".join(f"wort{i}" for i in range(80))
suffix = " ".join(f"danach{i}" for i in range(80))
text = f"{prefix} der schlüssel liegt im alten haus {suffix}"
timeline = build_timeline(1, _segments((text, 0.0, 60.0)))
start_index = len(prefix.split())
context = build_context(timeline, start_index, start_index + 6, context_words=50)
assert len(context.before.split()) == 50
assert len(context.after.split()) == 50
assert context.before.split()[-1] == "wort79"
assert context.after.split()[0] == "danach0"
assert context.match.split()[0] == "der"
assert "haus" in context.match
def test_context_spans_segment_boundaries_without_loss():
first = "Der Professor betrat den Raum und sah sich suchend um"
second = "der Schlüssel lag im alten Haus auf dem Tisch"
timeline = build_timeline(1, _segments((first, 5.0, 10.0), (second, 10.0, 15.0)))
assert len(timeline.tokens) == len(first.split()) + len(second.split())
start_index = len(first.split())
context = build_context(timeline, start_index, start_index + 4, context_words=50)
assert context.before.endswith("suchend um")
assert context.match.startswith("der Schlüssel")
assert context.start_time == pytest.approx(10.0, abs=1.5)
def test_timestamps_use_word_precision_when_available():
texts = ["Eins zwei drei vier fünf"]
segments = _segments((texts[0], 0.0, 5.0))
words = {
1: [
{"word": "Eins", "start_time": 10.0, "end_time": 10.5},
{"word": "zwei", "start_time": 10.5, "end_time": 11.0},
{"word": "drei", "start_time": 11.0, "end_time": 11.5},
{"word": "vier", "start_time": 11.5, "end_time": 12.0},
{"word": "fünf", "start_time": 12.0, "end_time": 12.5},
]
}
timeline = build_timeline(1, segments, words)
context = build_context(timeline, 2, 4, context_words=10)
assert context.precision == "word"
assert context.start_time == pytest.approx(11.0)
assert context.end_time == pytest.approx(12.0)
assert context.match == "drei vier"
def test_timestamps_fall_back_to_segment_precision():
timeline = build_timeline(1, _segments(("Eins zwei drei vier", 30.0, 34.0)))
context = build_context(timeline, 0, 2, context_words=5)
assert context.precision == "segment"
assert 30.0 <= context.start_time <= 32.0
assert context.end_time <= 34.0
def test_context_block_formatting():
timeline = build_timeline(1, _segments(("vorher der Schlüssel nachher", 0.0, 5.0)))
context = build_context(timeline, 1, 2, context_words=10)
block = context.as_block()
assert block.startswith('"\u2026vorher')
assert "der Schlüssel" in block
assert block.endswith('nachher\u2026"')
alone = build_context(timeline, 0, 4, context_words=0)
assert alone.as_block() == '"vorher der Schlüssel nachher"'
def test_empty_timeline_is_safe():
timeline = build_timeline(1, [])
context = build_context(timeline, 0, 1, 50)
assert context.match == ""
assert context.start_time == 0.0
+167
View File
@@ -0,0 +1,167 @@
"""Tests fuer Datenbank, Repository, Migrationen und Suchindex."""
from __future__ import annotations
from pathlib import Path
from sjaudiosearch.database.migrations import current_version, migrate
from sjaudiosearch.database.repository import Repository
def test_schema_and_migration_version(repo):
assert current_version(repo.conn) == 1
# Migration ist idempotent.
assert migrate(repo.conn) == 1
tables = {
row["name"]
for row in repo.conn.execute(
"SELECT name FROM sqlite_master WHERE type IN ('table', 'view')"
)
}
for table in (
"audio_files",
"transcripts",
"segments",
"words",
"search_index",
"search_trigram",
):
assert table in tables
def test_transcript_stored_with_segments_and_words(repo, add_audio, store_transcript):
record = add_audio("/tmp/folge1.mp3", duration=125.0)
store_transcript(
record.id,
[("Der Professor sah sich um.", 0.0, 4.0), ("Der Schlüssel liegt im alten Haus.", 4.0, 9.5)],
with_words=True,
duration=9.5,
)
stored = repo.get_transcript(record.id)
assert stored is not None
assert stored["model"] == "test-model"
assert stored["language"] == "de"
assert "Schlüssel" in stored["full_text"]
segments = repo.get_segments(record.id)
assert len(segments) == 2
assert segments[1]["start_time"] == 4.0
words = repo.get_words([int(segment["id"]) for segment in segments])
first_segment_words = words[int(segments[0]["id"])]
assert [row["word"] for row in first_segment_words][:2] == ["Der", "Professor"]
updated = repo.get_audio(record.id)
assert updated.status == "done"
assert updated.duration == 125.0
def test_status_management_and_retry(repo, add_audio):
ok = add_audio("/tmp/a.mp3")
failing = add_audio("/tmp/b.mp3")
repo.set_status(ok.id, "processing")
repo.set_status(failing.id, "error", "Decoder kaputt")
counts = repo.count_by_status()
assert counts["processing"] == 1
assert counts["error"] == 1
# Abgestuerzte 'processing'-Eintraege werden zurueckgesetzt
assert repo.reset_stale_processing() == 1
assert repo.get_audio(ok.id).status == "pending"
assert repo.reset_errors() == 1
reset = repo.get_audio(failing.id)
assert reset.status == "pending"
assert reset.error_message is None
def test_queue_skips_duplicates_and_done(repo, add_audio):
done = add_audio("/tmp/done.mp3", status="done")
pending = add_audio("/tmp/pending.mp3")
duplicate = add_audio("/tmp/dup.mp3", duplicate_of=done.id)
queue = repo.transcription_queue()
assert [record.id for record in queue] == [pending.id]
forced = repo.transcription_queue(force=True)
assert {record.id for record in forced} == {done.id, pending.id}
assert duplicate.id not in {record.id for record in forced}
def test_delete_audio_keeps_file_and_removes_index(repo, add_audio, store_transcript, tmp_path):
audio_file = tmp_path / "folge.mp3"
audio_file.write_bytes(b"audio")
record = add_audio(audio_file)
store_transcript(record.id, ["Der Schlüssel liegt im alten Haus."])
assert repo.fts_candidates('"schlüssel"', 10)
assert repo.delete_audio(record.id) is True
assert audio_file.exists()
assert repo.get_audio(record.id) is None
assert repo.get_transcript(record.id) is None
assert repo.fts_candidates('"schlüssel"', 10) == []
def test_foreign_key_cascade(repo, add_audio, store_transcript):
record = add_audio("/tmp/a.mp3")
store_transcript(record.id, ["Erster Satz.", "Zweiter Satz."])
assert repo.get_segments(record.id)
with repo.conn:
repo.conn.execute("DELETE FROM audio_files WHERE id = ?", (record.id,))
segments = repo.conn.execute("SELECT COUNT(*) AS amount FROM segments").fetchone()
words = repo.conn.execute("SELECT COUNT(*) AS amount FROM words").fetchone()
transcripts = repo.conn.execute("SELECT COUNT(*) AS amount FROM transcripts").fetchone()
assert segments["amount"] == 0
assert words["amount"] == 0
assert transcripts["amount"] == 0
def test_reindex_preserves_transcripts(repo, add_audio, store_transcript):
record = add_audio("/tmp/a.mp3")
store_transcript(record.id, ["Der Schlüssel liegt im alten Haus.", "Niemand bemerkte es."])
before_transcript = dict(repo.get_transcript(record.id))
before_segments = [dict(row) for row in repo.get_segments(record.id)]
assert repo.fts_candidates('"schlüssel"', 10)
indexed = repo.rebuild_search_index()
assert indexed == 2
after_transcript = dict(repo.get_transcript(record.id))
after_segments = [dict(row) for row in repo.get_segments(record.id)]
assert before_transcript == after_transcript
assert before_segments == after_segments
assert repo.fts_candidates('"schlüssel"', 10)
def test_saving_new_transcript_replaces_old(repo, add_audio, store_transcript):
record = add_audio("/tmp/a.mp3")
store_transcript(record.id, ["Alter Text mit Schlüssel."])
store_transcript(record.id, ["Neuer Text ohne alles."])
transcript = repo.get_transcript(record.id)
assert transcript["full_text"] == "Neuer Text ohne alles."
assert len(repo.get_segments(record.id)) == 1
assert repo.fts_candidates('"schlüssel"', 10) == []
def test_database_size_and_stats(repo, add_audio, store_transcript):
first = add_audio("/tmp/a.mp3", duration=100.0)
second = add_audio("/tmp/b.mp3", duration=50.0, status="error")
store_transcript(first.id, ["Ein Satz."], duration=100.0)
stats = repo.stats(repo.databases_size())
assert stats.total_files == 2
assert stats.done == 1
assert stats.error == 1
assert stats.total_duration == 150.0
assert stats.transcripts == 1
assert stats.segment_count == 1
assert stats.database_size > 0
assert "test-model" in stats.models
+131
View File
@@ -0,0 +1,131 @@
"""Tests fuer den rekursiven Audio-Scanner."""
from __future__ import annotations
from pathlib import Path
import pytest
from sjaudiosearch.core.metadata import AudioMetadataError, read_metadata
from sjaudiosearch.core.scanner import scan
from conftest import set_mtime
def test_scan_finds_multiple_audio_files(repo, make_wav, tmp_path, processing):
music = tmp_path / "music"
make_wav("music/a.mp3.wav", 1.0)
make_wav("music/sub/b.wav", 1.0)
make_wav("music/sub/deep/c.wav", 1.0)
(music / "notes.txt").write_text("kein Audio", encoding="utf-8")
result = scan(music, repo, processing)
assert result.files_found == 3
assert result.added == 3
assert result.changed == 0
assert result.errors == []
records = repo.list_audio()
assert len(records) == 3
assert {record.filename for record in records} == {"a.mp3.wav", "b.wav", "c.wav"}
for record in records:
assert record.status == "pending"
assert Path(record.path).is_absolute()
assert record.duration == pytest.approx(1.0, abs=0.05)
assert record.file_size > 0
def test_scan_detects_duplicate_content(repo, make_wav, tmp_path, processing):
root = tmp_path / "music"
original = make_wav("music/original.wav", 2.0)
copy = root / "kopie.wav"
copy.parent.mkdir(parents=True, exist_ok=True)
copy.write_bytes(original.read_bytes())
result = scan(root, repo, processing)
assert result.added == 2
assert result.duplicates == 1
records = {record.filename: record for record in repo.list_audio()}
first, second = records["kopie.wav"], records["original.wav"]
# Der Scan ist alphabetisch sortiert, daher ist "kopie.wav" zuerst dran und
# "original.wav" wird als Duplikat markiert. Genau einer der beiden muss
# auf den anderen zeigen.
assert bool(first.duplicate_of) ^ bool(second.duplicate_of)
duplicate, keeper = (first, second) if first.duplicate_of else (second, first)
assert duplicate.duplicate_of == keeper.id
assert duplicate.content_hash == keeper.content_hash
# Duplikate werden nicht erneut transkribiert.
queue = [record.filename for record in repo.transcription_queue()]
assert queue == [keeper.filename]
def test_scan_detects_changed_file(repo, make_wav, tmp_path, processing):
root = tmp_path / "music"
root.mkdir()
first = scan(root, repo, processing)
assert first.added == 0
path = make_wav("music/folge1.wav", 1.0)
assert scan(root, repo, processing).added == 1
record = repo.get_audio_by_path(str(path.resolve()))
assert record is not None and record.status == "pending"
# Inhalt aendern (andere Laenge) und Aenderungszeit setzen
make_wav("music/folge1.wav", 3.0)
set_mtime(path, path.stat().st_mtime + 30)
result = scan(root, repo, processing)
assert result.added == 0
assert result.changed == 1
assert result.unchanged == 0
updated = repo.get_audio(record.id)
assert updated is not None
assert updated.status == "pending"
assert updated.duration == pytest.approx(3.0, abs=0.05)
def test_scan_keeps_unchanged_files(repo, make_wav, tmp_path, processing):
root = tmp_path / "music"
make_wav("music/a.wav", 1.0)
make_wav("music/b.wav", 1.0)
first = scan(root, repo, processing)
assert first.added == 2
second = scan(root, repo, processing)
assert second.added == 0
assert second.changed == 0
assert second.unchanged == 2
def test_scan_logs_corrupt_files(repo, tmp_path, processing):
root = tmp_path / "music"
root.mkdir()
broken = root / "kaputt.mp3"
broken.write_bytes(b"das ist kein audio")
result = scan(root, repo, processing)
assert result.files_found == 1
assert result.added == 1
assert len(result.errors) == 1
assert "kaputt.mp3" in result.errors[0].path
record = repo.get_audio_by_path(str(broken.resolve()))
assert record is not None
assert record.status == "error"
assert record.error_message
def test_scan_rejects_missing_path(repo, tmp_path, processing):
with pytest.raises(FileNotFoundError):
scan(tmp_path / "gibt-es-nicht", repo, processing)
def test_metadata_requires_real_audio(tmp_path):
fake = tmp_path / "fake.flac"
fake.write_bytes(b"nope")
with pytest.raises(AudioMetadataError):
read_metadata(fake)
+192
View File
@@ -0,0 +1,192 @@
"""Tests fuer die Suchmaschine (Fuzzy, Sliding Window, Fallback)."""
from __future__ import annotations
import pytest
from sjaudiosearch.search.engine import SearchEngine, SearchError
from sjaudiosearch.search.fuzzy import build_fts_query, normalize_text, score_window
def _engine(repo, search_config) -> SearchEngine:
return SearchEngine(repo, search_config)
def test_fuzzy_search_finds_slightly_different_sentence(repo, add_audio, store_transcript, search_config):
record = add_audio("/tmp/hoerspiel_042.mp3", duration=3000.0)
store_transcript(
record.id,
[("Der Professor sah sich um. Der Schlüssel befindet sich in dem alten Haus. "
"Niemand hatte das bisher bemerkt.", 42 * 60 + 10.0, 42 * 60 + 20.0)],
duration=3000.0,
)
hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus")
assert len(hits) == 1
hit = hits[0]
assert hit.filename == "hoerspiel_042.mp3"
assert hit.path == record.path
assert hit.score >= 85
assert 42 * 60 <= hit.start_time <= 42 * 60 + 40
assert "Schlüssel" in hit.matched_text
assert hit.context.before
def test_exact_phrase_scores_100(repo, add_audio, store_transcript, search_config):
record = add_audio("/tmp/exakt.mp3")
store_transcript(record.id, ["Der Schlüssel liegt im alten Haus."])
hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus")
assert hits
assert hits[0].score == pytest.approx(100.0, abs=0.1)
def test_search_across_segment_boundaries(repo, add_audio, store_transcript, search_config):
record = add_audio("/tmp/krimi.mp3")
store_transcript(
record.id,
[
("Der Professor sah sich um. Der Schlüssel", 10.0, 15.0),
("liegt in dem alten Haus. Niemand bemerkte es.", 15.0, 20.0),
],
)
hits = _engine(repo, search_config).search("Der Schlüssel liegt in dem alten Haus")
assert hits
hit = hits[0]
assert hit.score >= 90
assert "Schlüssel" in hit.matched_text
assert "Haus" in hit.matched_text
# Der Treffer beginnt beim Wort "Schlüssel" innerhalb des ersten Segments.
assert 10.0 <= hit.start_time <= 16.0
def test_fallback_finds_word_fts_cannot_match(repo, add_audio, store_transcript, search_config):
"""Whisper-Transkripte koennen Woerter komplett falsch schreiben.
Ohne FTS-Treffer muss der Fallback-Lauf die Datei trotzdem durchsuchen.
"""
record = add_audio("/tmp/misheard.mp3")
store_transcript(record.id, [("Die Sage vom Schluessel im alten Gemäuer.", 0.0, 6.0)])
# Keine exakte Uebereinstimmung fuer den Suchbegriff im FTS-Index
assert repo.fts_candidates(build_fts_query(["schlüssel"]), 10) == []
assert repo.trigram_candidates(["schlüssel"], 10) == []
hits = _engine(repo, search_config).search("Schlüssel")
assert len(hits) == 1
assert hits[0].audio_id == record.id
assert hits[0].source == "fallback"
assert hits[0].score >= 70
def test_search_returns_nothing_for_unrelated_text(repo, add_audio, store_transcript, search_config):
record = add_audio("/tmp/unrelated.mp3")
store_transcript(record.id, ["Es war einmal ein kleiner Drache im Wald."])
hits = _engine(repo, search_config).search("Quantenphysik Vorlesung")
assert hits == []
def test_min_score_and_limit(repo, add_audio, store_transcript, search_config):
for index in range(5):
record = add_audio(f"/tmp/folge{index}.mp3")
store_transcript(
record.id,
[f"Der Schlüssel liegt im alten Haus Nummer {index}.", ],
)
engine = _engine(repo, search_config)
all_hits = engine.search("Der Schlüssel liegt im alten Haus", limit=10, min_score=50)
assert len(all_hits) >= 1
limited = engine.search("Der Schlüssel liegt im alten Haus", limit=2, min_score=50)
assert len(limited) == 2
strict = engine.search("Der Schlüssel liegt im alten Haus", limit=10, min_score=99.5)
assert len(strict) <= len(all_hits)
def test_results_sorted_by_score(repo, add_audio, store_transcript, search_config):
exact = add_audio("/tmp/exact.mp3")
store_transcript(exact.id, ["Der Schlüssel liegt im alten Haus."])
fuzzy = add_audio("/tmp/fuzzy.mp3")
store_transcript(fuzzy.id, ["Der Schlüssel ist wohl im alten Haus zu finden, oder?"])
hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus", min_score=60)
scores = [hit.score for hit in hits]
assert scores == sorted(scores, reverse=True)
assert hits[0].filename == "exact.mp3"
def test_language_filter(repo, add_audio, store_transcript, search_config):
german = add_audio("/tmp/de.mp3")
store_transcript(german.id, ["Der Schlüssel liegt im alten Haus."], language="de")
english = add_audio("/tmp/en.mp3")
store_transcript(english.id, ["The key is in the old house."], language="en")
hits = _engine(repo, search_config).search(
"The key is in the old house", language="en", min_score=60
)
assert [hit.audio_id for hit in hits] == [english.id]
german_hits = _engine(repo, search_config).search(
"Der Schlüssel liegt im alten Haus", language="de", min_score=60
)
assert [hit.audio_id for hit in german_hits] == [german.id]
def test_duplicate_files_are_not_searched(repo, add_audio, store_transcript, search_config):
original = add_audio("/tmp/original.mp3")
store_transcript(original.id, ["Der Schlüssel liegt im alten Haus."])
duplicate = add_audio("/tmp/kopie.mp3", duplicate_of=original.id)
hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus")
assert {hit.audio_id for hit in hits} == {original.id}
def test_empty_query_is_rejected(repo, search_config):
with pytest.raises(SearchError):
_engine(repo, search_config).search(" ... ")
def test_word_insertions_and_deletions(repo, add_audio, store_transcript, search_config):
record = add_audio("/tmp/insert.mp3")
store_transcript(
record.id,
["Also, der Schlüssel liegt wirklich direkt im alten Haus, nicht wahr?"],
)
hits = _engine(repo, search_config).search("Schlüssel liegt im alten Haus")
assert hits
assert hits[0].score >= 80
def test_multiword_query_matches_whole_phrase_not_single_word(repo, add_audio, store_transcript, search_config):
"""Regression: token_set_ratio darf ein Einzelwort nicht als 100% werten."""
record = add_audio("/tmp/kurz.mp3")
store_transcript(record.id, [("Leider zu spät.", 0.0, 2.0)])
hits = _engine(repo, search_config).search("Leider zu spät", limit=5, min_score=70)
assert hits
assert hits[0].context.match == "Leider zu spät."
assert hits[0].score == pytest.approx(100.0, abs=0.1)
def test_normalization_and_scoring_helpers():
assert normalize_text("Der Schlüssel, liegt im alten Haus!") == "der schlüssel liegt im alten haus"
assert score_window("der schlüssel liegt im alten haus", "der schlüssel liegt im alten haus").score == 100.0
assert score_window("der schlüssel liegt im alten haus", "").score == 0.0
assert build_fts_query(["haus"]).startswith('"haus"')
def test_partial_window_is_not_a_perfect_match():
# Ein einzelnes Wort einer mehrwoertrigen Anfrage darf nicht 100 ergeben.
assert score_window("leider zu spät", "leider").score < 70
assert score_window("der schlüssel liegt im alten haus", "haus").score < 70
+247
View File
@@ -0,0 +1,247 @@
"""Tests fuer Transkriptionswarteschlange, Wiederaufnahme und GPU-Diagnose."""
from __future__ import annotations
from pathlib import Path
import pytest
from sjaudiosearch.config import ProcessingConfig
from sjaudiosearch.core.models import FileStatus
from sjaudiosearch.system.gpu import (
GpuInfo,
RuntimeInfo,
resolve_execution,
)
from sjaudiosearch.transcription.models import TranscriptResult, TranscriptSegment, WordToken
from sjaudiosearch.transcription.worker import TranscriptionWorker
class FakeEngine:
"""Ersetzt faster-whisper; laedt nie ein Modell und braucht kein Netzwerk."""
model_name = "fake-model"
def __init__(self, fail_for: set[str] | None = None):
self.fail_for = fail_for or set()
self.calls: list[str] = []
def transcribe(self, path: str | Path) -> TranscriptResult:
name = Path(path).name
self.calls.append(name)
if name in self.fail_for:
raise RuntimeError("Simulierter Decoderfehler")
words = [
WordToken(word="Der", start=0.0, end=0.4),
WordToken(word="Schlüssel", start=0.4, end=1.0),
WordToken(word="liegt", start=1.0, end=1.4),
WordToken(word="im", start=1.4, end=1.6),
WordToken(word="alten", start=1.6, end=2.0),
WordToken(word="Haus.", start=2.0, end=2.6),
]
segment = TranscriptSegment(
text="Der Schlüssel liegt im alten Haus.",
start=0.0,
end=2.6,
confidence=0.95,
words=words,
)
return TranscriptResult(
segments=[segment],
full_text=segment.text,
language="de",
language_probability=0.99,
duration=10.0,
model=self.model_name,
)
def test_worker_transcribes_and_indexes(repo, add_audio, make_wav, search_config):
record = add_audio(make_wav("folge1.wav"))
engine = FakeEngine()
worker = TranscriptionWorker(repo, engine, ProcessingConfig())
stats = worker.run()
assert stats.queued == 1
assert stats.processed == 1
assert stats.failed == 0
assert engine.calls == ["folge1.wav"]
updated = repo.get_audio(record.id)
assert updated.status == FileStatus.DONE.value
transcript = repo.get_transcript(record.id)
assert transcript is not None
assert transcript["model"] == "fake-model"
from sjaudiosearch.search.engine import SearchEngine
hits = SearchEngine(repo, search_config).search("Schlüssel liegt im alten Haus")
assert hits and hits[0].audio_id == record.id
def test_worker_isolates_errors(repo, add_audio, make_wav):
good = add_audio(make_wav("good.wav"))
bad = add_audio(make_wav("bad.wav"))
engine = FakeEngine(fail_for={"bad.wav"})
worker = TranscriptionWorker(repo, engine, ProcessingConfig())
stats = worker.run()
assert stats.queued == 2
assert stats.processed == 1
assert stats.failed == 1
assert stats.errors[0][0].endswith("bad.wav")
assert repo.get_audio(good.id).status == "done"
failed = repo.get_audio(bad.id)
assert failed.status == "error"
assert "Simulierter Decoderfehler" in failed.error_message
def test_worker_skips_done_and_resumes(repo, add_audio, make_wav):
record = add_audio(make_wav("done.wav"), status="done")
engine = FakeEngine()
worker = TranscriptionWorker(repo, engine, ProcessingConfig())
stats = worker.run()
assert stats.queued == 0
assert engine.calls == []
stats_forced = worker.run(force=True)
assert stats_forced.queued == 1
assert engine.calls == ["done.wav"]
assert repo.get_audio(record.id).status == "done"
def test_worker_resets_stale_processing(repo, add_audio, make_wav):
record = add_audio(make_wav("haengt.wav"), status="processing")
worker = TranscriptionWorker(repo, FakeEngine(), ProcessingConfig(resume=True))
stats = worker.run()
assert stats.processed == 1
assert repo.get_audio(record.id).status == "done"
def test_worker_limits_queue(repo, add_audio, make_wav):
for index in range(3):
add_audio(make_wav(f"folge{index}.wav"))
engine = FakeEngine()
worker = TranscriptionWorker(repo, engine, ProcessingConfig())
stats = worker.run(limit=2)
assert stats.queued == 2
assert len(engine.calls) == 2
def test_failed_transcription_can_be_retried(repo, add_audio, make_wav):
record = add_audio(make_wav("retry.wav"))
failing = TranscriptionWorker(repo, FakeEngine(fail_for={"retry.wav"}), ProcessingConfig())
failing.run()
assert repo.get_audio(record.id).status == "error"
assert repo.reset_errors() == 1
retrying = TranscriptionWorker(repo, FakeEngine(), ProcessingConfig())
stats = retrying.run()
assert stats.processed == 1
assert repo.get_audio(record.id).status == "done"
def test_worker_marks_missing_file(repo, add_audio):
record = add_audio("/tmp/gibt-es-nicht-mehr.mp3")
Path(record.path).unlink(missing_ok=True)
stats = TranscriptionWorker(repo, FakeEngine(), ProcessingConfig()).run()
assert stats.failed == 1
assert repo.get_audio(record.id).status == "error"
def test_worker_events_are_emitted(repo, add_audio, make_wav):
add_audio(make_wav("event.wav"))
events = []
TranscriptionWorker(repo, FakeEngine(), ProcessingConfig()).run(event_callback=events.append)
kinds = [event.kind for event in events]
assert kinds[0] == "start"
assert "done" in kinds
assert kinds[-1] == "finished"
# ----------------------------------------------------------------------
# GPU-/Ausfuehrungsdiagnose (reine Funktionen, keine echte GPU noetig)
# ----------------------------------------------------------------------
def _pascal_gpu() -> GpuInfo:
return GpuInfo(
name="NVIDIA GeForce GTX 1070 Ti",
driver_version="580.196.07",
cuda_driver_version="13.0",
memory_total_mb=8192,
compute_capability="6.1",
)
def _runtime(**overrides) -> RuntimeInfo:
runtime = RuntimeInfo(
ctranslate2_version="4.9.0",
cuda_device_count=1,
supported_compute_types={
"cpu": ["float32", "int16", "int8", "int8_float32"],
"cuda": ["float32", "int8", "int8_float32"],
},
cuda_libraries=["/usr/lib/libcudart.so.12"],
cudnn_libraries=["/usr/lib/libcudnn.so.9"],
python_version="3.13.15",
platform="Linux",
sqlite_version="3.50.0",
fts5_available=True,
trigram_available=True,
)
for key, value in overrides.items():
setattr(runtime, key, value)
return runtime
def test_pascal_uses_int8_float32():
resolved = resolve_execution("cuda", "int8_float32", _pascal_gpu(), _runtime())
assert resolved.device == "cuda"
assert resolved.compute_type == "int8_float32"
assert not resolved.used_fallback
def test_diagnostics_rejects_float16_on_incompatible_gpu():
resolved = resolve_execution("cuda", "float16", _pascal_gpu(), _runtime())
assert resolved.compute_type != "float16"
assert resolved.compute_type == "int8_float32"
assert any("Pascal" in note for note in resolved.notes)
def test_diagnostics_falls_back_when_int8_float32_unsupported():
runtime = _runtime(
supported_compute_types={"cpu": ["float32", "int8"], "cuda": ["float32"]}
)
resolved = resolve_execution("cuda", "int8_float32", _pascal_gpu(), runtime)
assert resolved.compute_type == "float32"
assert resolved.device == "cuda"
def test_cpu_fallback_when_cuda_unavailable():
runtime = _runtime(
cuda_device_count=0,
supported_compute_types={"cpu": ["float32", "int8"], "cuda": []},
)
resolved = resolve_execution("cuda", "int8_float32", None, runtime)
assert resolved.device == "cpu"
assert resolved.used_fallback
assert resolved.compute_type in {"int8", "int8_float32", "float32"}
assert any("CPU" in note for note in resolved.notes)
def test_cpu_mode_never_returns_gpu_compute_type():
resolved = resolve_execution("cpu", "float16", _pascal_gpu(), _runtime())
assert resolved.device == "cpu"
assert resolved.compute_type not in {"float16", "bfloat16"}