Initial commit
This commit is contained in:
@@ -0,0 +1,168 @@
|
||||
"""Gemeinsame Fixtures fuer die SjAudioSearch-Tests.
|
||||
|
||||
Die Tests erzeugen winzige echte WAV-Dateien und verwenden eine gefakte
|
||||
Transkriptions-Engine. Es wird nie ein Whisper-Modell heruntergeladen.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import wave
|
||||
from pathlib import Path
|
||||
from typing import Callable, Iterable, Sequence
|
||||
|
||||
import pytest
|
||||
|
||||
from sjaudiosearch.config import ProcessingConfig, SearchConfig
|
||||
from sjaudiosearch.core.models import AudioFileRecord, ScanCandidate
|
||||
from sjaudiosearch.database.repository import Repository
|
||||
from sjaudiosearch.transcription.models import TranscriptResult, TranscriptSegment, WordToken
|
||||
|
||||
|
||||
def write_wav(path: Path, seconds: float = 1.0, sample_rate: int = 8000) -> Path:
|
||||
"""Schreibt eine stille WAV-Datei (gueltiges Audio, sehr klein)."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
frames = int(sample_rate * seconds)
|
||||
with wave.open(str(path), "wb") as handle:
|
||||
handle.setnchannels(1)
|
||||
handle.setsampwidth(2)
|
||||
handle.setframerate(sample_rate)
|
||||
handle.writeframes(b"\x00\x00" * frames)
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def repo(tmp_path: Path) -> Iterable[Repository]:
|
||||
repository = Repository.open(tmp_path / "sjaudiosearch.db")
|
||||
try:
|
||||
yield repository
|
||||
finally:
|
||||
repository.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def processing() -> ProcessingConfig:
|
||||
return ProcessingConfig(compute_hash="auto")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def search_config() -> SearchConfig:
|
||||
return SearchConfig(
|
||||
default_limit=50,
|
||||
default_min_score=70.0,
|
||||
context_words=50,
|
||||
exhaustive_fallback=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_wav(tmp_path: Path) -> Callable[..., Path]:
|
||||
def factory(name: str, seconds: float = 1.0, sample_rate: int = 8000) -> Path:
|
||||
path = tmp_path / name
|
||||
return write_wav(path, seconds=seconds, sample_rate=sample_rate)
|
||||
|
||||
return factory
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def add_audio(repo: Repository) -> Callable[..., AudioFileRecord]:
|
||||
def factory(
|
||||
path: str | Path,
|
||||
*,
|
||||
duration: float | None = 60.0,
|
||||
status: str = "pending",
|
||||
content_hash: str | None = None,
|
||||
duplicate_of: int | None = None,
|
||||
) -> AudioFileRecord:
|
||||
file_path = Path(path)
|
||||
if not file_path.exists():
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_path.write_bytes(b"\x00")
|
||||
candidate = ScanCandidate(
|
||||
path=str(file_path),
|
||||
filename=file_path.name,
|
||||
file_size=file_path.stat().st_size,
|
||||
modified_at=file_path.stat().st_mtime,
|
||||
duration=duration,
|
||||
content_hash=content_hash,
|
||||
)
|
||||
audio_id = repo.insert_audio(
|
||||
candidate, status=status, duplicate_of=duplicate_of
|
||||
)
|
||||
record = repo.get_audio(audio_id)
|
||||
assert record is not None
|
||||
return record
|
||||
|
||||
return factory
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store_transcript(repo: Repository) -> Callable[..., int]:
|
||||
"""Speichert ein Transkript mit optionalen Wort-Zeitstempeln."""
|
||||
|
||||
def factory(
|
||||
audio_id: int,
|
||||
segments: Sequence[str | tuple[str, float, float]],
|
||||
*,
|
||||
language: str = "de",
|
||||
model: str = "test-model",
|
||||
with_words: bool = False,
|
||||
duration: float = 120.0,
|
||||
) -> int:
|
||||
built: list[TranscriptSegment] = []
|
||||
cursor = 0.0
|
||||
for item in segments:
|
||||
if isinstance(item, str):
|
||||
text = item
|
||||
start = cursor
|
||||
end = cursor + 5.0
|
||||
else:
|
||||
text, start, end = item
|
||||
cursor = end
|
||||
|
||||
words: list[WordToken] = []
|
||||
if with_words:
|
||||
tokens = text.split()
|
||||
step = (end - start) / max(1, len(tokens))
|
||||
words = [
|
||||
WordToken(
|
||||
word=token,
|
||||
start=start + index * step,
|
||||
end=start + (index + 1) * step,
|
||||
)
|
||||
for index, token in enumerate(tokens)
|
||||
]
|
||||
|
||||
built.append(
|
||||
TranscriptSegment(
|
||||
text=text,
|
||||
start=float(start),
|
||||
end=float(end),
|
||||
confidence=0.9,
|
||||
words=words,
|
||||
)
|
||||
)
|
||||
|
||||
full_text = " ".join(segment.text for segment in built)
|
||||
result = TranscriptResult(
|
||||
segments=built,
|
||||
full_text=full_text,
|
||||
language=language,
|
||||
language_probability=0.99,
|
||||
duration=duration,
|
||||
model=model,
|
||||
)
|
||||
return repo.save_transcript(
|
||||
audio_id,
|
||||
language=result.language,
|
||||
model=result.model,
|
||||
full_text=result.full_text,
|
||||
segments=result.segments,
|
||||
duration=result.duration,
|
||||
)
|
||||
|
||||
return factory
|
||||
|
||||
|
||||
def set_mtime(path: Path, seconds: float) -> None:
|
||||
os.utime(path, (seconds, seconds))
|
||||
@@ -0,0 +1,101 @@
|
||||
"""Tests fuer Timeline, Kontextfenster und Zeitstempel."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from sjaudiosearch.search.context import build_context, build_timeline
|
||||
|
||||
|
||||
def _segments(*specs):
|
||||
return [
|
||||
{
|
||||
"id": index + 1,
|
||||
"start_time": start,
|
||||
"end_time": end,
|
||||
"text": text,
|
||||
"confidence": 0.9,
|
||||
}
|
||||
for index, (text, start, end) in enumerate(specs)
|
||||
]
|
||||
|
||||
|
||||
def test_context_contains_fifty_words_before_and_after():
|
||||
prefix = " ".join(f"wort{i}" for i in range(80))
|
||||
suffix = " ".join(f"danach{i}" for i in range(80))
|
||||
text = f"{prefix} der schlüssel liegt im alten haus {suffix}"
|
||||
timeline = build_timeline(1, _segments((text, 0.0, 60.0)))
|
||||
|
||||
start_index = len(prefix.split())
|
||||
context = build_context(timeline, start_index, start_index + 6, context_words=50)
|
||||
|
||||
assert len(context.before.split()) == 50
|
||||
assert len(context.after.split()) == 50
|
||||
assert context.before.split()[-1] == "wort79"
|
||||
assert context.after.split()[0] == "danach0"
|
||||
assert context.match.split()[0] == "der"
|
||||
assert "haus" in context.match
|
||||
|
||||
|
||||
def test_context_spans_segment_boundaries_without_loss():
|
||||
first = "Der Professor betrat den Raum und sah sich suchend um"
|
||||
second = "der Schlüssel lag im alten Haus auf dem Tisch"
|
||||
timeline = build_timeline(1, _segments((first, 5.0, 10.0), (second, 10.0, 15.0)))
|
||||
|
||||
assert len(timeline.tokens) == len(first.split()) + len(second.split())
|
||||
start_index = len(first.split())
|
||||
context = build_context(timeline, start_index, start_index + 4, context_words=50)
|
||||
|
||||
assert context.before.endswith("suchend um")
|
||||
assert context.match.startswith("der Schlüssel")
|
||||
assert context.start_time == pytest.approx(10.0, abs=1.5)
|
||||
|
||||
|
||||
def test_timestamps_use_word_precision_when_available():
|
||||
texts = ["Eins zwei drei vier fünf"]
|
||||
segments = _segments((texts[0], 0.0, 5.0))
|
||||
words = {
|
||||
1: [
|
||||
{"word": "Eins", "start_time": 10.0, "end_time": 10.5},
|
||||
{"word": "zwei", "start_time": 10.5, "end_time": 11.0},
|
||||
{"word": "drei", "start_time": 11.0, "end_time": 11.5},
|
||||
{"word": "vier", "start_time": 11.5, "end_time": 12.0},
|
||||
{"word": "fünf", "start_time": 12.0, "end_time": 12.5},
|
||||
]
|
||||
}
|
||||
timeline = build_timeline(1, segments, words)
|
||||
context = build_context(timeline, 2, 4, context_words=10)
|
||||
|
||||
assert context.precision == "word"
|
||||
assert context.start_time == pytest.approx(11.0)
|
||||
assert context.end_time == pytest.approx(12.0)
|
||||
assert context.match == "drei vier"
|
||||
|
||||
|
||||
def test_timestamps_fall_back_to_segment_precision():
|
||||
timeline = build_timeline(1, _segments(("Eins zwei drei vier", 30.0, 34.0)))
|
||||
context = build_context(timeline, 0, 2, context_words=5)
|
||||
|
||||
assert context.precision == "segment"
|
||||
assert 30.0 <= context.start_time <= 32.0
|
||||
assert context.end_time <= 34.0
|
||||
|
||||
|
||||
def test_context_block_formatting():
|
||||
timeline = build_timeline(1, _segments(("vorher der Schlüssel nachher", 0.0, 5.0)))
|
||||
context = build_context(timeline, 1, 2, context_words=10)
|
||||
block = context.as_block()
|
||||
|
||||
assert block.startswith('"\u2026vorher')
|
||||
assert "der Schlüssel" in block
|
||||
assert block.endswith('nachher\u2026"')
|
||||
|
||||
alone = build_context(timeline, 0, 4, context_words=0)
|
||||
assert alone.as_block() == '"vorher der Schlüssel nachher"'
|
||||
|
||||
|
||||
def test_empty_timeline_is_safe():
|
||||
timeline = build_timeline(1, [])
|
||||
context = build_context(timeline, 0, 1, 50)
|
||||
assert context.match == ""
|
||||
assert context.start_time == 0.0
|
||||
@@ -0,0 +1,167 @@
|
||||
"""Tests fuer Datenbank, Repository, Migrationen und Suchindex."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from sjaudiosearch.database.migrations import current_version, migrate
|
||||
from sjaudiosearch.database.repository import Repository
|
||||
|
||||
|
||||
def test_schema_and_migration_version(repo):
|
||||
assert current_version(repo.conn) == 1
|
||||
# Migration ist idempotent.
|
||||
assert migrate(repo.conn) == 1
|
||||
tables = {
|
||||
row["name"]
|
||||
for row in repo.conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type IN ('table', 'view')"
|
||||
)
|
||||
}
|
||||
for table in (
|
||||
"audio_files",
|
||||
"transcripts",
|
||||
"segments",
|
||||
"words",
|
||||
"search_index",
|
||||
"search_trigram",
|
||||
):
|
||||
assert table in tables
|
||||
|
||||
|
||||
def test_transcript_stored_with_segments_and_words(repo, add_audio, store_transcript):
|
||||
record = add_audio("/tmp/folge1.mp3", duration=125.0)
|
||||
store_transcript(
|
||||
record.id,
|
||||
[("Der Professor sah sich um.", 0.0, 4.0), ("Der Schlüssel liegt im alten Haus.", 4.0, 9.5)],
|
||||
with_words=True,
|
||||
duration=9.5,
|
||||
)
|
||||
|
||||
stored = repo.get_transcript(record.id)
|
||||
assert stored is not None
|
||||
assert stored["model"] == "test-model"
|
||||
assert stored["language"] == "de"
|
||||
assert "Schlüssel" in stored["full_text"]
|
||||
|
||||
segments = repo.get_segments(record.id)
|
||||
assert len(segments) == 2
|
||||
assert segments[1]["start_time"] == 4.0
|
||||
|
||||
words = repo.get_words([int(segment["id"]) for segment in segments])
|
||||
first_segment_words = words[int(segments[0]["id"])]
|
||||
assert [row["word"] for row in first_segment_words][:2] == ["Der", "Professor"]
|
||||
|
||||
updated = repo.get_audio(record.id)
|
||||
assert updated.status == "done"
|
||||
assert updated.duration == 125.0
|
||||
|
||||
|
||||
def test_status_management_and_retry(repo, add_audio):
|
||||
ok = add_audio("/tmp/a.mp3")
|
||||
failing = add_audio("/tmp/b.mp3")
|
||||
|
||||
repo.set_status(ok.id, "processing")
|
||||
repo.set_status(failing.id, "error", "Decoder kaputt")
|
||||
|
||||
counts = repo.count_by_status()
|
||||
assert counts["processing"] == 1
|
||||
assert counts["error"] == 1
|
||||
|
||||
# Abgestuerzte 'processing'-Eintraege werden zurueckgesetzt
|
||||
assert repo.reset_stale_processing() == 1
|
||||
assert repo.get_audio(ok.id).status == "pending"
|
||||
|
||||
assert repo.reset_errors() == 1
|
||||
reset = repo.get_audio(failing.id)
|
||||
assert reset.status == "pending"
|
||||
assert reset.error_message is None
|
||||
|
||||
|
||||
def test_queue_skips_duplicates_and_done(repo, add_audio):
|
||||
done = add_audio("/tmp/done.mp3", status="done")
|
||||
pending = add_audio("/tmp/pending.mp3")
|
||||
duplicate = add_audio("/tmp/dup.mp3", duplicate_of=done.id)
|
||||
|
||||
queue = repo.transcription_queue()
|
||||
assert [record.id for record in queue] == [pending.id]
|
||||
|
||||
forced = repo.transcription_queue(force=True)
|
||||
assert {record.id for record in forced} == {done.id, pending.id}
|
||||
assert duplicate.id not in {record.id for record in forced}
|
||||
|
||||
|
||||
def test_delete_audio_keeps_file_and_removes_index(repo, add_audio, store_transcript, tmp_path):
|
||||
audio_file = tmp_path / "folge.mp3"
|
||||
audio_file.write_bytes(b"audio")
|
||||
record = add_audio(audio_file)
|
||||
store_transcript(record.id, ["Der Schlüssel liegt im alten Haus."])
|
||||
|
||||
assert repo.fts_candidates('"schlüssel"', 10)
|
||||
|
||||
assert repo.delete_audio(record.id) is True
|
||||
assert audio_file.exists()
|
||||
assert repo.get_audio(record.id) is None
|
||||
assert repo.get_transcript(record.id) is None
|
||||
assert repo.fts_candidates('"schlüssel"', 10) == []
|
||||
|
||||
|
||||
def test_foreign_key_cascade(repo, add_audio, store_transcript):
|
||||
record = add_audio("/tmp/a.mp3")
|
||||
store_transcript(record.id, ["Erster Satz.", "Zweiter Satz."])
|
||||
assert repo.get_segments(record.id)
|
||||
|
||||
with repo.conn:
|
||||
repo.conn.execute("DELETE FROM audio_files WHERE id = ?", (record.id,))
|
||||
|
||||
segments = repo.conn.execute("SELECT COUNT(*) AS amount FROM segments").fetchone()
|
||||
words = repo.conn.execute("SELECT COUNT(*) AS amount FROM words").fetchone()
|
||||
transcripts = repo.conn.execute("SELECT COUNT(*) AS amount FROM transcripts").fetchone()
|
||||
assert segments["amount"] == 0
|
||||
assert words["amount"] == 0
|
||||
assert transcripts["amount"] == 0
|
||||
|
||||
|
||||
def test_reindex_preserves_transcripts(repo, add_audio, store_transcript):
|
||||
record = add_audio("/tmp/a.mp3")
|
||||
store_transcript(record.id, ["Der Schlüssel liegt im alten Haus.", "Niemand bemerkte es."])
|
||||
|
||||
before_transcript = dict(repo.get_transcript(record.id))
|
||||
before_segments = [dict(row) for row in repo.get_segments(record.id)]
|
||||
assert repo.fts_candidates('"schlüssel"', 10)
|
||||
|
||||
indexed = repo.rebuild_search_index()
|
||||
assert indexed == 2
|
||||
|
||||
after_transcript = dict(repo.get_transcript(record.id))
|
||||
after_segments = [dict(row) for row in repo.get_segments(record.id)]
|
||||
assert before_transcript == after_transcript
|
||||
assert before_segments == after_segments
|
||||
assert repo.fts_candidates('"schlüssel"', 10)
|
||||
|
||||
|
||||
def test_saving_new_transcript_replaces_old(repo, add_audio, store_transcript):
|
||||
record = add_audio("/tmp/a.mp3")
|
||||
store_transcript(record.id, ["Alter Text mit Schlüssel."])
|
||||
store_transcript(record.id, ["Neuer Text ohne alles."])
|
||||
|
||||
transcript = repo.get_transcript(record.id)
|
||||
assert transcript["full_text"] == "Neuer Text ohne alles."
|
||||
assert len(repo.get_segments(record.id)) == 1
|
||||
assert repo.fts_candidates('"schlüssel"', 10) == []
|
||||
|
||||
|
||||
def test_database_size_and_stats(repo, add_audio, store_transcript):
|
||||
first = add_audio("/tmp/a.mp3", duration=100.0)
|
||||
second = add_audio("/tmp/b.mp3", duration=50.0, status="error")
|
||||
store_transcript(first.id, ["Ein Satz."], duration=100.0)
|
||||
|
||||
stats = repo.stats(repo.databases_size())
|
||||
assert stats.total_files == 2
|
||||
assert stats.done == 1
|
||||
assert stats.error == 1
|
||||
assert stats.total_duration == 150.0
|
||||
assert stats.transcripts == 1
|
||||
assert stats.segment_count == 1
|
||||
assert stats.database_size > 0
|
||||
assert "test-model" in stats.models
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Tests fuer den rekursiven Audio-Scanner."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from sjaudiosearch.core.metadata import AudioMetadataError, read_metadata
|
||||
from sjaudiosearch.core.scanner import scan
|
||||
from conftest import set_mtime
|
||||
|
||||
|
||||
def test_scan_finds_multiple_audio_files(repo, make_wav, tmp_path, processing):
|
||||
music = tmp_path / "music"
|
||||
make_wav("music/a.mp3.wav", 1.0)
|
||||
make_wav("music/sub/b.wav", 1.0)
|
||||
make_wav("music/sub/deep/c.wav", 1.0)
|
||||
(music / "notes.txt").write_text("kein Audio", encoding="utf-8")
|
||||
|
||||
result = scan(music, repo, processing)
|
||||
|
||||
assert result.files_found == 3
|
||||
assert result.added == 3
|
||||
assert result.changed == 0
|
||||
assert result.errors == []
|
||||
|
||||
records = repo.list_audio()
|
||||
assert len(records) == 3
|
||||
assert {record.filename for record in records} == {"a.mp3.wav", "b.wav", "c.wav"}
|
||||
for record in records:
|
||||
assert record.status == "pending"
|
||||
assert Path(record.path).is_absolute()
|
||||
assert record.duration == pytest.approx(1.0, abs=0.05)
|
||||
assert record.file_size > 0
|
||||
|
||||
|
||||
def test_scan_detects_duplicate_content(repo, make_wav, tmp_path, processing):
|
||||
root = tmp_path / "music"
|
||||
original = make_wav("music/original.wav", 2.0)
|
||||
copy = root / "kopie.wav"
|
||||
copy.parent.mkdir(parents=True, exist_ok=True)
|
||||
copy.write_bytes(original.read_bytes())
|
||||
|
||||
result = scan(root, repo, processing)
|
||||
|
||||
assert result.added == 2
|
||||
assert result.duplicates == 1
|
||||
|
||||
records = {record.filename: record for record in repo.list_audio()}
|
||||
first, second = records["kopie.wav"], records["original.wav"]
|
||||
# Der Scan ist alphabetisch sortiert, daher ist "kopie.wav" zuerst dran und
|
||||
# "original.wav" wird als Duplikat markiert. Genau einer der beiden muss
|
||||
# auf den anderen zeigen.
|
||||
assert bool(first.duplicate_of) ^ bool(second.duplicate_of)
|
||||
duplicate, keeper = (first, second) if first.duplicate_of else (second, first)
|
||||
assert duplicate.duplicate_of == keeper.id
|
||||
assert duplicate.content_hash == keeper.content_hash
|
||||
# Duplikate werden nicht erneut transkribiert.
|
||||
queue = [record.filename for record in repo.transcription_queue()]
|
||||
assert queue == [keeper.filename]
|
||||
|
||||
|
||||
def test_scan_detects_changed_file(repo, make_wav, tmp_path, processing):
|
||||
root = tmp_path / "music"
|
||||
root.mkdir()
|
||||
first = scan(root, repo, processing)
|
||||
assert first.added == 0
|
||||
|
||||
path = make_wav("music/folge1.wav", 1.0)
|
||||
assert scan(root, repo, processing).added == 1
|
||||
record = repo.get_audio_by_path(str(path.resolve()))
|
||||
assert record is not None and record.status == "pending"
|
||||
|
||||
# Inhalt aendern (andere Laenge) und Aenderungszeit setzen
|
||||
make_wav("music/folge1.wav", 3.0)
|
||||
set_mtime(path, path.stat().st_mtime + 30)
|
||||
|
||||
result = scan(root, repo, processing)
|
||||
assert result.added == 0
|
||||
assert result.changed == 1
|
||||
assert result.unchanged == 0
|
||||
|
||||
updated = repo.get_audio(record.id)
|
||||
assert updated is not None
|
||||
assert updated.status == "pending"
|
||||
assert updated.duration == pytest.approx(3.0, abs=0.05)
|
||||
|
||||
|
||||
def test_scan_keeps_unchanged_files(repo, make_wav, tmp_path, processing):
|
||||
root = tmp_path / "music"
|
||||
make_wav("music/a.wav", 1.0)
|
||||
make_wav("music/b.wav", 1.0)
|
||||
|
||||
first = scan(root, repo, processing)
|
||||
assert first.added == 2
|
||||
second = scan(root, repo, processing)
|
||||
assert second.added == 0
|
||||
assert second.changed == 0
|
||||
assert second.unchanged == 2
|
||||
|
||||
|
||||
def test_scan_logs_corrupt_files(repo, tmp_path, processing):
|
||||
root = tmp_path / "music"
|
||||
root.mkdir()
|
||||
broken = root / "kaputt.mp3"
|
||||
broken.write_bytes(b"das ist kein audio")
|
||||
|
||||
result = scan(root, repo, processing)
|
||||
|
||||
assert result.files_found == 1
|
||||
assert result.added == 1
|
||||
assert len(result.errors) == 1
|
||||
assert "kaputt.mp3" in result.errors[0].path
|
||||
|
||||
record = repo.get_audio_by_path(str(broken.resolve()))
|
||||
assert record is not None
|
||||
assert record.status == "error"
|
||||
assert record.error_message
|
||||
|
||||
|
||||
def test_scan_rejects_missing_path(repo, tmp_path, processing):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
scan(tmp_path / "gibt-es-nicht", repo, processing)
|
||||
|
||||
|
||||
def test_metadata_requires_real_audio(tmp_path):
|
||||
fake = tmp_path / "fake.flac"
|
||||
fake.write_bytes(b"nope")
|
||||
with pytest.raises(AudioMetadataError):
|
||||
read_metadata(fake)
|
||||
@@ -0,0 +1,192 @@
|
||||
"""Tests fuer die Suchmaschine (Fuzzy, Sliding Window, Fallback)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from sjaudiosearch.search.engine import SearchEngine, SearchError
|
||||
from sjaudiosearch.search.fuzzy import build_fts_query, normalize_text, score_window
|
||||
|
||||
|
||||
def _engine(repo, search_config) -> SearchEngine:
|
||||
return SearchEngine(repo, search_config)
|
||||
|
||||
|
||||
def test_fuzzy_search_finds_slightly_different_sentence(repo, add_audio, store_transcript, search_config):
|
||||
record = add_audio("/tmp/hoerspiel_042.mp3", duration=3000.0)
|
||||
store_transcript(
|
||||
record.id,
|
||||
[("Der Professor sah sich um. Der Schlüssel befindet sich in dem alten Haus. "
|
||||
"Niemand hatte das bisher bemerkt.", 42 * 60 + 10.0, 42 * 60 + 20.0)],
|
||||
duration=3000.0,
|
||||
)
|
||||
|
||||
hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus")
|
||||
|
||||
assert len(hits) == 1
|
||||
hit = hits[0]
|
||||
assert hit.filename == "hoerspiel_042.mp3"
|
||||
assert hit.path == record.path
|
||||
assert hit.score >= 85
|
||||
assert 42 * 60 <= hit.start_time <= 42 * 60 + 40
|
||||
assert "Schlüssel" in hit.matched_text
|
||||
assert hit.context.before
|
||||
|
||||
|
||||
def test_exact_phrase_scores_100(repo, add_audio, store_transcript, search_config):
|
||||
record = add_audio("/tmp/exakt.mp3")
|
||||
store_transcript(record.id, ["Der Schlüssel liegt im alten Haus."])
|
||||
|
||||
hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus")
|
||||
|
||||
assert hits
|
||||
assert hits[0].score == pytest.approx(100.0, abs=0.1)
|
||||
|
||||
|
||||
def test_search_across_segment_boundaries(repo, add_audio, store_transcript, search_config):
|
||||
record = add_audio("/tmp/krimi.mp3")
|
||||
store_transcript(
|
||||
record.id,
|
||||
[
|
||||
("Der Professor sah sich um. Der Schlüssel", 10.0, 15.0),
|
||||
("liegt in dem alten Haus. Niemand bemerkte es.", 15.0, 20.0),
|
||||
],
|
||||
)
|
||||
|
||||
hits = _engine(repo, search_config).search("Der Schlüssel liegt in dem alten Haus")
|
||||
|
||||
assert hits
|
||||
hit = hits[0]
|
||||
assert hit.score >= 90
|
||||
assert "Schlüssel" in hit.matched_text
|
||||
assert "Haus" in hit.matched_text
|
||||
# Der Treffer beginnt beim Wort "Schlüssel" innerhalb des ersten Segments.
|
||||
assert 10.0 <= hit.start_time <= 16.0
|
||||
|
||||
|
||||
def test_fallback_finds_word_fts_cannot_match(repo, add_audio, store_transcript, search_config):
|
||||
"""Whisper-Transkripte koennen Woerter komplett falsch schreiben.
|
||||
|
||||
Ohne FTS-Treffer muss der Fallback-Lauf die Datei trotzdem durchsuchen.
|
||||
"""
|
||||
record = add_audio("/tmp/misheard.mp3")
|
||||
store_transcript(record.id, [("Die Sage vom Schluessel im alten Gemäuer.", 0.0, 6.0)])
|
||||
|
||||
# Keine exakte Uebereinstimmung fuer den Suchbegriff im FTS-Index
|
||||
assert repo.fts_candidates(build_fts_query(["schlüssel"]), 10) == []
|
||||
assert repo.trigram_candidates(["schlüssel"], 10) == []
|
||||
|
||||
hits = _engine(repo, search_config).search("Schlüssel")
|
||||
|
||||
assert len(hits) == 1
|
||||
assert hits[0].audio_id == record.id
|
||||
assert hits[0].source == "fallback"
|
||||
assert hits[0].score >= 70
|
||||
|
||||
|
||||
def test_search_returns_nothing_for_unrelated_text(repo, add_audio, store_transcript, search_config):
|
||||
record = add_audio("/tmp/unrelated.mp3")
|
||||
store_transcript(record.id, ["Es war einmal ein kleiner Drache im Wald."])
|
||||
|
||||
hits = _engine(repo, search_config).search("Quantenphysik Vorlesung")
|
||||
assert hits == []
|
||||
|
||||
|
||||
def test_min_score_and_limit(repo, add_audio, store_transcript, search_config):
|
||||
for index in range(5):
|
||||
record = add_audio(f"/tmp/folge{index}.mp3")
|
||||
store_transcript(
|
||||
record.id,
|
||||
[f"Der Schlüssel liegt im alten Haus Nummer {index}.", ],
|
||||
)
|
||||
|
||||
engine = _engine(repo, search_config)
|
||||
all_hits = engine.search("Der Schlüssel liegt im alten Haus", limit=10, min_score=50)
|
||||
assert len(all_hits) >= 1
|
||||
|
||||
limited = engine.search("Der Schlüssel liegt im alten Haus", limit=2, min_score=50)
|
||||
assert len(limited) == 2
|
||||
|
||||
strict = engine.search("Der Schlüssel liegt im alten Haus", limit=10, min_score=99.5)
|
||||
assert len(strict) <= len(all_hits)
|
||||
|
||||
|
||||
def test_results_sorted_by_score(repo, add_audio, store_transcript, search_config):
|
||||
exact = add_audio("/tmp/exact.mp3")
|
||||
store_transcript(exact.id, ["Der Schlüssel liegt im alten Haus."])
|
||||
fuzzy = add_audio("/tmp/fuzzy.mp3")
|
||||
store_transcript(fuzzy.id, ["Der Schlüssel ist wohl im alten Haus zu finden, oder?"])
|
||||
|
||||
hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus", min_score=60)
|
||||
|
||||
scores = [hit.score for hit in hits]
|
||||
assert scores == sorted(scores, reverse=True)
|
||||
assert hits[0].filename == "exact.mp3"
|
||||
|
||||
|
||||
def test_language_filter(repo, add_audio, store_transcript, search_config):
|
||||
german = add_audio("/tmp/de.mp3")
|
||||
store_transcript(german.id, ["Der Schlüssel liegt im alten Haus."], language="de")
|
||||
english = add_audio("/tmp/en.mp3")
|
||||
store_transcript(english.id, ["The key is in the old house."], language="en")
|
||||
|
||||
hits = _engine(repo, search_config).search(
|
||||
"The key is in the old house", language="en", min_score=60
|
||||
)
|
||||
assert [hit.audio_id for hit in hits] == [english.id]
|
||||
|
||||
german_hits = _engine(repo, search_config).search(
|
||||
"Der Schlüssel liegt im alten Haus", language="de", min_score=60
|
||||
)
|
||||
assert [hit.audio_id for hit in german_hits] == [german.id]
|
||||
|
||||
|
||||
def test_duplicate_files_are_not_searched(repo, add_audio, store_transcript, search_config):
|
||||
original = add_audio("/tmp/original.mp3")
|
||||
store_transcript(original.id, ["Der Schlüssel liegt im alten Haus."])
|
||||
duplicate = add_audio("/tmp/kopie.mp3", duplicate_of=original.id)
|
||||
|
||||
hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus")
|
||||
assert {hit.audio_id for hit in hits} == {original.id}
|
||||
|
||||
|
||||
def test_empty_query_is_rejected(repo, search_config):
|
||||
with pytest.raises(SearchError):
|
||||
_engine(repo, search_config).search(" ... ")
|
||||
|
||||
|
||||
def test_word_insertions_and_deletions(repo, add_audio, store_transcript, search_config):
|
||||
record = add_audio("/tmp/insert.mp3")
|
||||
store_transcript(
|
||||
record.id,
|
||||
["Also, der Schlüssel liegt wirklich direkt im alten Haus, nicht wahr?"],
|
||||
)
|
||||
|
||||
hits = _engine(repo, search_config).search("Schlüssel liegt im alten Haus")
|
||||
assert hits
|
||||
assert hits[0].score >= 80
|
||||
|
||||
|
||||
def test_multiword_query_matches_whole_phrase_not_single_word(repo, add_audio, store_transcript, search_config):
|
||||
"""Regression: token_set_ratio darf ein Einzelwort nicht als 100% werten."""
|
||||
record = add_audio("/tmp/kurz.mp3")
|
||||
store_transcript(record.id, [("Leider zu spät.", 0.0, 2.0)])
|
||||
|
||||
hits = _engine(repo, search_config).search("Leider zu spät", limit=5, min_score=70)
|
||||
|
||||
assert hits
|
||||
assert hits[0].context.match == "Leider zu spät."
|
||||
assert hits[0].score == pytest.approx(100.0, abs=0.1)
|
||||
|
||||
|
||||
def test_normalization_and_scoring_helpers():
|
||||
assert normalize_text("Der Schlüssel, liegt im alten Haus!") == "der schlüssel liegt im alten haus"
|
||||
assert score_window("der schlüssel liegt im alten haus", "der schlüssel liegt im alten haus").score == 100.0
|
||||
assert score_window("der schlüssel liegt im alten haus", "").score == 0.0
|
||||
assert build_fts_query(["haus"]).startswith('"haus"')
|
||||
|
||||
|
||||
def test_partial_window_is_not_a_perfect_match():
|
||||
# Ein einzelnes Wort einer mehrwoertrigen Anfrage darf nicht 100 ergeben.
|
||||
assert score_window("leider zu spät", "leider").score < 70
|
||||
assert score_window("der schlüssel liegt im alten haus", "haus").score < 70
|
||||
@@ -0,0 +1,247 @@
|
||||
"""Tests fuer Transkriptionswarteschlange, Wiederaufnahme und GPU-Diagnose."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from sjaudiosearch.config import ProcessingConfig
|
||||
from sjaudiosearch.core.models import FileStatus
|
||||
from sjaudiosearch.system.gpu import (
|
||||
GpuInfo,
|
||||
RuntimeInfo,
|
||||
resolve_execution,
|
||||
)
|
||||
from sjaudiosearch.transcription.models import TranscriptResult, TranscriptSegment, WordToken
|
||||
from sjaudiosearch.transcription.worker import TranscriptionWorker
|
||||
|
||||
|
||||
class FakeEngine:
|
||||
"""Ersetzt faster-whisper; laedt nie ein Modell und braucht kein Netzwerk."""
|
||||
|
||||
model_name = "fake-model"
|
||||
|
||||
def __init__(self, fail_for: set[str] | None = None):
|
||||
self.fail_for = fail_for or set()
|
||||
self.calls: list[str] = []
|
||||
|
||||
def transcribe(self, path: str | Path) -> TranscriptResult:
|
||||
name = Path(path).name
|
||||
self.calls.append(name)
|
||||
if name in self.fail_for:
|
||||
raise RuntimeError("Simulierter Decoderfehler")
|
||||
|
||||
words = [
|
||||
WordToken(word="Der", start=0.0, end=0.4),
|
||||
WordToken(word="Schlüssel", start=0.4, end=1.0),
|
||||
WordToken(word="liegt", start=1.0, end=1.4),
|
||||
WordToken(word="im", start=1.4, end=1.6),
|
||||
WordToken(word="alten", start=1.6, end=2.0),
|
||||
WordToken(word="Haus.", start=2.0, end=2.6),
|
||||
]
|
||||
segment = TranscriptSegment(
|
||||
text="Der Schlüssel liegt im alten Haus.",
|
||||
start=0.0,
|
||||
end=2.6,
|
||||
confidence=0.95,
|
||||
words=words,
|
||||
)
|
||||
return TranscriptResult(
|
||||
segments=[segment],
|
||||
full_text=segment.text,
|
||||
language="de",
|
||||
language_probability=0.99,
|
||||
duration=10.0,
|
||||
model=self.model_name,
|
||||
)
|
||||
|
||||
|
||||
def test_worker_transcribes_and_indexes(repo, add_audio, make_wav, search_config):
|
||||
record = add_audio(make_wav("folge1.wav"))
|
||||
engine = FakeEngine()
|
||||
worker = TranscriptionWorker(repo, engine, ProcessingConfig())
|
||||
|
||||
stats = worker.run()
|
||||
|
||||
assert stats.queued == 1
|
||||
assert stats.processed == 1
|
||||
assert stats.failed == 0
|
||||
assert engine.calls == ["folge1.wav"]
|
||||
|
||||
updated = repo.get_audio(record.id)
|
||||
assert updated.status == FileStatus.DONE.value
|
||||
transcript = repo.get_transcript(record.id)
|
||||
assert transcript is not None
|
||||
assert transcript["model"] == "fake-model"
|
||||
|
||||
from sjaudiosearch.search.engine import SearchEngine
|
||||
|
||||
hits = SearchEngine(repo, search_config).search("Schlüssel liegt im alten Haus")
|
||||
assert hits and hits[0].audio_id == record.id
|
||||
|
||||
|
||||
def test_worker_isolates_errors(repo, add_audio, make_wav):
|
||||
good = add_audio(make_wav("good.wav"))
|
||||
bad = add_audio(make_wav("bad.wav"))
|
||||
engine = FakeEngine(fail_for={"bad.wav"})
|
||||
worker = TranscriptionWorker(repo, engine, ProcessingConfig())
|
||||
|
||||
stats = worker.run()
|
||||
|
||||
assert stats.queued == 2
|
||||
assert stats.processed == 1
|
||||
assert stats.failed == 1
|
||||
assert stats.errors[0][0].endswith("bad.wav")
|
||||
|
||||
assert repo.get_audio(good.id).status == "done"
|
||||
failed = repo.get_audio(bad.id)
|
||||
assert failed.status == "error"
|
||||
assert "Simulierter Decoderfehler" in failed.error_message
|
||||
|
||||
|
||||
def test_worker_skips_done_and_resumes(repo, add_audio, make_wav):
|
||||
record = add_audio(make_wav("done.wav"), status="done")
|
||||
engine = FakeEngine()
|
||||
worker = TranscriptionWorker(repo, engine, ProcessingConfig())
|
||||
|
||||
stats = worker.run()
|
||||
assert stats.queued == 0
|
||||
assert engine.calls == []
|
||||
|
||||
stats_forced = worker.run(force=True)
|
||||
assert stats_forced.queued == 1
|
||||
assert engine.calls == ["done.wav"]
|
||||
assert repo.get_audio(record.id).status == "done"
|
||||
|
||||
|
||||
def test_worker_resets_stale_processing(repo, add_audio, make_wav):
|
||||
record = add_audio(make_wav("haengt.wav"), status="processing")
|
||||
worker = TranscriptionWorker(repo, FakeEngine(), ProcessingConfig(resume=True))
|
||||
|
||||
stats = worker.run()
|
||||
|
||||
assert stats.processed == 1
|
||||
assert repo.get_audio(record.id).status == "done"
|
||||
|
||||
|
||||
def test_worker_limits_queue(repo, add_audio, make_wav):
|
||||
for index in range(3):
|
||||
add_audio(make_wav(f"folge{index}.wav"))
|
||||
engine = FakeEngine()
|
||||
worker = TranscriptionWorker(repo, engine, ProcessingConfig())
|
||||
|
||||
stats = worker.run(limit=2)
|
||||
|
||||
assert stats.queued == 2
|
||||
assert len(engine.calls) == 2
|
||||
|
||||
|
||||
def test_failed_transcription_can_be_retried(repo, add_audio, make_wav):
|
||||
record = add_audio(make_wav("retry.wav"))
|
||||
failing = TranscriptionWorker(repo, FakeEngine(fail_for={"retry.wav"}), ProcessingConfig())
|
||||
failing.run()
|
||||
assert repo.get_audio(record.id).status == "error"
|
||||
|
||||
assert repo.reset_errors() == 1
|
||||
retrying = TranscriptionWorker(repo, FakeEngine(), ProcessingConfig())
|
||||
stats = retrying.run()
|
||||
|
||||
assert stats.processed == 1
|
||||
assert repo.get_audio(record.id).status == "done"
|
||||
|
||||
|
||||
def test_worker_marks_missing_file(repo, add_audio):
|
||||
record = add_audio("/tmp/gibt-es-nicht-mehr.mp3")
|
||||
Path(record.path).unlink(missing_ok=True)
|
||||
stats = TranscriptionWorker(repo, FakeEngine(), ProcessingConfig()).run()
|
||||
|
||||
assert stats.failed == 1
|
||||
assert repo.get_audio(record.id).status == "error"
|
||||
|
||||
|
||||
def test_worker_events_are_emitted(repo, add_audio, make_wav):
|
||||
add_audio(make_wav("event.wav"))
|
||||
events = []
|
||||
|
||||
TranscriptionWorker(repo, FakeEngine(), ProcessingConfig()).run(event_callback=events.append)
|
||||
|
||||
kinds = [event.kind for event in events]
|
||||
assert kinds[0] == "start"
|
||||
assert "done" in kinds
|
||||
assert kinds[-1] == "finished"
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# GPU-/Ausfuehrungsdiagnose (reine Funktionen, keine echte GPU noetig)
|
||||
# ----------------------------------------------------------------------
|
||||
def _pascal_gpu() -> GpuInfo:
|
||||
return GpuInfo(
|
||||
name="NVIDIA GeForce GTX 1070 Ti",
|
||||
driver_version="580.196.07",
|
||||
cuda_driver_version="13.0",
|
||||
memory_total_mb=8192,
|
||||
compute_capability="6.1",
|
||||
)
|
||||
|
||||
|
||||
def _runtime(**overrides) -> RuntimeInfo:
|
||||
runtime = RuntimeInfo(
|
||||
ctranslate2_version="4.9.0",
|
||||
cuda_device_count=1,
|
||||
supported_compute_types={
|
||||
"cpu": ["float32", "int16", "int8", "int8_float32"],
|
||||
"cuda": ["float32", "int8", "int8_float32"],
|
||||
},
|
||||
cuda_libraries=["/usr/lib/libcudart.so.12"],
|
||||
cudnn_libraries=["/usr/lib/libcudnn.so.9"],
|
||||
python_version="3.13.15",
|
||||
platform="Linux",
|
||||
sqlite_version="3.50.0",
|
||||
fts5_available=True,
|
||||
trigram_available=True,
|
||||
)
|
||||
for key, value in overrides.items():
|
||||
setattr(runtime, key, value)
|
||||
return runtime
|
||||
|
||||
|
||||
def test_pascal_uses_int8_float32():
|
||||
resolved = resolve_execution("cuda", "int8_float32", _pascal_gpu(), _runtime())
|
||||
assert resolved.device == "cuda"
|
||||
assert resolved.compute_type == "int8_float32"
|
||||
assert not resolved.used_fallback
|
||||
|
||||
|
||||
def test_diagnostics_rejects_float16_on_incompatible_gpu():
|
||||
resolved = resolve_execution("cuda", "float16", _pascal_gpu(), _runtime())
|
||||
assert resolved.compute_type != "float16"
|
||||
assert resolved.compute_type == "int8_float32"
|
||||
assert any("Pascal" in note for note in resolved.notes)
|
||||
|
||||
|
||||
def test_diagnostics_falls_back_when_int8_float32_unsupported():
|
||||
runtime = _runtime(
|
||||
supported_compute_types={"cpu": ["float32", "int8"], "cuda": ["float32"]}
|
||||
)
|
||||
resolved = resolve_execution("cuda", "int8_float32", _pascal_gpu(), runtime)
|
||||
assert resolved.compute_type == "float32"
|
||||
assert resolved.device == "cuda"
|
||||
|
||||
|
||||
def test_cpu_fallback_when_cuda_unavailable():
|
||||
runtime = _runtime(
|
||||
cuda_device_count=0,
|
||||
supported_compute_types={"cpu": ["float32", "int8"], "cuda": []},
|
||||
)
|
||||
resolved = resolve_execution("cuda", "int8_float32", None, runtime)
|
||||
assert resolved.device == "cpu"
|
||||
assert resolved.used_fallback
|
||||
assert resolved.compute_type in {"int8", "int8_float32", "float32"}
|
||||
assert any("CPU" in note for note in resolved.notes)
|
||||
|
||||
|
||||
def test_cpu_mode_never_returns_gpu_compute_type():
|
||||
resolved = resolve_execution("cpu", "float16", _pascal_gpu(), _runtime())
|
||||
assert resolved.device == "cpu"
|
||||
assert resolved.compute_type not in {"float16", "bfloat16"}
|
||||
Reference in New Issue
Block a user