commit 97e5c9ef4d3a9dfe9be37f839b53cd82a35d1844 Author: jhkaktus Date: Mon Nov 16 16:29:32 2026 +0100 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b937731 --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +.venv/ +venv/ +__pycache__/ +*.py[cod] +*.egg-info/ +build/ +dist/ +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ + +# Lokale Daten (Datenbank, Modelle, Logs) +data/* +!data/.gitkeep +logs/* +!logs/.gitkeep + +config.local.toml +*.db +*.db-wal +*.db-shm + +# makepkg-Artefakte +pkg/ +src/ +*.pkg.tar.* diff --git a/PKGBUILD b/PKGBUILD new file mode 100644 index 0000000..0c88643 --- /dev/null +++ b/PKGBUILD @@ -0,0 +1,62 @@ +# Maintainer: SjAudioSearch +# +# Baut das Paket aus diesem Quellbaum und installiert es systemweit unter /usr: +# +# makepkg -si +# +# Fuer eine Veroeffentlichung im AUR muss der Quelltext als Release-Tarball +# bereitstehen (z. B. PyPI-sdist oder GitHub-Tag). Dann nur diese beiden +# Stellen anpassen: +# +# source=("$pkgname-$pkgver.tar.gz::https://example.org/$pkgname-$pkgver.tar.gz") +# sha256sums=('') +# _srcbase="$srcdir/$pkgname-$pkgver" +# +# Alles andere bleibt unveraendert. + +pkgname=sjaudiosearch +pkgver=1.0.0 +pkgrel=1 +pkgdesc="Lokale Suchmaschine fuer Hoerspiele, Hoerbuecher und Audiosammlungen (faster-whisper + SQLite FTS5)" +arch=('any') +url="https://example.org/sjaudiosearch" +license=('MIT') +depends=( + 'python' + 'python-typer' + 'python-rich' + 'python-faster-whisper' + 'python-rapidfuzz' + 'python-mutagen' + 'python-av' +) +makedepends=( + 'python-build' + 'python-installer' + 'python-setuptools' + 'python-wheel' +) +optdepends=( + 'ffmpeg: Wiedergabe der Fundstellen ueber ffplay und Unterstuetzung fuer WMA' + 'mpv: alternative Wiedergabe der Fundstellen' + 'vlc: alternative Wiedergabe der Fundstellen' + 'python-ctranslate2-cuda: CUDA-Beschleunigung auf NVIDIA-GPUs' +) +# Hinweis: faster-whisper 1.2.1 benoetigt PyAV < 17 (Argument metadata_errors). +source=() +sha256sums=() +_srcbase="$startdir" + +build() { + cd "$_srcbase" + python -m build --wheel --no-isolation --outdir "$srcdir/dist" +} + +package() { + cd "$_srcbase" + + python -m installer --destdir="$pkgdir" "$srcdir"/dist/*.whl + + install -Dm644 README.md "$pkgdir/usr/share/doc/$pkgname/README.md" + install -Dm644 config.toml "$pkgdir/usr/share/doc/$pkgname/config.toml.example" +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..ec25b90 --- /dev/null +++ b/README.md @@ -0,0 +1,515 @@ +# SjAudioSearch + +Lokale Suchmaschine für Hörspiele, Hörbücher und andere Audiosammlungen. + +SjAudioSearch liest einen kompletten Audioordner rekursiv ein, transkribiert alle +Dateien lokal mit **faster-whisper** und durchsucht anschließend sämtliche +Transkripte. Die Suche funktioniert auch dann, wenn man sich nur ungefähr an +einen Satz erinnert oder wenn Whisper Wörter falsch erkannt hat. + +``` +Gesucht: "Der Schlüssel liegt im alten Haus" +Transkript: "Der Schlüssel befindet sich in dem alten Haus." +Ergebnis: Ähnlichkeit 89 %, Zeitstempel 00:42:13, Kontext 50 Wörter davor/danach +``` + +Es werden **keine Cloud-Dienste und keine kostenpflichtigen APIs** verwendet. +Alles läuft lokal: Transkription, Datenbank, Volltextsuche und Kontextaufbau. + +--- + +## Funktionsumfang + +- Rekursiver Scan von MP3, WAV, FLAC, M4A, OGG, OPUS und WMA +- Änderungserkennung über Dateigröße und Änderungszeit (SHA256 nur bei Bedarf) +- Duplikaterkennung über Inhalts-Hash +- Lokale Transkription mit faster-whisper/CTranslate2, Modell wird nur einmal geladen +- Fortsetzbare Warteschlange, Fehler einer Datei stoppen den Lauf nicht +- SQLite-Datenbank mit FTS5-Volltextindex, Trigramm-Index und Wort-Zeitstempeln +- Fuzzy-Suche mit RapidFuzz und Sliding-Window-Vergleich über Segmentgrenzen +- Fallback-Suche auch ohne FTS-Treffer (wichtig bei komplett falsch erkannten Wörtern) +- Kontext mit 50 Wörtern vor und nach dem Treffer, Zeitstempel aus der Originaldatei +- Wiedergabe der Fundstelle über `ffplay`, `mpv` oder `vlc` +- GPU-Diagnose inklusive tatsächlich unterstützter Compute Types +- Automatischer CPU-Fallback + +--- + +## Voraussetzungen + +| Komponente | Anforderung | +| --- | --- | +| Betriebssystem | Linux (getestet: CachyOS/Arch, Kernel 7.3), Windows/macOS mit CPU-Fallback möglich | +| Python | 3.10 bis 3.13 (getestet mit 3.13.15) | +| GPU | optional; getestet mit NVIDIA GTX 1070 Ti | +| NVIDIA-Treiber | ≥ 525 für CUDA 12 (getestet: 580.196.07) | +| ffmpeg/ffplay | optional, für Wiedergabe und einige WMA-Codecs | +| Freier Speicher | large-v3 benötigt ca. 3 GB, small ca. 0,5 GB | + +> **Python 3.14 wird derzeit nicht unterstützt**, weil für CTranslate2/faster-whisper +> keine passenden Wheels verfügbar sind. Unter CachyOS/Arch ist `python3.13` +> installiert und kann direkt verwendet werden. + +--- + +## Installation + +```bash +cd /pfad/zu/sjaudiosearch + +# Isolierte Umgebung mit Python 3.13 +python3.13 -m venv .venv +source .venv/bin/activate + +# Projekt + Abhängigkeiten installieren (inkl. CUDA-12-Laufzeit) +pip install -e ".[cuda]" + +# Alternativ rein auf der CPU: +# pip install -e . +``` + +### Arch Linux (AUR) + +Das Projekt bringt ein `PKGBUILD` mit, das SjAudioSearch systemweit unter +`/usr` installiert: + +```bash +makepkg -si +``` + +Die CUDA-Laufzeit wird über die offiziellen NVIDIA-Pip-Pakete installiert +(`nvidia-cublas-cu12`, `nvidia-cudnn-cu12`). Es ist **keine systemweite +CUDA-Installation** und kein Eingriff in `/etc/ld.so.conf` nötig. + +### Feste Versionen + +| Paket | Version | +| --- | --- | +| faster-whisper | 1.2.1 | +| ctranslate2 | 4.9.0 | +| av (PyAV) | 16.1.0 (siehe Hinweis unten) | +| typer | 0.29.1 | +| rich | 15.0.0 | +| rapidfuzz | 3.14.6 | +| mutagen | 1.48.1 | +| pytest | 9.2.0 | +| nvidia-cublas-cu12 | 12.9.3.13 | +| nvidia-cudnn-cu12 | 9.28.1.4 | + +**Warum PyAV 16.1.0?** faster-whisper 1.2.1 ruft `av.open(..., metadata_errors="ignore")` +auf. PyAV 17+ hat dieses Argument entfernt, wodurch jede Transkription mit +`TypeError: open() got an unexpected keyword argument 'metadata_errors'` +abbrechen würde. 16.1.0 ist die neueste Version, die beides erfüllt: +kompatibel zu faster-whisper und mit Wheels für Python 3.13. + +--- + +## GPU-Konfiguration (GTX 1070 Ti / Pascal) + +Die GTX 1070 Ti ist eine Pascal-Karte mit **Compute Capability 6.1**. Sie hat +keine FP16-/BF16-Tensor-Cores. Deshalb gilt: + +- **`compute_type = int8_float32`** ist der bevorzugte Modus. +- **FP16 und BF16 werden nie automatisch verwendet.** +- Falls `int8_float32` nicht gemeldet wird, wird `float32` versucht, danach `int8`. +- Ist CUDA nicht nutzbar, läuft die Transkription automatisch auf der CPU. + +Die verfügbaren Compute Types werden zur Laufzeit über CTranslate2 ermittelt, +nicht geraten: + +```bash +sjaudiosearch diagnose +``` + +Verifiziertes Ergebnis auf diesem System: + +``` +NVIDIA GeForce GTX 1070 Ti +Treiber 580.196.07 +CUDA (Treiber) 13.0 +VRAM 8192 MB +Compute Capability 6.1 +CUDA-Geräte 1 +Compute Types (CUDA) float32, int8, int8_float32 +Compute Types (CPU) float32, int16, int8, int8_float32 +Verwendet cuda / int8_float32 +FP16/BF16 nicht verwendet +``` + +### CUDA-Bibliotheken ohne Systemeingriff + +CTranslate2 linkt dynamisch gegen `libcublas.so.12` und `libcudnn.so.9`. Die +Pip-Pakete legen diese Bibliotheken unter +`.venv/lib/python3.13/site-packages/nvidia/.../lib` ab, wo der dynamische +Linker sie nicht von allein findet. SjAudioSearch lädt sie deshalb beim Start +mit `ctypes` und `RTLD_GLOBAL` vor (`sjaudiosearch/system/gpu.py`, +`ensure_cuda_libraries`). Das ist kein Hack an Systemdateien, sondern eine +lokale, nachvollziehbare Initialisierung. + +### Modellspeicher + +| Modell | Download | VRAM (int8_float32) | +| --- | --- | --- | +| large-v3 (Default) | ca. 3,4 GB | ca. 1,5–2 GB | +| medium | ca. 1,5 GB | ca. 1 GB | +| small | ca. 0,5 GB | ca. 0,5 GB | +| base | ca. 0,15 GB | ca. 0,2 GB | + +Die Modelle landen in `./data/models` (konfigurierbar über +`transcription.model_dir`). + +### CPU-Fallback + +Ohne GPU oder bei einem CUDA-Fehler: + +```bash +sjaudiosearch transcribe --device cpu --compute-type int8 +``` + +Das Programm wechselt auch automatisch auf die CPU, wenn das Laden im +GPU-Modus fehlschlägt, und informiert darüber im Terminal. Die CPU ist +deutlich langsamer; die Ergebnisse sind identisch. + +--- + +## Ersteinrichtung + +1. Konfiguration prüfen (optional anpassen): + + ```bash + cat config.toml + ``` + +2. Systemdiagnose ausführen: + + ```bash + sjaudiosearch diagnose + ``` + +3. Datenbank und Logs werden beim ersten Befehl automatisch angelegt. + +Relative Pfade in `config.toml` werden relativ zur Konfigurationsdatei +aufgelöst. Eine andere Konfiguration lässt sich mit `--config` oder der +Umgebungsvariable `SJAUDIOSEARCH_CONFIG` verwenden. Ohne eigene Konfiguration +wird zusätzlich `~/.config/sjaudiosearch/config.toml` und +`/etc/sjaudiosearch/config.toml` durchsucht; nach einer systemweiten +Installation landen Datenbank und Logs in `~/.local/share/sjaudiosearch`. + +--- + +## Erster Scan + +```bash +sjaudiosearch scan "/home/user/Music/Hoerspiele" +``` + +Ausgabe: gefundene Dateien, neue/geänderte/unveränderte Dateien, Duplikate und +Dateien mit Problemen. Fehlerhafte Dateien werden mit Status `error` gespeichert +und im Log protokolliert. + +Erneutes Scannen ist günstig: unveränderte Dateien werden nur über Größe und +Änderungszeit geprüft. Hashes werden im Modus `auto` nur berechnet, wenn +Duplikate möglich sind. + +--- + +## Erste Transkription + +```bash +# Standard: large-v3, CUDA, int8_float32, Deutsch +sjaudiosearch transcribe + +# Weniger VRAM / schneller Test +sjaudiosearch transcribe --model small + +# Nur 10 Dateien, bereits fertige erneut verarbeiten +sjaudiosearch transcribe --limit 10 --force + +# Andere Sprache oder automatische Erkennung +sjaudiosearch transcribe --language en +sjaudiosearch transcribe --language auto +``` + +Während des Laufs werden aktuelle Datei, Fortschritt, Dauer, Fehler und die +durchschnittliche Geschwindigkeit angezeigt. Der Status jeder Datei wird +dauerhaft gespeichert. Nach einem Abbruch oder Neustart macht +`sjaudiosearch transcribe` an der letzten Stelle weiter. + +--- + +## Erste Suche + +```bash +sjaudiosearch search "Der Schlüssel liegt im alten Haus" +``` + +Optionen: + +```bash +sjaudiosearch search "..." --limit 50 --min-score 70 --context-words 50 +sjaudiosearch search "..." --language de +sjaudiosearch search "..." --json +sjaudiosearch search "..." --verbose +``` + +Jeder Treffer enthält Dateiname, vollständigen Pfad, Ähnlichkeit (0–100 %), +Start-/Endzeit, 50 Wörter Kontext davor und danach sowie die Fundstelle selbst. + +Der Ähnlichkeitswert ist eine Vergleichsbewertung, **keine statistische +Wahrscheinlichkeit**. + +--- + +## CLI-Befehlsübersicht + +| Befehl | Zweck | +| --- | --- | +| `sjaudiosearch diagnose` | Systembericht: GPU, CUDA, Compute Types, FTS5, Player | +| `sjaudiosearch scan ` | Ordner rekursiv scannen, neue/geänderte Dateien vormerken | +| `sjaudiosearch transcribe` | Transkriptionswarteschlange abarbeiten | +| `sjaudiosearch search ""` | Fuzzy-Suche über alle Transkripte | +| `sjaudiosearch status` | Anzahl Dateien, Status, Spieldauer, Datenbankgröße, Modell | +| `sjaudiosearch list` | Bekannte Dateien auflisten (`--status`, `--search`, `--limit`) | +| `sjaudiosearch info ` | Details zu Datei und Transkript | +| `sjaudiosearch play --time 120` | Datei ab 120 s abspielen (ffplay/mpv/vlc) | +| `sjaudiosearch retry` | Fehlgeschlagene Transkriptionen zurücksetzen | +| `sjaudiosearch reindex` | Suchindex aus vorhandenen Transkripten neu aufbauen | +| `sjaudiosearch remove ` | Datei aus dem Index entfernen (Original bleibt erhalten) | +| `sjaudiosearch check` | Datenbank, FTS5, Trigramme und Pfade prüfen | + +`--version` zeigt die Programmversion. `--config` wählt eine andere TOML-Datei. + +--- + +## Konfiguration + +`config.toml` (vollständig kommentiert im Projekt): + +```toml +[transcription] +model = "large-v3" +device = "cuda" +compute_type = "int8_float32" +language = "de" # "auto" für automatische Erkennung +beam_size = 5 +word_timestamps = true +vad_filter = true +model_dir = "./data/models" + +[search] +default_limit = 50 +default_min_score = 70 +context_words = 50 +window_before = 2 # Suchfenster = Anfrage - 2 Wörter +window_after = 4 # Suchfenster = Anfrage + 4 Wörter +candidate_limit = 4000 +min_token_overlap = 0.34 +exhaustive_fallback = true +max_hits_per_file = 5 + +[processing] +workers = 1 +resume = true +compute_hash = "auto" # auto, always, never +hash_max_size_mb = 4096 + +[database] +path = "./data/sjaudiosearch.db" + +[logging] +level = "INFO" +path = "./logs/sjaudiosearch.log" +``` + +Unbekannte Schlüssel werden beim Start gemeldet, nicht stillschweigend ignoriert. +`workers` ist bewusst auf 1 festgelegt: Die GPU soll nur von einem +Transkriptionsprozess genutzt werden. + +--- + +## Wie die Suche funktioniert + +1. **Normalisierung** der Anfrage (Kleinschreibung, Satzzeichen entfernen, + Leerraum vereinheitlichen). Der gespeicherte Originaltext bleibt unverändert. +2. **Kandidatensuche** über SQLite FTS5 (ODER-Verknüpfung, Präfixe für längere + Wörter) und den FTS5-Trigramm-Index (Teilzeichenketten). +3. **Sliding Window** über die Wort-Timeline jeder Kandidatendatei. Die + Fenstergröße liegt bei Anfrage −2 bis Anfrage +4 Wörtern, die Timeline läuft + über Segmentgrenzen hinweg. +4. **Bewertung** mit RapidFuzz (`ratio`, `token_set_ratio`, `WRatio`, + `partial_ratio`). Teilfenster werden über das Längenverhältnis gedämpft, + damit ein einzelnes Wort einer mehrwörtigen Anfrage nicht als 100-%-Treffer + gilt. +5. **Fallback**: Reichen die FTS-Kandidaten nicht aus, werden alle übrigen + Dateien vollständig durchsucht. Dadurch wird z. B. `austeilen` gefunden, + obwohl Whisper `Aussteilen` geschrieben hat und FTS keinen Treffer liefert. +6. **Entduplizierung und Sortierung** nach Ähnlichkeit, danach Begrenzung auf + `--limit`. + +### Kontext und Zeitstempel + +Der Kontext wird aus der durchgehenden Wort-Timeline geschnitten, nicht aus +einzelnen Segmenten. Dadurch beginnt und endet er exakt an der Fundstelle, auch +mitten in einem Segment. Liegen Wort-Zeitstempel vor, wird die Genauigkeit als +`Wort` ausgewiesen; andernfalls werden Zeiten innerhalb des Segments verteilt +und als `Segment` gekennzeichnet. Die Zeit bezieht sich immer auf die +Originaldatei. + +--- + +## Datenbank + +SQLite mit WAL, Foreign Keys und Migrationen (`PRAGMA user_version`). + +| Tabelle | Inhalt | +| --- | --- | +| `audio_files` | Pfad, Dateiname, Dauer, Größe, Änderungszeit, Hash, Status, Duplikat-Verweis | +| `transcripts` | ein Transkript pro Datei, Sprache, Modell, Volltext | +| `segments` | Text mit Start-/Endzeit und Confidence | +| `words` | optionale Wort-Zeitstempel | +| `search_index` | FTS5-Volltextindex (unicode61, ohne Diakritika) | +| `search_trigram` | FTS5-Trigramm-Index für Teilzeichenketten | +| `meta` | interne Metadaten | + +Transkripte werden pro Datei in einer Transaktion gespeichert: Entweder ist +eine Datei vollständig verarbeitet oder sie bleibt `pending`/`error`. Ein +Absturz hinterlässt keinen halben Suchindex. + +--- + +## Tests + +```bash +pip install -e ".[test]" +pytest +``` + +Die Tests verwenden winzige, künstlich erzeugte WAV-Dateien und eine +Fake-Transkriptions-Engine. Es wird **kein Whisper-Modell heruntergeladen**. + +Getestet werden unter anderem: + +- Scan mehrerer Dateien, Duplikat- und Änderungserkennung, defekte Dateien +- Datenbankschema, Migrationen, Foreign-Key-Kaskaden, Transkripte, Wörter +- Statusverwaltung, Retry, Wiederaufnahme, `--force`, `--limit` +- Fuzzy-Suche mit fehlenden/zusätzlichen/falsch erkannten Wörtern +- Suche über Segmentgrenzen hinweg +- Fallback-Suche ohne FTS-Treffer +- 50 Wörter Kontext davor/danach und Zeitstempel-Genauigkeit +- Reindex ohne Veränderung der Transkripte +- CPU-Fallback und Ablehnung von FP16 auf Pascal + +Ergebnis dieser Umgebung: + +``` +49 passed +``` + +--- + +## Verifizierter GPU-Test + +Auf der GTX 1070 Ti wurde der komplette Ablauf mit echten deutschen +Sprachaufnahmen (Public-Domain-Dateien von Wikimedia Commons) ausgeführt: + +``` +Modell large-v3 +Gerät cuda (GPU) +Compute Type int8_float32 +GPU NVIDIA GeForce GTX 1070 Ti (CC 6.1, 8192 MB) +Warteschlange 2 +Erfolgreich 2 +Audiodauer 24 s +Laufzeit 2 s +Durchschnitt 13.10x Echtzeit +``` + +Beispiel für die Fallback-Suche (Whisper schrieb `Aussteilen` statt +`austeilen`): + +``` +$ sjaudiosearch search "Austeilen" + +[1] hoerspiel_02.wav +Similarity: 94.7% +Timestamp: 00:00:06 (bis 00:00:08, Wort-Genauigkeit) +Quelle: fallback +``` + +Das sind kurze Testclips, kein Benchmark über komplette Hörspiele. Die +Transkription lief vollständig auf der GPU; die Geschwindigkeit hängt von +Dateilänge, Modell und Festplattenzugriff ab. + +--- + +## Bekannte Einschränkungen + +- **Wiederholtes identisches Audio:** Whisper (large-v3) hat in einem Testclip + drei identische Sätze hintereinander nur einmal transkribiert. Das ist ein + Modellverhalten, kein Fehler der Datenbank oder der Suche. Unterschiedliche + Dialoge sind davon nicht betroffen. +- **Wiedergabe** benötigt ein funktionierendes Audio-Ausgabegerät. Der Befehl + wurde hier mit einem Dummy-Audiotreiber geprüft; auf einem normalen Desktop + startet `ffplay` hörbar. +- **WMA** ist nur zuverlässig, wenn PyAV/FFmpeg den jeweiligen Codec kennt. +- Ohne Wort-Zeitstempel (`word_timestamps = false`) sind Zeitangaben + segmentgenau; das wird im Ergebnis entsprechend gekennzeichnet. +- Die Fallback-Suche durchsucht bei wenigen Treffern alle übrigen Dateien. Das + ist für Korrektheit wichtig, kostet bei sehr großen Sammlungen aber Zeit. + Mit `exhaustive_fallback = false` lässt sich das abschalten. +- Es läuft bewusst nur ein Transkriptionsprozess gleichzeitig (eine GPU). +- Python 3.14 wird wegen fehlender CTranslate2-Wheels nicht unterstützt. +- `remove` löscht ausschließlich Indexdaten, niemals die Audiodatei. +- Ein vollständiger Benchmark über mehrere Stunden Hörspielmaterial wurde in + dieser Umgebung nicht durchgeführt. + +--- + +## Fehlerbehebung + +| Problem | Lösung | +| --- | --- | +| `Library libcublas.so.12 is not found` | `pip install -e ".[cuda]"` (nvidia-cublas-cu12, nvidia-cudnn-cu12) | +| `CUDA ist nicht nutzbar` | Treiber prüfen (`nvidia-smi`), danach `sjaudiosearch diagnose`; notfalls `--device cpu` | +| `float16` nicht unterstützt | Pascal: `compute_type = "int8_float32"` oder `"float32"` verwenden | +| `TypeError ... metadata_errors` | PyAV-Version prüfen: `pip install av==16.1.0` | +| Kein FTS5 | Python mit FTS5-Unterstützung installieren (Standard bei python.org/Arch) | +| Kein Player gefunden | `ffmpeg` installieren (enthält `ffplay`) oder `mpv`/`vlc` | +| Modell-Download schlägt fehl | Internetverbindung prüfen; Modell manuell nach `./data/models` legen | +| GPU-Speicher voll | kleineres Modell (`--model small`), `--limit`, oder CPU verwenden | +| Suche findet nichts | `--min-score` senken, `--context-words` erhöhen, `sjaudiosearch reindex` | +| Datei dauerhaft `error` | Ursache in `logs/sjaudiosearch.log`; danach `sjaudiosearch retry` | + +Logs: `./logs/sjaudiosearch.log` (rotierend, 5 MB × 3). +Datenbank: `./data/sjaudiosearch.db`. + +--- + +## Projektstruktur + +``` +sjaudiosearch/ +├── sjaudiosearch/ +│ ├── cli.py # Typer-CLI +│ ├── config.py # TOML-Konfiguration +│ ├── core/ # Scanner, Metadaten, Audio, Modelle +│ ├── transcription/ # faster-whisper-Engine und Warteschlange +│ ├── search/ # Fuzzy, Sliding Window, Kontext, Ranking +│ ├── database/ # Verbindung, Schema, Migrationen, Repository +│ ├── system/ # GPU-Diagnose und Systembericht +│ └── utils/ # Hashing, Formatierung, Logging +├── tests/ # pytest-Suite (49 Tests) +├── config.toml +├── pyproject.toml +├── PKGBUILD # Paket fuer Arch Linux / AUR +├── data/ # Datenbank und Modelle (nicht versioniert) +├── logs/ # Logdateien (nicht versioniert) +└── README.md +``` + +--- + +## Lizenz + +MIT. Die Testaufnahmen stammen aus Wikimedia Commons (Public Domain bzw. +freie Lizenzen) und sind nicht Teil des Projekts. diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..d0d6fab --- /dev/null +++ b/config.toml @@ -0,0 +1,47 @@ +# SjAudioSearch Konfiguration +# Alle relativen Pfade werden relativ zu dieser Datei aufgeloest. + +[transcription] +# Modellgroessen: large-v3, medium, small, base, tiny +model = "large-v3" +# "cuda" oder "cpu" +device = "cuda" +# Bevorzugt int8_float32. FP16 wird auf Pascal (GTX 1070 Ti) nie verwendet. +compute_type = "int8_float32" +# "de" fuer Deutsch, "auto" fuer automatische Spracherkennung +language = "de" +beam_size = 5 +word_timestamps = true +vad_filter = true +# Ablageort fuer heruntergeladene Whisper-Modelle +model_dir = "./data/models" + +[search] +default_limit = 50 +default_min_score = 70 +context_words = 50 +# Suchfenster: Fenstergroesse = Anfrage +- diese Werte (in Woertern) +window_before = 2 +window_after = 4 +# Maximale Anzahl FTS-Kandidatensegmente +candidate_limit = 4000 +# Mindestanteil gemeinsamer Woerter fuer den Sliding-Window-Vorfilter +min_token_overlap = 0.34 +# Auch Dateien ohne FTS-Treffer durchsuchen (Pflicht fuer falsch erkannte Woerter) +exhaustive_fallback = true +# Maximale Treffer pro Audiodatei +max_hits_per_file = 5 + +[processing] +workers = 1 +resume = true +# SHA256 nur bei Verdacht auf Duplikate berechnen: auto, always, never +compute_hash = "auto" +hash_max_size_mb = 4096 + +[database] +path = "./data/sjaudiosearch.db" + +[logging] +level = "INFO" +path = "./logs/sjaudiosearch.log" diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/logs/.gitkeep b/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5734f08 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,42 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "sjaudiosearch" +version = "1.0.0" +description = "Lokale Suchmaschine fuer Hoerspiele, Hoerbuecher und Audiosammlungen (faster-whisper + SQLite FTS5)" +readme = "README.md" +requires-python = ">=3.10,<3.14" +license = { text = "MIT" } +authors = [{ name = "SjAudioSearch" }] +keywords = ["audio", "transcription", "search", "whisper", "sqlite"] + +dependencies = [ + "typer==0.29.1", + "rich==15.0.0", + "faster-whisper==1.2.1", + "ctranslate2==4.9.0", + "rapidfuzz==3.14.6", + "mutagen==1.48.1", + "av==16.1.0", +] + +[project.optional-dependencies] +test = ["pytest==9.2.0"] +# CUDA-12-Laufzeit fuer NVIDIA-GPUs (keine systemweite CUDA-Installation noetig). +cuda = [ + "nvidia-cublas-cu12==12.9.3.13", + "nvidia-cudnn-cu12==9.28.1.4", +] + +[project.scripts] +sjaudiosearch = "sjaudiosearch.cli:app" + +[tool.setuptools.packages.find] +include = ["sjaudiosearch*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q" +filterwarnings = ["ignore::DeprecationWarning"] diff --git a/sjaudiosearch/__init__.py b/sjaudiosearch/__init__.py new file mode 100644 index 0000000..7813002 --- /dev/null +++ b/sjaudiosearch/__init__.py @@ -0,0 +1,5 @@ +"""SjAudioSearch - lokale Suchmaschine fuer transkribierte Audiosammlungen.""" + +__version__ = "1.0.0" + +__all__ = ["__version__"] diff --git a/sjaudiosearch/__main__.py b/sjaudiosearch/__main__.py new file mode 100644 index 0000000..896cca8 --- /dev/null +++ b/sjaudiosearch/__main__.py @@ -0,0 +1,6 @@ +"""Ermoeglicht ``python -m sjaudiosearch``.""" + +from .cli import app + +if __name__ == "__main__": + app() diff --git a/sjaudiosearch/cli.py b/sjaudiosearch/cli.py new file mode 100644 index 0000000..a16cce2 --- /dev/null +++ b/sjaudiosearch/cli.py @@ -0,0 +1,715 @@ +"""Kommandozeilenschnittstelle von SjAudioSearch.""" + +from __future__ import annotations + +import json +import sqlite3 +import sys +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Optional + +import typer +from rich.console import Console +from rich.markup import escape +from rich.panel import Panel +from rich.progress import ( + BarColumn, + MofNCompleteColumn, + Progress, + SpinnerColumn, + TextColumn, + TimeElapsedColumn, +) +from rich.table import Table + +from . import __version__ +from .config import Config, load_config +from .core.audio import PlaybackError, play as play_audio +from .core.metadata import AudioMetadataError, read_metadata +from .core.models import FileStatus +from .core.scanner import scan as run_scan +from .database.connection import check_fts5 +from .database.repository import Repository +from .search.engine import SearchEngine, SearchError +from .system.diagnostics import collect_report, render_report +from .system.gpu import collect_runtime_info, detect_gpu, resolve_execution +from .transcription.engine import FasterWhisperEngine, TranscriptionError +from .transcription.worker import RunStats, TranscriptionWorker, WorkerEvent +from .utils.formatting import ( + format_datetime, + format_duration, + format_rate, + format_score, + format_size, + format_timestamp_verbose, + shorten_path, +) +from .utils.logging import get_logger, setup_logging + +app = typer.Typer( + add_completion=False, + no_args_is_help=True, + help="Lokale Suchmaschine fuer transkribierte Hoerspiele und Hoerbuecher.", +) + +console = Console() +logger = get_logger("cli") + + +@dataclass +class AppContext: + """Globaler Zustand der CLI.""" + + config: Config + console: Console + _repository: Repository | None = None + + def repository(self) -> Repository: + if self._repository is None: + self._repository = Repository.open(self.config.database_path) + return self._repository + + def close(self) -> None: + if self._repository is not None: + self._repository.close() + self._repository = None + + +def _fail(message: str, code: int = 1) -> None: + console.print(f"[red]Fehler:[/red] {escape(message)}") + raise typer.Exit(code) + + +def _get_context(ctx: typer.Context) -> AppContext: + if not isinstance(ctx.obj, AppContext): + _fail("Interner Fehler: CLI-Kontext fehlt.") + return ctx.obj + + +@app.callback() +def main( + ctx: typer.Context, + config: Optional[Path] = typer.Option( + None, "--config", "-c", help="Pfad zur config.toml", show_default=False + ), + version: bool = typer.Option(False, "--version", help="Version anzeigen", is_eager=True), +) -> None: + """SjAudioSearch initialisieren.""" + if version: + console.print(f"SjAudioSearch {__version__}") + raise typer.Exit() + + cfg = load_config(config) + setup_logging(cfg.log_path, cfg.logging.level) + app_context = AppContext(config=cfg, console=console) + ctx.obj = app_context + ctx.call_on_close(app_context.close) + + for warning in cfg.load_warnings: + console.print(f"[yellow]Hinweis:[/yellow] {escape(warning)}") + for key in cfg.unknown_keys: + console.print(f"[yellow]Hinweis:[/yellow] unbekannter Konfigurationsschluessel '{escape(key)}'") + + +# ---------------------------------------------------------------------- +# diagnose +# ---------------------------------------------------------------------- +@app.command() +def diagnose( + ctx: typer.Context, + device: Optional[str] = typer.Option(None, help="Zu pruefendes Geraet: cuda oder cpu"), + compute_type: Optional[str] = typer.Option(None, "--compute-type", help="Zu pruefender Compute Type"), +) -> None: + """Zeigt einen vollstaendigen Systembericht (GPU, CUDA, Compute Types).""" + app_context = _get_context(ctx) + report = collect_report( + app_context.config, + requested_device=device, + requested_compute_type=compute_type, + ) + render_report(report, app_context.console) + + +# ---------------------------------------------------------------------- +# scan +# ---------------------------------------------------------------------- +@app.command() +def scan( + ctx: typer.Context, + path: Path = typer.Argument(..., help="Ordner mit Audiodateien"), +) -> None: + """Durchsucht einen Ordner rekursiv und nimmt neue Dateien in den Index auf.""" + app_context = _get_context(ctx) + repository = app_context.repository() + + with Progress( + SpinnerColumn(), + TextColumn("[bold]{task.description}"), + BarColumn(), + MofNCompleteColumn(), + TimeElapsedColumn(), + console=app_context.console, + transient=True, + ) as progress: + task = progress.add_task("Scanne Dateien", total=None) + + def on_progress(current: int, current_path: Path) -> None: + progress.update( + task, + total=max(current, progress.tasks[0].total or 0), + description=f"Scanne {escape(current_path.name)}", + ) + + try: + result = run_scan(path, repository, app_context.config.processing, progress=on_progress) + except (FileNotFoundError, NotADirectoryError) as exc: + _fail(str(exc)) + + table = Table(title="Scan-Ergebnis", show_header=False, box=None) + table.add_column(style="bold cyan") + table.add_column() + table.add_row("Ordner", result.root) + table.add_row("Gefundene Audiodateien", str(result.files_found)) + table.add_row("Neu aufgenommen", str(result.added)) + table.add_row("Geaendert", str(result.changed)) + table.add_row("Unveraendert", str(result.unchanged)) + table.add_row("Duplikate", str(result.duplicates)) + table.add_row("Fehler", str(len(result.errors))) + console.print(table) + + if result.errors: + error_table = Table(title="Dateien mit Problemen") + error_table.add_column("Datei", overflow="fold") + error_table.add_column("Grund", overflow="fold") + for item in result.errors[:15]: + error_table.add_row(shorten_path(item.path, 70), escape(item.message)) + console.print(error_table) + console.print( + "[yellow]Hinweis:[/yellow] Diese Dateien wurden mit Status 'error' gespeichert. " + "Details stehen im Log." + ) + + +# ---------------------------------------------------------------------- +# transcribe +# ---------------------------------------------------------------------- +def _resolve_transcription(ctx_app: AppContext, model: str | None, device: str | None, + compute_type: str | None, language: str | None): + config = ctx_app.config + settings = config.transcription + if model: + settings = replace(settings, model=model) + if language is not None: + settings = replace(settings, language=language) + + requested_device = device or settings.device + requested_compute = compute_type or settings.compute_type + gpu = detect_gpu() + runtime = collect_runtime_info(check_fts=False) + resolved = resolve_execution(requested_device, requested_compute, gpu, runtime) + return settings, resolved, gpu, runtime + + +@app.command() +def transcribe( + ctx: typer.Context, + model: Optional[str] = typer.Option(None, help="Modell, z. B. large-v3, medium, small"), + device: Optional[str] = typer.Option(None, help="cuda oder cpu"), + compute_type: Optional[str] = typer.Option(None, "--compute-type", help="z. B. int8_float32, float32"), + language: Optional[str] = typer.Option(None, help="Sprache, z. B. de oder auto"), + limit: Optional[int] = typer.Option(None, help="Maximale Anzahl Dateien in diesem Lauf"), + force: bool = typer.Option(False, "--force", help="Bereits transkribierte Dateien erneut verarbeiten"), +) -> None: + """Arbeitet die Transkriptionswarteschlange ab.""" + app_context = _get_context(ctx) + repository = app_context.repository() + config = app_context.config + + settings, resolved, gpu, runtime = _resolve_transcription( + app_context, model, device, compute_type, language + ) + + mode_table = Table(show_header=False, box=None, padding=(0, 1)) + mode_table.add_column(style="bold cyan", no_wrap=True) + mode_table.add_column() + mode_table.add_row("Modell", settings.model) + mode_table.add_row("Geraet", f"{resolved.device}" + (" (GPU)" if resolved.device == "cuda" else " (CPU)")) + mode_table.add_row("Compute Type", resolved.compute_type) + mode_table.add_row("Sprache", settings.language_for_whisper or "automatisch") + if gpu is not None: + mode_table.add_row( + "GPU", f"{gpu.name} (CC {gpu.compute_capability or '?'}, {gpu.memory_total_mb or '?'} MB)" + ) + console.print(Panel(mode_table, title="Transkription", border_style="cyan")) + + for note in resolved.notes: + console.print(f"[yellow]Hinweis:[/yellow] {escape(note)}") + if not runtime.cuda_runtime_usable and resolved.device == "cpu": + console.print( + "[yellow]Hinweis:[/yellow] CPU-Modus aktiv. Die Verarbeitung wird deutlich langsamer sein." + ) + + queue = repository.transcription_queue(force=force) + if limit is not None: + queue = queue[: max(0, limit)] + if not queue: + console.print("Keine Dateien in der Warteschlange. Nutze zuerst 'sjaudiosearch scan '.") + return + + engine = FasterWhisperEngine( + settings, + device=resolved.device, + compute_type=resolved.compute_type, + model_dir=config.model_path, + ) + with console.status(f"Lade Modell {settings.model} ...", spinner="dots"): + try: + engine.load() + except TranscriptionError as exc: + if resolved.device == "cuda": + console.print(f"[yellow]GPU-Modus fehlgeschlagen:[/yellow] {escape(str(exc))}") + console.print("[yellow]Wechsle automatisch auf CPU (int8).[/yellow]") + engine = FasterWhisperEngine( + settings, + device="cpu", + compute_type="int8", + model_dir=config.model_path, + ) + resolved = replace(resolved, device="cpu", compute_type="int8") + try: + engine.load() + except TranscriptionError as cpu_exc: + _fail(f"Auch der CPU-Modus ist fehlgeschlagen: {cpu_exc}") + else: + _fail(str(exc)) + + worker = TranscriptionWorker( + repository, + engine, + config.processing, + device=resolved.device, + compute_type=resolved.compute_type, + ) + + columns = [ + SpinnerColumn(), + TextColumn("[bold]{task.description}"), + BarColumn(), + MofNCompleteColumn(), + TimeElapsedColumn(), + TextColumn("{task.fields[rate]}"), + ] + errors: list[tuple[str, str]] = [] + + with Progress(*columns, console=console) as progress: + task = progress.add_task("Starte Transkription", total=len(queue), rate="") + + def on_event(event: WorkerEvent) -> None: + if event.kind == "start" and event.record: + progress.update( + task, description=escape(event.record.filename), rate="" + ) + elif event.kind == "done": + stats = worker.stats + progress.update( + task, + advance=1, + rate=format_rate(stats.audio_seconds, stats.elapsed) if stats.elapsed else "", + ) + elif event.kind == "error" and event.record: + errors.append((event.record.path, event.message or "unbekannter Fehler")) + progress.update(task, advance=1, rate="[red]Fehler[/red]") + + try: + stats = worker.run(limit=limit, force=force, event_callback=on_event) + except KeyboardInterrupt: + console.print() + console.print("[yellow]Abgebrochen.[/yellow] Der aktuelle Stand ist gespeichert, " + "die Unterbrechung wird beim naechsten Lauf fortgesetzt.") + raise typer.Exit(130) + except TranscriptionError as exc: + _fail(str(exc)) + + _render_run_summary(stats, settings.model, resolved.device, resolved.compute_type, errors) + + +def _render_run_summary( + stats: RunStats, model: str, device: str, compute_type: str, errors: list[tuple[str, str]] +) -> None: + table = Table(title="Transkription abgeschlossen", show_header=False, box=None) + table.add_column(style="bold cyan", no_wrap=True) + table.add_column() + table.add_row("Modell", model) + table.add_row("Ausfuehrung", f"{device} / {compute_type}") + table.add_row("Warteschlange", str(stats.queued)) + table.add_row("Erfolgreich", str(stats.processed)) + table.add_row("Fehlgeschlagen", str(stats.failed)) + table.add_row("Verarbeitete Audiodauer", format_duration(stats.audio_seconds)) + table.add_row("Laufzeit", format_duration(stats.elapsed)) + table.add_row("Durchschnitt", f"{format_rate(stats.audio_seconds, stats.elapsed)}") + console.print(table) + + if errors: + error_table = Table(title="Fehlerhafte Dateien") + error_table.add_column("Datei", overflow="fold") + error_table.add_column("Grund", overflow="fold") + for path, message in errors[:15]: + error_table.add_row(shorten_path(path, 70), escape(message)) + console.print(error_table) + console.print("Mit [bold]sjaudiosearch retry[/bold] koennen fehlgeschlagene Dateien erneut versucht werden.") + + +# ---------------------------------------------------------------------- +# search +# ---------------------------------------------------------------------- +@app.command() +def search( + ctx: typer.Context, + query: str = typer.Argument(..., help="Gesuchter Text (auch ungenau)"), + limit: Optional[int] = typer.Option(None, help="Maximale Anzahl Treffer"), + min_score: Optional[float] = typer.Option(None, "--min-score", help="Mindestaehnlichkeit 0-100"), + context_words: Optional[int] = typer.Option(None, "--context-words", help="Woerter vor/nach dem Treffer"), + language: Optional[str] = typer.Option(None, help="Nur Transkripte dieser Sprache"), + json_output: bool = typer.Option(False, "--json", help="Ergebnisse als JSON ausgeben"), + verbose: bool = typer.Option(False, "--verbose", help="Zusaetzliche Details anzeigen"), +) -> None: + """Sucht in allen Transkripten - auch bei ungenauen Erinnerungen.""" + app_context = _get_context(ctx) + repository = app_context.repository() + config = app_context.config + engine = SearchEngine(repository, config.search) + + effective_limit = limit if limit is not None else config.search.default_limit + effective_min = min_score if min_score is not None else config.search.default_min_score + effective_context = context_words if context_words is not None else config.search.context_words + + progress_state: dict[str, int] = {} + + with console.status("[cyan]Suche ...[/cyan]", spinner="dots") as status: + + def on_progress(current: int, total: int) -> None: + progress_state["current"] = current + progress_state["total"] = total + status.update(f"[cyan]Suche ... {current}/{total} Dateien[/cyan]") + + try: + hits = engine.search( + query, + limit=effective_limit, + min_score=effective_min, + context_words=effective_context, + language=language, + progress=on_progress, + ) + except SearchError as exc: + _fail(str(exc)) + except sqlite3.Error as exc: + _fail(f"Datenbankfehler bei der Suche: {exc}") + + if json_output: + payload = { + "query": query, + "limit": effective_limit, + "min_score": effective_min, + "context_words": effective_context, + "count": len(hits), + "results": [hit.to_dict(effective_context) for hit in hits], + } + print(json.dumps(payload, ensure_ascii=False, indent=2)) + return + + console.print(Panel.fit("[bold]SjAudioSearch[/bold]", border_style="cyan")) + console.print(f'Query: [bold]"{escape(query)}"[/bold]') + console.print( + f"[bold]{len(hits)}[/bold] Treffer (Mindestaehnlichkeit {effective_min:.0f}%, " + f"Kontext {effective_context} Woerter)" + ) + if not hits: + console.print() + console.print( + "[yellow]Keine Treffer.[/yellow] Versuche eine hoehere Kontextlaenge, " + "eine niedrigere Mindestaehnlichkeit oder ein anderes Modell." + ) + + for index, hit in enumerate(hits, start=1): + header = f"[bold][{index}] {escape(hit.filename)}[/bold]" + details = [ + f"Similarity: [bold green]{format_score(hit.score)}[/bold green]", + f"Timestamp: [bold]{format_timestamp_verbose(hit.start_time)}[/bold] " + f"(bis {format_timestamp_verbose(hit.end_time)}, " + f"{'Wort' if hit.context.precision == 'word' else 'Segment'}-Genauigkeit)", + ] + if verbose: + details.append( + f"Audio-ID: {hit.audio_id} | Sprache: {hit.language or '?'} | Modell: {escape(hit.model or '?')} | " + f"Quelle: {hit.source}" + ) + body = "\n".join(details) + console.print() + console.print(header) + console.print(body) + console.print() + console.print(Panel(hit.context.as_block(), title="Context", border_style="dim")) + console.print(f"Path: {escape(hit.path)}") + console.print("[dim]" + "-" * 40 + "[/dim]") + + +# ---------------------------------------------------------------------- +# status +# ---------------------------------------------------------------------- +@app.command() +def status(ctx: typer.Context) -> None: + """Zeigt Kennzahlen der Sammlung und den Datenbankzustand.""" + app_context = _get_context(ctx) + repository = app_context.repository() + stats = repository.stats(repository.databases_size()) + config = app_context.config + + table = Table(title="SjAudioSearch Status", show_header=False, box=None) + table.add_column(style="bold cyan", no_wrap=True) + table.add_column() + table.add_row("Audiodateien gesamt", str(stats.total_files)) + table.add_row("Transkribiert", str(stats.done)) + table.add_row("Ausstehend", str(stats.pending)) + table.add_row("In Bearbeitung", str(stats.processing)) + table.add_row("Fehlgeschlagen", str(stats.error)) + table.add_row("Duplikate", str(stats.duplicates)) + table.add_row("Gesamtspieldauer", format_duration(stats.total_duration)) + table.add_row("Transkripte", str(stats.transcripts)) + table.add_row("Segmente", str(stats.segment_count)) + table.add_row("Woerter", str(stats.word_count)) + table.add_row("Datenbank", f"{config.database_path} ({format_size(stats.database_size)})") + model_text = ", ".join(stats.models) if stats.models else f"{config.transcription.model} (noch nicht verwendet)" + table.add_row("Transkriptionsmodell", model_text) + console.print(table) + + if stats.error: + console.print("[yellow]Hinweis:[/yellow] 'sjaudiosearch retry' setzt fehlgeschlagene Dateien zurueck.") + if stats.pending or stats.processing: + console.print("[yellow]Hinweis:[/yellow] 'sjaudiosearch transcribe' arbeitet die Warteschlange ab.") + + +# ---------------------------------------------------------------------- +# list +# ---------------------------------------------------------------------- +@app.command("list") +def list_files( + ctx: typer.Context, + limit: Optional[int] = typer.Option(None, help="Maximale Anzahl Eintraege"), + status_filter: Optional[str] = typer.Option( + None, "--status", help="pending, processing, done oder error" + ), + search: Optional[str] = typer.Option(None, "--search", help="Dateiname/Pfad filtern"), +) -> None: + """Listet bekannte Audiodateien auf.""" + app_context = _get_context(ctx) + repository = app_context.repository() + if status_filter and status_filter not in FileStatus.values(): + _fail(f"Unbekannter Status '{status_filter}'. Erlaubt: {', '.join(FileStatus.values())}") + + records = repository.list_audio(status=status_filter, limit=limit, search=search) + if not records: + console.print("Keine Dateien gefunden. Nutze 'sjaudiosearch scan '.") + return + + table = Table(title=f"Audiodateien ({len(records)})", show_lines=False) + table.add_column("ID", justify="right", style="cyan") + table.add_column("Status") + table.add_column("Dauer", justify="right") + table.add_column("Groesse", justify="right") + table.add_column("Datei", overflow="fold") + table.add_column("Pfad", overflow="fold") + + colors = { + FileStatus.DONE.value: "green", + FileStatus.PENDING.value: "yellow", + FileStatus.PROCESSING.value: "cyan", + FileStatus.ERROR.value: "red", + } + for record in records: + status_text = record.status + if record.duplicate_of: + status_text = f"duplikat von #{record.duplicate_of}" + color = colors.get(record.status, "white") + table.add_row( + str(record.id), + f"[{color}]{status_text}[/{color}]", + format_duration(record.duration), + format_size(record.file_size), + escape(record.filename), + escape(shorten_path(record.path, 70)), + ) + console.print(table) + + +# ---------------------------------------------------------------------- +# info +# ---------------------------------------------------------------------- +@app.command() +def info( + ctx: typer.Context, + audio_id: int = typer.Argument(..., help="ID der Audiodatei"), +) -> None: + """Zeigt Details zu einer Audiodatei und ihrem Transkript.""" + app_context = _get_context(ctx) + repository = app_context.repository() + record = repository.get_audio(audio_id) + if record is None: + _fail(f"Keine Audiodatei mit ID {audio_id} gefunden.") + + table = Table(show_header=False, box=None) + table.add_column(style="bold cyan", no_wrap=True) + table.add_column(overflow="fold") + table.add_row("ID", str(record.id)) + table.add_row("Dateiname", escape(record.filename)) + table.add_row("Pfad", escape(record.path)) + table.add_row("Status", record.status) + table.add_row("Dauer", format_duration(record.duration)) + table.add_row("Groesse", format_size(record.file_size)) + table.add_row("Geaendert", format_datetime(record.updated_at)) + table.add_row("Index-Hash", record.content_hash or "-") + if record.duplicate_of: + table.add_row("Duplikat von", f"#{record.duplicate_of}") + if record.error_message: + table.add_row("Fehler", escape(record.error_message)) + + try: + meta = read_metadata(record.path) + if meta.tags: + tags = ", ".join(f"{key}={value}" for key, value in list(meta.tags.items())[:6]) + table.add_row("Tags", escape(tags)) + except AudioMetadataError: + pass + + console.print(Panel(table, title=f"Audiodatei #{record.id}", border_style="cyan")) + + transcript = repository.get_transcript(record.id) + if transcript is None: + console.print("[yellow]Kein Transkript vorhanden.[/yellow]") + return + + segments = repository.get_segments(record.id) + console.print( + f"Transkript: {len(segments)} Segmente, Modell [bold]{escape(transcript['model'])}[/bold], " + f"Sprache {transcript['language'] or '?'}, erstellt {format_datetime(transcript['created_at'])}" + ) + preview = Table(title="Segmente (Auszug)") + preview.add_column("Start", justify="right") + preview.add_column("Ende", justify="right") + preview.add_column("Text", overflow="fold") + for segment in segments[:20]: + preview.add_row( + format_timestamp_verbose(segment["start_time"]), + format_timestamp_verbose(segment["end_time"]), + escape(segment["text"]), + ) + console.print(preview) + if len(segments) > 20: + console.print(f"[dim]... {len(segments) - 20} weitere Segmente[/dim]") + + +# ---------------------------------------------------------------------- +# play +# ---------------------------------------------------------------------- +@app.command() +def play( + ctx: typer.Context, + audio_id: int = typer.Argument(..., help="ID der Audiodatei"), + time: float = typer.Option(0.0, "--time", help="Startzeit in Sekunden"), +) -> None: + """Spielt eine Audiodatei ab einer Zeitposition ab.""" + app_context = _get_context(ctx) + repository = app_context.repository() + record = repository.get_audio(audio_id) + if record is None: + _fail(f"Keine Audiodatei mit ID {audio_id} gefunden.") + + console.print( + f"Spiele [bold]{escape(record.filename)}[/bold] ab {format_timestamp_verbose(time)} ..." + ) + try: + play_audio(record.path, time) + except PlaybackError as exc: + _fail(str(exc), code=127) + + +# ---------------------------------------------------------------------- +# retry / reindex / remove +# ---------------------------------------------------------------------- +@app.command() +def retry(ctx: typer.Context) -> None: + """Setzt fehlgeschlagene Transkriptionen zurueck auf 'pending'.""" + app_context = _get_context(ctx) + repository = app_context.repository() + count = repository.reset_errors() + console.print(f"{count} Datei(en) zurueckgesetzt. Starte 'sjaudiosearch transcribe'.") + + +@app.command() +def reindex(ctx: typer.Context) -> None: + """Baut den Suchindex aus den vorhandenen Transkripten neu auf.""" + app_context = _get_context(ctx) + repository = app_context.repository() + before = repository.stats().transcripts + with console.status("Baue Suchindex neu ...", spinner="dots"): + indexed = repository.rebuild_search_index() + after = repository.stats().transcripts + if before != after: + _fail(f"Transkriptanzahl hat sich geaendert ({before} -> {after}) - bitte pruefen.") + console.print( + f"Suchindex neu aufgebaut: {indexed} Segmente indexiert. " + f"Transkripte unveraendert ({after})." + ) + + +@app.command() +def remove( + ctx: typer.Context, + audio_id: int = typer.Argument(..., help="ID der Audiodatei"), +) -> None: + """Entfernt eine Datei aus dem Index (die Audiodatei selbst bleibt erhalten).""" + app_context = _get_context(ctx) + repository = app_context.repository() + record = repository.get_audio(audio_id) + if record is None: + _fail(f"Keine Audiodatei mit ID {audio_id} gefunden.") + + if not repository.delete_audio(audio_id): + _fail(f"Datei #{audio_id} konnte nicht entfernt werden.") + + console.print( + f"[green]Entfernt:[/green] #{audio_id} {escape(record.filename)} " + "(Transkript und Suchindex geloescht)" + ) + console.print(f"[dim]Die Originaldatei bleibt erhalten: {escape(record.path)}[/dim]") + + +# ---------------------------------------------------------------------- +# search-index Diagnose +# ---------------------------------------------------------------------- +@app.command("check") +def check(ctx: typer.Context) -> None: + """Prueft Datenbank, FTS5 und Trigramm-Unterstuetzung.""" + app_context = _get_context(ctx) + config = app_context.config + conn = app_context.repository().conn + fts5, trigram = check_fts5(conn) + console.print(f"Datenbank: {config.database_path}") + console.print(f"FTS5: {'OK' if fts5 else 'nicht verfuegbar'}") + console.print(f"Trigramm-Tokenizer: {'OK' if trigram else 'nicht verfuegbar'}") + console.print(f"Logdatei: {config.log_path}") + console.print(f"Konfiguration: {config.source_path or 'eingebaute Defaults'}") + + +def run() -> None: + """Einstiegspunkt fuer ``python -m sjaudiosearch``.""" + try: + app() + except KeyboardInterrupt: + console.print("\n[yellow]Abgebrochen.[/yellow]") + sys.exit(130) + + +if __name__ == "__main__": # pragma: no cover + run() diff --git a/sjaudiosearch/config.py b/sjaudiosearch/config.py new file mode 100644 index 0000000..a10c7fb --- /dev/null +++ b/sjaudiosearch/config.py @@ -0,0 +1,225 @@ +"""Konfigurationsverwaltung fuer SjAudioSearch. + +Die Konfiguration wird aus einer TOML-Datei gelesen. Fehlende Werte werden +durch Defaults ersetzt, unbekannte Schluessel werden gesammelt und gemeldet, +statt stillschweigend ignoriert zu werden. +""" + +from __future__ import annotations + +import os +import tomllib +from dataclasses import dataclass, field, fields +from pathlib import Path +from typing import Any + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_CONFIG_FILENAME = "config.toml" +ENV_CONFIG_PATH = "SJAUDIOSEARCH_CONFIG" + + +def _xdg_dir(env_var: str, fallback: str) -> Path: + """Ermittelt ein XDG-Verzeichnis (Konfiguration bzw. Daten).""" + base = os.environ.get(env_var) + root = Path(base).expanduser() if base else Path.home() / fallback + return root / "sjaudiosearch" + + +# ~/.config/sjaudiosearch/config.toml (bzw. $XDG_CONFIG_HOME/...) +USER_CONFIG_DIR = _xdg_dir("XDG_CONFIG_HOME", ".config") +# Ablage fuer Datenbank und Logs, wenn das Programm systemweit installiert ist. +USER_DATA_DIR = _xdg_dir("XDG_DATA_HOME", ".local/share") +SYSTEM_CONFIG_PATH = Path("/etc/sjaudiosearch/config.toml") + + +@dataclass +class TranscriptionConfig: + model: str = "large-v3" + device: str = "cuda" + compute_type: str = "int8_float32" + language: str | None = "de" + beam_size: int = 5 + word_timestamps: bool = True + vad_filter: bool = True + model_dir: str = "./data/models" + + @property + def language_for_whisper(self) -> str | None: + """Gibt ``None`` zurueck, wenn die Sprache automatisch erkannt werden soll.""" + if self.language is None: + return None + value = self.language.strip().lower() + if value in {"", "auto", "none"}: + return None + return value + + +@dataclass +class SearchConfig: + default_limit: int = 50 + default_min_score: float = 70.0 + context_words: int = 50 + window_before: int = 2 + window_after: int = 4 + candidate_limit: int = 4000 + min_token_overlap: float = 0.34 + exhaustive_fallback: bool = True + max_hits_per_file: int = 5 + + +@dataclass +class ProcessingConfig: + workers: int = 1 + resume: bool = True + compute_hash: str = "auto" + hash_max_size_mb: int = 4096 + + +@dataclass +class DatabaseConfig: + path: str = "./data/sjaudiosearch.db" + + +@dataclass +class LoggingConfig: + level: str = "INFO" + path: str = "./logs/sjaudiosearch.log" + + +@dataclass +class Config: + transcription: TranscriptionConfig = field(default_factory=TranscriptionConfig) + search: SearchConfig = field(default_factory=SearchConfig) + processing: ProcessingConfig = field(default_factory=ProcessingConfig) + database: DatabaseConfig = field(default_factory=DatabaseConfig) + logging: LoggingConfig = field(default_factory=LoggingConfig) + + # Metadaten zur Herkunft der Konfiguration + source_path: Path | None = None + base_dir: Path = field(default_factory=lambda: PROJECT_ROOT) + unknown_keys: list[str] = field(default_factory=list) + load_warnings: list[str] = field(default_factory=list) + + def resolve_path(self, value: str) -> Path: + """Loest einen (moeglicherweise relativen) Pfad gegen das Config-Verzeichnis auf.""" + path = Path(value).expanduser() + if path.is_absolute(): + return path + return (self.base_dir / path).resolve() + + @property + def database_path(self) -> Path: + return self.resolve_path(self.database.path) + + @property + def log_path(self) -> Path: + return self.resolve_path(self.logging.path) + + @property + def model_path(self) -> Path: + return self.resolve_path(self.transcription.model_dir) + + +def _coerce(current: Any, value: Any) -> Any: + """Konvertiert TOML-Werte tolerant auf den Typ des Defaults.""" + if current is None: + return value + if isinstance(current, bool): + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in {"1", "true", "yes", "on"} + return bool(value) + if isinstance(current, int) and not isinstance(current, bool): + try: + return int(value) + except (TypeError, ValueError): + return current + if isinstance(current, float): + try: + return float(value) + except (TypeError, ValueError): + return current + if isinstance(current, str): + return str(value) + return value + + +def _apply_section(target: Any, table: dict[str, Any], section: str, config: Config) -> None: + known = {f.name: f for f in fields(target)} + for key, value in table.items(): + if key not in known: + config.unknown_keys.append(f"{section}.{key}") + continue + current = getattr(target, key) + setattr(target, key, _coerce(current, value)) + + +def _is_writable_dir(path: Path) -> bool: + try: + return path.is_dir() and os.access(path, os.W_OK) + except OSError: + return False + + +def find_config_path(explicit: str | Path | None = None) -> Path | None: + """Ermittelt die zu verwendende Konfigurationsdatei.""" + candidates: list[Path] = [] + if explicit: + candidates.append(Path(explicit).expanduser()) + env_value = os.environ.get(ENV_CONFIG_PATH) + if env_value: + candidates.append(Path(env_value).expanduser()) + candidates.append(Path.cwd() / DEFAULT_CONFIG_FILENAME) + candidates.append(PROJECT_ROOT / DEFAULT_CONFIG_FILENAME) + candidates.append(USER_CONFIG_DIR / DEFAULT_CONFIG_FILENAME) + candidates.append(SYSTEM_CONFIG_PATH) + + for candidate in candidates: + if candidate.is_file(): + return candidate.resolve() + return None + + +def load_config(explicit: str | Path | None = None) -> Config: + """Laedt die Konfiguration; ohne Datei werden die Defaults verwendet.""" + path = find_config_path(explicit) + config = Config() + + if path is None: + # Im Quellbaum (beschreibbar) bleiben Daten im Projekt, nach einer + # systemweiten Installation liegen sie im Benutzerverzeichnis. + config.base_dir = PROJECT_ROOT if _is_writable_dir(PROJECT_ROOT) else USER_DATA_DIR + if explicit: + config.load_warnings.append( + f"Konfigurationsdatei nicht gefunden: {explicit} - es gelten die Defaults." + ) + return config + + config.source_path = path + config.base_dir = path.parent + + try: + with path.open("rb") as handle: + raw = tomllib.load(handle) + except (OSError, tomllib.TOMLDecodeError) as exc: + config.load_warnings.append(f"Konfiguration konnte nicht gelesen werden ({path}): {exc}") + return config + + section_map = { + "transcription": config.transcription, + "search": config.search, + "processing": config.processing, + "database": config.database, + "logging": config.logging, + } + for section_name, section_obj in section_map.items(): + table = raw.get(section_name) + if isinstance(table, dict): + _apply_section(section_obj, table, section_name, config) + + for key in raw: + if key not in section_map: + config.unknown_keys.append(key) + + return config diff --git a/sjaudiosearch/core/audio.py b/sjaudiosearch/core/audio.py new file mode 100644 index 0000000..a78232a --- /dev/null +++ b/sjaudiosearch/core/audio.py @@ -0,0 +1,74 @@ +"""Audio-Wiedergabe und Dekodierpruefungen.""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +from .metadata import AudioMetadataError, read_metadata + + +class PlaybackError(RuntimeError): + """Wiedergabe nicht moeglich.""" + + +def find_player() -> str | None: + """Sucht einen installierten Player (bevorzugt ffplay).""" + for candidate in ("ffplay", "mpv", "vlc"): + executable = shutil.which(candidate) + if executable: + return executable + return None + + +def build_play_command(path: str | Path, start_time: float = 0.0) -> list[str]: + """Baut den Kommandozeilenaufruf zum Abspielen ab einer Zeitposition.""" + player = find_player() + if player is None: + raise PlaybackError( + "Kein Player gefunden. Installiere ffmpeg (ffplay), mpv oder vlc." + ) + name = Path(player).name + start = max(0.0, float(start_time)) + + if name.startswith("ffplay"): + # -nodisp: reine Audiowiedergabe ohne zusaetzliches Fenster. + command = [player, "-nodisp", "-autoexit", "-loglevel", "error"] + if start > 0: + command += ["-ss", f"{start:.3f}"] + command.append(str(path)) + return command + if name.startswith("mpv"): + command = [player] + if start > 0: + command += [f"--start={start:.3f}"] + command.append(str(path)) + return command + # vlc + command = [player] + if start > 0: + command += [f"--start-time={start:.3f}"] + command.append(str(path)) + return command + + +def play(path: str | Path, start_time: float = 0.0) -> None: + """Startet die Wiedergabe und wartet, bis der Player beendet wird.""" + file_path = Path(path) + if not file_path.is_file(): + raise PlaybackError(f"Audiodatei nicht gefunden: {file_path}") + command = build_play_command(file_path, start_time) + try: + subprocess.run(command, check=False) + except OSError as exc: + raise PlaybackError(f"Player konnte nicht gestartet werden: {exc}") from exc + + +def can_decode(path: str | Path) -> bool: + """Prueft grob, ob die Datei als Audio dekodierbar ist.""" + try: + read_metadata(path) + return True + except AudioMetadataError: + return False diff --git a/sjaudiosearch/core/metadata.py b/sjaudiosearch/core/metadata.py new file mode 100644 index 0000000..be9acd8 --- /dev/null +++ b/sjaudiosearch/core/metadata.py @@ -0,0 +1,143 @@ +"""Audio-Metadaten (Dauer, Tags, Format). + +Primaer wird Mutagen verwendet; wenn Mutagen eine Datei nicht lesen kann, wird +PyAV als Fallback genutzt. Beide Wege lesen nur Header bzw. Container, nicht +die komplette Audiodatei. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from ..utils.logging import get_logger + +logger = get_logger("core.metadata") + +SUPPORTED_EXTENSIONS = frozenset( + {".mp3", ".wav", ".flac", ".m4a", ".ogg", ".opus", ".wma"} +) +# WMA ist nur zuverlaessig dekodierbar, wenn ffmpeg/PyAV den Codec kennt. +CONDITIONAL_EXTENSIONS = frozenset({".wma"}) + + +class AudioMetadataError(RuntimeError): + """Die Datei konnte nicht als Audio gelesen werden.""" + + +@dataclass +class AudioInfo: + """Ergebnis der Metadatenanalyse.""" + + path: str + duration: float | None = None + codec: str | None = None + sample_rate: int | None = None + channels: int | None = None + bitrate: int | None = None + tags: dict[str, str] = field(default_factory=dict) + source: str = "mutagen" + + +def is_supported_audio(path: str | Path) -> bool: + return Path(path).suffix.lower() in SUPPORTED_EXTENSIONS + + +def _clean_tags(raw: dict[str, Any]) -> dict[str, str]: + tags: dict[str, str] = {} + for key, value in raw.items(): + try: + if isinstance(value, (list, tuple)): + text = ", ".join(str(item) for item in value if item is not None) + else: + text = str(value) + except Exception: # defekte Tag-Objekte duerfen den Scan nicht stoppen + continue + if text: + tags[str(key).lower()] = text + return tags + + +def _read_with_mutagen(path: Path) -> AudioInfo: + import mutagen + + audio = mutagen.File(str(path)) + if audio is None: + raise AudioMetadataError("Format wurde von Mutagen nicht erkannt") + + info = getattr(audio, "info", None) + duration = getattr(info, "length", None) + if duration is None: + raise AudioMetadataError("Mutagen konnte keine Dauer ermitteln") + + return AudioInfo( + path=str(path), + duration=float(duration), + codec=type(audio).__name__, + sample_rate=getattr(info, "sample_rate", None), + channels=getattr(info, "channels", None), + bitrate=getattr(info, "bitrate", None), + tags=_clean_tags(dict(audio.tags or {})), + source="mutagen", + ) + + +def _read_with_pyav(path: Path) -> AudioInfo: + import av + + try: + with av.open(str(path)) as container: + if not container.streams.audio: + raise AudioMetadataError("Keine Audiospur gefunden") + stream = container.streams.audio[0] + duration = None + if stream.duration is not None and stream.time_base is not None: + duration = float(stream.duration * stream.time_base) + elif container.duration is not None: + duration = float(container.duration / 1_000_000) + if duration is None or duration <= 0: + raise AudioMetadataError("Keine gueltige Dauer ermittelbar") + return AudioInfo( + path=str(path), + duration=duration, + codec=stream.codec_context.name, + sample_rate=stream.codec_context.sample_rate, + channels=stream.codec_context.channels, + bitrate=stream.bit_rate, + source="pyav", + ) + except AudioMetadataError: + raise + except Exception as exc: # PyAV wirft verschiedene Fehlertypen + raise AudioMetadataError(f"PyAV konnte die Datei nicht lesen: {exc}") from exc + + +def read_metadata(path: str | Path) -> AudioInfo: + """Liest Metadaten ueber Mutagen, bei Bedarf ueber PyAV.""" + file_path = Path(path) + if not file_path.is_file(): + raise AudioMetadataError(f"Datei nicht gefunden: {file_path}") + + errors: list[str] = [] + try: + return _read_with_mutagen(file_path) + except Exception as exc: + errors.append(f"mutagen: {exc}") + logger.debug("Mutagen fehlgeschlagen fuer %s: %s", file_path, exc) + + try: + return _read_with_pyav(file_path) + except Exception as exc: + errors.append(f"pyav: {exc}") + logger.debug("PyAV fehlgeschlagen fuer %s: %s", file_path, exc) + + raise AudioMetadataError("; ".join(errors)) + + +def read_duration(path: str | Path) -> float | None: + """Dauer einer Datei oder ``None``, wenn sie nicht ermittelbar ist.""" + try: + return read_metadata(path).duration + except AudioMetadataError: + return None diff --git a/sjaudiosearch/core/models.py b/sjaudiosearch/core/models.py new file mode 100644 index 0000000..dc10bc9 --- /dev/null +++ b/sjaudiosearch/core/models.py @@ -0,0 +1,89 @@ +"""Datenmodelle fuer Audiodateien und Scan-Ergebnisse.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum + + +class FileStatus(str, Enum): + """Transkriptionsstatus einer Audiodatei.""" + + PENDING = "pending" + PROCESSING = "processing" + DONE = "done" + ERROR = "error" + + @classmethod + def values(cls) -> tuple[str, ...]: + return tuple(item.value for item in cls) + + +@dataclass +class AudioFileRecord: + """Repraesentiert eine Zeile aus ``audio_files``.""" + + id: int + path: str + filename: str + file_size: int + modified_at: float + status: str + duration: float | None = None + content_hash: str | None = None + duplicate_of: int | None = None + error_message: str | None = None + created_at: str | None = None + updated_at: str | None = None + + +@dataclass +class ScanCandidate: + """Beim Scan gefundene Datei (noch nicht in der Datenbank).""" + + path: str + filename: str + file_size: int + modified_at: float + duration: float | None = None + content_hash: str | None = None + + +@dataclass +class ScanError: + """Fehler, der beim Einlesen einer Datei aufgetreten ist.""" + + path: str + message: str + + +@dataclass +class ScanResult: + """Zusammenfassung eines Scan-Laufs.""" + + root: str + files_found: int = 0 + added: int = 0 + changed: int = 0 + unchanged: int = 0 + duplicates: int = 0 + errors: list[ScanError] = field(default_factory=list) + skipped_extensions: int = 0 + + +@dataclass +class RepositoryStats: + """Kennzahlen fuer ``sjaudiosearch status``.""" + + total_files: int = 0 + done: int = 0 + pending: int = 0 + processing: int = 0 + error: int = 0 + duplicates: int = 0 + total_duration: float = 0.0 + database_size: int = 0 + segment_count: int = 0 + word_count: int = 0 + transcripts: int = 0 + models: list[str] = field(default_factory=list) diff --git a/sjaudiosearch/core/scanner.py b/sjaudiosearch/core/scanner.py new file mode 100644 index 0000000..ee12195 --- /dev/null +++ b/sjaudiosearch/core/scanner.py @@ -0,0 +1,254 @@ +"""Rekursiver Audio-Scanner. + +Der Scanner nutzt Dateigroesse und Aenderungszeit fuer eine schnelle +Aenderungspruefung. SHA256-Hashes werden nur berechnet, wenn sie fuer die +Duplikaterkennung tatsaechlich gebraucht werden (Modus ``auto``). +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable + +from ..config import ProcessingConfig +from ..database.repository import Repository +from ..utils.hashing import sha256_file +from ..utils.logging import get_logger +from .metadata import AudioMetadataError, is_supported_audio, read_metadata +from .models import FileStatus, ScanCandidate, ScanError, ScanResult + +logger = get_logger("core.scanner") + +ScanProgress = Callable[[int, Path], None] +# Toleranz fuer Dateisystem-Zeitstempel (Sekunden). +MTIME_TOLERANCE = 1.0 + + +@dataclass +class _DiscoveredFile: + path: Path + size: int + mtime: float + duration: float | None = None + content_hash: str | None = None + error: str | None = None + existing_id: int | None = None + changed: bool = False + + +@dataclass +class _ScanState: + new_files: list[_DiscoveredFile] = field(default_factory=list) + changed_files: list[_DiscoveredFile] = field(default_factory=list) + + +def _iter_audio_files(root: Path) -> list[Path]: + found: list[Path] = [] + for dirpath, dirnames, filenames in os.walk(root, followlinks=False): + dirnames.sort() + for filename in sorted(filenames): + if filename.startswith("."): + continue + path = Path(dirpath) / filename + if is_supported_audio(path): + found.append(path) + return found + + +def _should_hash(mode: str) -> bool: + return mode.strip().lower() == "always" + + +def scan( + root: str | Path, + repository: Repository, + processing: ProcessingConfig | None = None, + *, + progress: ScanProgress | None = None, +) -> ScanResult: + """Scannt ``root`` rekursiv und aktualisiert die Datenbank.""" + processing = processing or ProcessingConfig() + root_path = Path(root).expanduser().resolve() + if not root_path.exists(): + raise FileNotFoundError(f"Pfad existiert nicht: {root_path}") + if not root_path.is_dir(): + raise NotADirectoryError(f"Pfad ist kein Ordner: {root_path}") + + result = ScanResult(root=str(root_path)) + state = _ScanState() + + files = _iter_audio_files(root_path) + result.files_found = len(files) + + for index, path in enumerate(files, start=1): + if progress: + progress(index, path) + try: + stat = path.stat() + except OSError as exc: + result.errors.append(ScanError(str(path), f"Dateizugriff fehlgeschlagen: {exc}")) + continue + + existing = repository.get_audio_by_path(str(path)) + discovered = _DiscoveredFile( + path=path, + size=int(stat.st_size), + mtime=float(stat.st_mtime), + existing_id=existing.id if existing else None, + ) + + if existing is not None: + modified = abs(float(existing.modified_at) - discovered.mtime) > MTIME_TOLERANCE + if int(existing.file_size) != discovered.size or modified: + discovered.changed = True + state.changed_files.append(discovered) + else: + result.unchanged += 1 + continue + else: + state.new_files.append(discovered) + + try: + info = read_metadata(path) + discovered.duration = info.duration + except AudioMetadataError as exc: + discovered.error = str(exc) + result.errors.append(ScanError(str(path), str(exc))) + logger.warning("Metadaten konnten nicht gelesen werden: %s (%s)", path, exc) + + _assign_hashes(state, repository, processing) + _insert_new_files(state.new_files, repository, processing, result) + _update_changed_files(state.changed_files, repository, processing, result) + + logger.info( + "Scan abgeschlossen: %d gefunden, %d neu, %d geaendert, %d unveraendert, %d Duplikate, %d Fehler", + result.files_found, + result.added, + result.changed, + result.unchanged, + result.duplicates, + len(result.errors), + ) + return result + + +def _assign_hashes( + state: _ScanState, repository: Repository, processing: ProcessingConfig +) -> None: + """Berechnet Hashes nur dort, wo Duplikate moeglich sind.""" + mode = (processing.compute_hash or "auto").strip().lower() + if mode == "never": + return + + candidates = state.new_files + state.changed_files + if not candidates: + return + + limit_bytes = max(0, int(processing.hash_max_size_mb)) * 1024 * 1024 + + if _should_hash(mode): + targets = [item for item in candidates if item.error is None] + else: + size_counts: dict[int, int] = {} + for item in candidates: + size_counts[item.size] = size_counts.get(item.size, 0) + 1 + targets = [] + for item in candidates: + if item.error is not None: + continue + needs_check = size_counts.get(item.size, 0) > 1 + if not needs_check: + needs_check = bool(repository.paths_with_size(item.size)) + if needs_check: + targets.append(item) + + for item in targets: + if limit_bytes and item.size > limit_bytes: + logger.info("Datei zu gross fuer Hashing, wird uebersprungen: %s", item.path) + continue + try: + item.content_hash = sha256_file(item.path) + except OSError as exc: + logger.warning("Hashing fehlgeschlagen fuer %s: %s", item.path, exc) + + +def _insert_new_files( + files: list[_DiscoveredFile], + repository: Repository, + processing: ProcessingConfig, + result: ScanResult, +) -> None: + seen_hashes: dict[str, int] = {} + for item in files: + duplicate_of: int | None = None + status = FileStatus.PENDING.value + if item.content_hash: + existing = repository.find_by_hash(item.content_hash) + if existing is not None: + duplicate_of = existing.id + elif item.content_hash in seen_hashes: + duplicate_of = seen_hashes[item.content_hash] + if item.error is not None and duplicate_of is None: + status = FileStatus.ERROR.value + + candidate = ScanCandidate( + path=str(item.path), + filename=item.path.name, + file_size=item.size, + modified_at=item.mtime, + duration=item.duration, + content_hash=item.content_hash, + ) + audio_id = repository.insert_audio( + candidate, + status=status, + duplicate_of=duplicate_of, + error_message=item.error, + ) + result.added += 1 + if duplicate_of is not None: + result.duplicates += 1 + elif item.content_hash: + seen_hashes[item.content_hash] = audio_id + + +def _update_changed_files( + files: list[_DiscoveredFile], + repository: Repository, + processing: ProcessingConfig, + result: ScanResult, +) -> None: + for item in files: + assert item.existing_id is not None + duplicate_of: int | None = None + status = FileStatus.PENDING.value + if item.content_hash: + existing = repository.find_by_hash(item.content_hash, exclude_id=item.existing_id) + if existing is not None: + duplicate_of = existing.id + if item.error is not None and duplicate_of is None: + status = FileStatus.ERROR.value + + candidate = ScanCandidate( + path=str(item.path), + filename=item.path.name, + file_size=item.size, + modified_at=item.mtime, + duration=item.duration, + content_hash=item.content_hash, + ) + repository.update_scan_data( + item.existing_id, + candidate, + status=status, + duplicate_of=duplicate_of, + error_message=item.error, + ) + result.changed += 1 + if duplicate_of is not None: + result.duplicates += 1 + + if files: + logger.info("%d geaenderte Dateien zur erneuten Verarbeitung vorgemerkt", len(files)) diff --git a/sjaudiosearch/database/connection.py b/sjaudiosearch/database/connection.py new file mode 100644 index 0000000..36b0430 --- /dev/null +++ b/sjaudiosearch/database/connection.py @@ -0,0 +1,63 @@ +"""Verbindungsaufbau fuer die SQLite-Datenbank.""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +from ..utils.logging import get_logger + +logger = get_logger("database") + + +def connect(db_path: str | Path, *, create: bool = True) -> sqlite3.Connection: + """Oeffnet die Datenbank mit sinnvollen PRAGMAs. + + ``create=False`` verhindert, dass versehentlich eine neue Datenbankdatei + angelegt wird (z. B. bei reiner Diagnose). + """ + path = Path(db_path) + if not create and not path.exists(): + raise FileNotFoundError(f"Datenbank nicht gefunden: {path}") + + path.parent.mkdir(parents=True, exist_ok=True) + + conn = sqlite3.connect(str(path), timeout=30.0) + conn.row_factory = sqlite3.Row + conn.isolation_level = "" # explizite Transaktionen via "with conn:" + + conn.execute("PRAGMA foreign_keys = ON") + conn.execute("PRAGMA journal_mode = WAL") + conn.execute("PRAGMA synchronous = NORMAL") + conn.execute("PRAGMA busy_timeout = 30000") + conn.execute("PRAGMA temp_store = MEMORY") + + return conn + + +def check_fts5(conn: sqlite3.Connection) -> tuple[bool, bool]: + """Prueft, ob FTS5 und der Trigram-Tokenizer verfuegbar sind.""" + fts5 = False + trigram = False + try: + conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS _fts_check USING fts5(x)") + conn.execute("DROP TABLE IF EXISTS _fts_check") + fts5 = True + except sqlite3.OperationalError as exc: + logger.warning("FTS5 nicht verfuegbar: %s", exc) + return False, False + + if fts5: + try: + conn.execute( + "CREATE VIRTUAL TABLE IF NOT EXISTS _trigram_check USING fts5(x, tokenize='trigram')" + ) + conn.execute("DROP TABLE IF EXISTS _trigram_check") + trigram = True + except sqlite3.OperationalError as exc: + logger.warning("FTS5-Trigram-Tokenizer nicht verfuegbar: %s", exc) + return fts5, trigram + + +def sqlite_version() -> str: + return sqlite3.sqlite_version diff --git a/sjaudiosearch/database/migrations.py b/sjaudiosearch/database/migrations.py new file mode 100644 index 0000000..6459624 --- /dev/null +++ b/sjaudiosearch/database/migrations.py @@ -0,0 +1,55 @@ +"""Schema-Migrationen fuer SjAudioSearch. + +Die Version wird in ``PRAGMA user_version`` gespeichert. Jede Migration laeuft +in einer eigenen Transaktion, damit ein Abbruch keinen halb migrierten Zustand +hinterlaesst. +""" + +from __future__ import annotations + +import sqlite3 + +from ..utils.logging import get_logger +from .schema import SCHEMA_VERSION, TABLES_SQL + +logger = get_logger("database.migrations") + + +def current_version(conn: sqlite3.Connection) -> int: + row = conn.execute("PRAGMA user_version").fetchone() + return int(row[0]) if row else 0 + + +def _migration_1(conn: sqlite3.Connection) -> None: + conn.executescript(TABLES_SQL) + + +MIGRATIONS: list[tuple[int, object]] = [ + (1, _migration_1), +] + + +def migrate(conn: sqlite3.Connection, target: int = SCHEMA_VERSION) -> int: + """Fuehrt alle ausstehenden Migrationen aus und liefert die neue Version.""" + version = current_version(conn) + if version > target: + raise RuntimeError( + f"Datenbankversion {version} ist neuer als die unterstuetzte Version {target}." + ) + if version == target: + return version + + for migration_version, migration in MIGRATIONS: + if migration_version <= version: + continue + logger.info("Migration %s wird angewendet", migration_version) + with conn: + migration(conn) # type: ignore[operator] + conn.execute(f"PRAGMA user_version = {int(migration_version)}") + version = migration_version + + if version != target: + raise RuntimeError( + f"Migration unvollstaendig: Version {version}, erwartet {target}." + ) + return version diff --git a/sjaudiosearch/database/repository.py b/sjaudiosearch/database/repository.py new file mode 100644 index 0000000..e3f0acb --- /dev/null +++ b/sjaudiosearch/database/repository.py @@ -0,0 +1,547 @@ +"""Datenbankzugriff fuer SjAudioSearch. + +Alle Schreibvorgaenge laufen in expliziten Transaktionen, damit ein Abbruch +keine halben Transkripte hinterlaesst. +""" + +from __future__ import annotations + +import sqlite3 +from collections.abc import Iterator, Sequence +from pathlib import Path +from typing import Any + +from ..core.models import AudioFileRecord, FileStatus, RepositoryStats, ScanCandidate +from ..search.fuzzy import normalize_text +from ..utils.formatting import utc_now_iso +from ..utils.logging import get_logger +from .connection import connect +from .migrations import migrate + +logger = get_logger("database.repository") + +_AUDIO_COLUMNS = """ + id, path, filename, duration, file_size, modified_at, content_hash, + status, duplicate_of, error_message, created_at, updated_at +""" + + +def _row_to_audio(row: sqlite3.Row) -> AudioFileRecord: + return AudioFileRecord( + id=row["id"], + path=row["path"], + filename=row["filename"], + duration=row["duration"], + file_size=row["file_size"], + modified_at=row["modified_at"], + content_hash=row["content_hash"], + status=row["status"], + duplicate_of=row["duplicate_of"], + error_message=row["error_message"], + created_at=row["created_at"], + updated_at=row["updated_at"], + ) + + +class Repository: + """Kapselt saemtliche SQL-Zugriffe.""" + + def __init__(self, connection: sqlite3.Connection): + self.conn = connection + + @classmethod + def open(cls, db_path: str | Path) -> "Repository": + conn = connect(db_path) + migrate(conn) + return cls(conn) + + def close(self) -> None: + self.conn.close() + + def __enter__(self) -> "Repository": + return self + + def __exit__(self, *exc_info: object) -> None: + self.close() + + # ------------------------------------------------------------------ + # audio_files + # ------------------------------------------------------------------ + def get_audio(self, audio_id: int) -> AudioFileRecord | None: + row = self.conn.execute( + f"SELECT {_AUDIO_COLUMNS} FROM audio_files WHERE id = ?", (audio_id,) + ).fetchone() + return _row_to_audio(row) if row else None + + def get_audio_by_path(self, path: str) -> AudioFileRecord | None: + row = self.conn.execute( + f"SELECT {_AUDIO_COLUMNS} FROM audio_files WHERE path = ?", (str(path),) + ).fetchone() + return _row_to_audio(row) if row else None + + def list_audio( + self, + *, + status: str | None = None, + limit: int | None = None, + search: str | None = None, + include_duplicates: bool = True, + ) -> list[AudioFileRecord]: + clauses: list[str] = [] + params: list[Any] = [] + if status: + clauses.append("status = ?") + params.append(status) + if not include_duplicates: + clauses.append("duplicate_of IS NULL") + if search: + clauses.append("(filename LIKE ? OR path LIKE ?)") + like = f"%{search}%" + params.extend([like, like]) + where = f"WHERE {' AND '.join(clauses)}" if clauses else "" + sql = f"SELECT {_AUDIO_COLUMNS} FROM audio_files {where} ORDER BY filename COLLATE NOCASE, id" + if limit: + sql += " LIMIT ?" + params.append(int(limit)) + rows = self.conn.execute(sql, params).fetchall() + return [_row_to_audio(row) for row in rows] + + def insert_audio( + self, + candidate: ScanCandidate, + *, + status: str = FileStatus.PENDING.value, + duplicate_of: int | None = None, + error_message: str | None = None, + ) -> int: + now = utc_now_iso() + with self.conn: + cursor = self.conn.execute( + """ + INSERT INTO audio_files + (path, filename, duration, file_size, modified_at, content_hash, + status, duplicate_of, error_message, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + candidate.path, + candidate.filename, + candidate.duration, + int(candidate.file_size), + float(candidate.modified_at), + candidate.content_hash, + status, + duplicate_of, + error_message, + now, + now, + ), + ) + return int(cursor.lastrowid) + + def update_scan_data( + self, + audio_id: int, + candidate: ScanCandidate, + *, + status: str = FileStatus.PENDING.value, + duplicate_of: int | None = None, + error_message: str | None = None, + ) -> None: + with self.conn: + self.conn.execute( + """ + UPDATE audio_files + SET duration = ?, file_size = ?, modified_at = ?, content_hash = ?, + status = ?, duplicate_of = ?, error_message = ?, updated_at = ? + WHERE id = ? + """, + ( + candidate.duration, + int(candidate.file_size), + float(candidate.modified_at), + candidate.content_hash, + status, + duplicate_of, + error_message, + utc_now_iso(), + audio_id, + ), + ) + + def set_duration(self, audio_id: int, duration: float | None) -> None: + if duration is None: + return + with self.conn: + self.conn.execute( + "UPDATE audio_files SET duration = ?, updated_at = ? WHERE id = ? AND duration IS NULL", + (float(duration), utc_now_iso(), audio_id), + ) + + def set_status(self, audio_id: int, status: str, error_message: str | None = None) -> None: + with self.conn: + self.conn.execute( + "UPDATE audio_files SET status = ?, error_message = ?, updated_at = ? WHERE id = ?", + (status, error_message, utc_now_iso(), audio_id), + ) + + def reset_stale_processing(self) -> int: + """Setzt abgebrochene 'processing'-Eintraege nach einem Absturz zurueck.""" + with self.conn: + cursor = self.conn.execute( + "UPDATE audio_files SET status = 'pending', updated_at = ? WHERE status = 'processing'", + (utc_now_iso(),), + ) + return cursor.rowcount or 0 + + def reset_errors(self) -> int: + """Setzt fehlgeschlagene Transkriptionen zurueck auf 'pending'.""" + with self.conn: + cursor = self.conn.execute( + """ + UPDATE audio_files + SET status = 'pending', error_message = NULL, updated_at = ? + WHERE status = 'error' + """, + (utc_now_iso(),), + ) + return cursor.rowcount or 0 + + def transcription_queue(self, *, force: bool = False) -> list[AudioFileRecord]: + if force: + sql = f""" + SELECT {_AUDIO_COLUMNS} FROM audio_files + WHERE duplicate_of IS NULL + ORDER BY id + """ + rows = self.conn.execute(sql).fetchall() + else: + sql = f""" + SELECT {_AUDIO_COLUMNS} FROM audio_files + WHERE duplicate_of IS NULL AND status IN ('pending', 'processing') + ORDER BY id + """ + rows = self.conn.execute(sql).fetchall() + return [_row_to_audio(row) for row in rows] + + def find_by_hash(self, content_hash: str, *, exclude_id: int | None = None) -> AudioFileRecord | None: + sql = f"SELECT {_AUDIO_COLUMNS} FROM audio_files WHERE content_hash = ? AND duplicate_of IS NULL" + params: list[Any] = [content_hash] + if exclude_id is not None: + sql += " AND id != ?" + params.append(exclude_id) + row = self.conn.execute(sql + " ORDER BY id LIMIT 1", params).fetchone() + return _row_to_audio(row) if row else None + + def paths_with_size(self, size: int) -> list[AudioFileRecord]: + rows = self.conn.execute( + f"SELECT {_AUDIO_COLUMNS} FROM audio_files WHERE file_size = ? AND content_hash IS NOT NULL", + (int(size),), + ).fetchall() + return [_row_to_audio(row) for row in rows] + + def count_by_status(self) -> dict[str, int]: + rows = self.conn.execute( + "SELECT status, COUNT(*) AS amount FROM audio_files GROUP BY status" + ).fetchall() + return {row["status"]: int(row["amount"]) for row in rows} + + def stats(self, db_size_bytes: int = 0) -> RepositoryStats: + counts = self.count_by_status() + total_row = self.conn.execute( + "SELECT COUNT(*) AS total, COALESCE(SUM(duration), 0) AS duration FROM audio_files" + ).fetchone() + dup_row = self.conn.execute( + "SELECT COUNT(*) AS amount FROM audio_files WHERE duplicate_of IS NOT NULL" + ).fetchone() + segment_row = self.conn.execute("SELECT COUNT(*) AS amount FROM segments").fetchone() + word_row = self.conn.execute("SELECT COUNT(*) AS amount FROM words").fetchone() + transcript_row = self.conn.execute("SELECT COUNT(*) AS amount FROM transcripts").fetchone() + model_rows = self.conn.execute("SELECT DISTINCT model FROM transcripts").fetchall() + return RepositoryStats( + total_files=int(total_row["total"]), + done=counts.get("done", 0), + pending=counts.get("pending", 0), + processing=counts.get("processing", 0), + error=counts.get("error", 0), + duplicates=int(dup_row["amount"]), + total_duration=float(total_row["duration"] or 0.0), + database_size=int(db_size_bytes), + segment_count=int(segment_row["amount"]), + word_count=int(word_row["amount"]), + transcripts=int(transcript_row["amount"]), + models=[row["model"] for row in model_rows], + ) + + def databases_size(self) -> int: + row = self.conn.execute("PRAGMA database_list").fetchone() + if not row: + return 0 + path = Path(row["file"]) + size = path.stat().st_size if path.exists() else 0 + wal = path.with_name(path.name + "-wal") + if wal.exists(): + size += wal.stat().st_size + return size + + # ------------------------------------------------------------------ + # Transkripte + # ------------------------------------------------------------------ + def save_transcript( + self, + audio_id: int, + *, + language: str | None, + model: str, + full_text: str, + segments: Sequence[Any], + duration: float | None = None, + ) -> int: + """Speichert ein Transkript samt Segmenten, Woertern und Suchindex. + + Vorhandene Daten der Datei werden ersetzt. Alles geschieht in einer + Transaktion, damit ein Abbruch keinen Teilzustand hinterlaesst. + """ + now = utc_now_iso() + with self.conn: + self.conn.execute("DELETE FROM transcripts WHERE audio_id = ?", (audio_id,)) + self.conn.execute("DELETE FROM search_index WHERE audio_id = ?", (audio_id,)) + self.conn.execute("DELETE FROM search_trigram WHERE audio_id = ?", (audio_id,)) + + cursor = self.conn.execute( + """ + INSERT INTO transcripts (audio_id, language, model, full_text, created_at) + VALUES (?, ?, ?, ?, ?) + """, + (audio_id, language, model, full_text, now), + ) + transcript_id = int(cursor.lastrowid) + + for segment in segments: + cursor = self.conn.execute( + """ + INSERT INTO segments (transcript_id, start_time, end_time, text, confidence) + VALUES (?, ?, ?, ?, ?) + """, + ( + transcript_id, + float(segment.start), + float(segment.end), + segment.text, + getattr(segment, "confidence", None), + ), + ) + segment_id = int(cursor.lastrowid) + + words = getattr(segment, "words", None) or [] + if words: + self.conn.executemany( + "INSERT INTO words (segment_id, word, start_time, end_time) VALUES (?, ?, ?, ?)", + [ + ( + segment_id, + word.word, + getattr(word, "start", None), + getattr(word, "end", None), + ) + for word in words + ], + ) + + normalized = normalize_text(segment.text) + if normalized: + self.conn.execute( + """ + INSERT INTO search_index (text, audio_id, transcript_id, segment_id) + VALUES (?, ?, ?, ?) + """, + (normalized, audio_id, transcript_id, segment_id), + ) + self.conn.execute( + "INSERT INTO search_trigram (text, audio_id, segment_id) VALUES (?, ?, ?)", + (normalized, audio_id, segment_id), + ) + + if duration is not None: + self.conn.execute( + """ + UPDATE audio_files + SET duration = COALESCE(duration, ?), status = 'done', + error_message = NULL, updated_at = ? + WHERE id = ? + """, + (float(duration), now, audio_id), + ) + else: + self.conn.execute( + """ + UPDATE audio_files + SET status = 'done', error_message = NULL, updated_at = ? + WHERE id = ? + """, + (now, audio_id), + ) + return transcript_id + + def get_transcript(self, audio_id: int) -> sqlite3.Row | None: + return self.conn.execute( + "SELECT * FROM transcripts WHERE audio_id = ?", (audio_id,) + ).fetchone() + + def get_segments(self, audio_id: int) -> list[sqlite3.Row]: + return self.conn.execute( + """ + SELECT s.id, s.start_time, s.end_time, s.text, s.confidence + FROM segments s + JOIN transcripts t ON t.id = s.transcript_id + WHERE t.audio_id = ? + ORDER BY s.start_time, s.id + """, + (audio_id,), + ).fetchall() + + def get_words(self, segment_ids: Sequence[int]) -> dict[int, list[sqlite3.Row]]: + result: dict[int, list[sqlite3.Row]] = {} + ids = list(segment_ids) + for start in range(0, len(ids), 500): + chunk = ids[start : start + 500] + placeholders = ",".join("?" for _ in chunk) + rows = self.conn.execute( + f""" + SELECT id, segment_id, word, start_time, end_time + FROM words WHERE segment_id IN ({placeholders}) + ORDER BY segment_id, start_time, id + """, + chunk, + ).fetchall() + for row in rows: + result.setdefault(int(row["segment_id"]), []).append(row) + return result + + def segment_texts(self, audio_id: int) -> list[str]: + return [row["text"] for row in self.get_segments(audio_id)] + + def iter_audio_ids_with_transcripts(self, language: str | None = None) -> Iterator[int]: + sql = "SELECT DISTINCT audio_id FROM transcripts" + params: list[Any] = [] + if language: + sql += " WHERE language = ?" + params.append(language) + sql += " ORDER BY audio_id" + cursor = self.conn.execute(sql, params) + for row in cursor: + yield int(row["audio_id"]) + + def transcript_languages(self) -> list[str]: + rows = self.conn.execute( + "SELECT DISTINCT language FROM transcripts WHERE language IS NOT NULL ORDER BY language" + ).fetchall() + return [row["language"] for row in rows] + + # ------------------------------------------------------------------ + # Suche + # ------------------------------------------------------------------ + def fts_candidates(self, match_query: str, limit: int) -> list[tuple[int, int]]: + """Liefert (audio_id, segment_id) fuer einen FTS5-Ausdruck.""" + if not match_query: + return [] + try: + rows = self.conn.execute( + """ + SELECT audio_id, segment_id, bm25(search_index) AS rank + FROM search_index + WHERE search_index MATCH ? + ORDER BY rank + LIMIT ? + """, + (match_query, int(limit)), + ).fetchall() + except sqlite3.OperationalError as exc: + logger.warning("FTS-Anfrage fehlgeschlagen (%s): %s", match_query, exc) + return [] + return [(int(row["audio_id"]), int(row["segment_id"])) for row in rows] + + def trigram_candidates(self, terms: Sequence[str], limit: int) -> list[tuple[int, int]]: + """Teilstring-Kandidaten ueber den FTS5-Trigram-Index.""" + results: list[tuple[int, int]] = [] + seen: set[tuple[int, int]] = set() + for term in terms: + escaped = '"' + term.replace('"', '""') + '"' + try: + rows = self.conn.execute( + """ + SELECT audio_id, segment_id FROM search_trigram + WHERE search_trigram MATCH ? + LIMIT ? + """, + (escaped, int(limit)), + ).fetchall() + except sqlite3.OperationalError as exc: + logger.warning("Trigramm-Anfrage fehlgeschlagen (%s): %s", term, exc) + continue + for row in rows: + key = (int(row["audio_id"]), int(row["segment_id"])) + if key not in seen: + seen.add(key) + results.append(key) + if len(results) >= limit: + return results + return results + + def timeline(self, audio_id: int) -> tuple[list[sqlite3.Row], dict[int, list[sqlite3.Row]]]: + segments = self.get_segments(audio_id) + words = self.get_words([int(segment["id"]) for segment in segments]) + return segments, words + + def rebuild_search_index(self) -> int: + """Baut den FTS-Index aus den gespeicherten Segmenten neu auf.""" + with self.conn: + self.conn.execute("DELETE FROM search_index") + self.conn.execute("DELETE FROM search_trigram") + rows = self.conn.execute( + """ + SELECT s.id AS segment_id, t.audio_id AS audio_id, t.id AS transcript_id, s.text + FROM segments s + JOIN transcripts t ON t.id = s.transcript_id + ORDER BY s.id + """ + ).fetchall() + index_rows = [] + trigram_rows = [] + for row in rows: + normalized = normalize_text(row["text"]) + if not normalized: + continue + index_rows.append( + (normalized, int(row["audio_id"]), int(row["transcript_id"]), int(row["segment_id"])) + ) + trigram_rows.append((normalized, int(row["audio_id"]), int(row["segment_id"]))) + self.conn.executemany( + "INSERT INTO search_index (text, audio_id, transcript_id, segment_id) VALUES (?, ?, ?, ?)", + index_rows, + ) + self.conn.executemany( + "INSERT INTO search_trigram (text, audio_id, segment_id) VALUES (?, ?, ?)", + trigram_rows, + ) + return len(index_rows) + + def delete_audio(self, audio_id: int) -> bool: + """Entfernt eine Datei aus dem Index (die Audiodatei bleibt unberuehrt).""" + with self.conn: + self.conn.execute("DELETE FROM search_index WHERE audio_id = ?", (audio_id,)) + self.conn.execute("DELETE FROM search_trigram WHERE audio_id = ?", (audio_id,)) + cursor = self.conn.execute("DELETE FROM audio_files WHERE id = ?", (audio_id,)) + return (cursor.rowcount or 0) > 0 + + def upsert_meta(self, key: str, value: str) -> None: + with self.conn: + self.conn.execute( + "INSERT INTO meta (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (key, value), + ) + + def get_meta(self, key: str) -> str | None: + row = self.conn.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone() + return row["value"] if row else None diff --git a/sjaudiosearch/database/schema.py b/sjaudiosearch/database/schema.py new file mode 100644 index 0000000..aa07744 --- /dev/null +++ b/sjaudiosearch/database/schema.py @@ -0,0 +1,80 @@ +"""SQL-Schema fuer SjAudioSearch (SQLite + FTS5).""" + +from __future__ import annotations + +SCHEMA_VERSION = 1 + +TABLES_SQL = """ +CREATE TABLE IF NOT EXISTS audio_files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL UNIQUE, + filename TEXT NOT NULL, + duration REAL, + file_size INTEGER NOT NULL, + modified_at REAL NOT NULL, + content_hash TEXT, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'processing', 'done', 'error')), + duplicate_of INTEGER REFERENCES audio_files(id) ON DELETE SET NULL, + error_message TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_audio_status ON audio_files(status); +CREATE INDEX IF NOT EXISTS idx_audio_hash ON audio_files(content_hash); +CREATE INDEX IF NOT EXISTS idx_audio_size ON audio_files(file_size); +CREATE INDEX IF NOT EXISTS idx_audio_duplicate ON audio_files(duplicate_of); + +CREATE TABLE IF NOT EXISTS transcripts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + audio_id INTEGER NOT NULL UNIQUE REFERENCES audio_files(id) ON DELETE CASCADE, + language TEXT, + model TEXT NOT NULL, + full_text TEXT NOT NULL, + created_at TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_transcripts_language ON transcripts(language); + +CREATE TABLE IF NOT EXISTS segments ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transcript_id INTEGER NOT NULL REFERENCES transcripts(id) ON DELETE CASCADE, + start_time REAL NOT NULL, + end_time REAL NOT NULL, + text TEXT NOT NULL, + confidence REAL +); + +CREATE INDEX IF NOT EXISTS idx_segments_transcript ON segments(transcript_id, start_time); + +CREATE TABLE IF NOT EXISTS words ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + segment_id INTEGER NOT NULL REFERENCES segments(id) ON DELETE CASCADE, + word TEXT NOT NULL, + start_time REAL, + end_time REAL +); + +CREATE INDEX IF NOT EXISTS idx_words_segment ON words(segment_id); + +CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5( + text, + audio_id UNINDEXED, + transcript_id UNINDEXED, + segment_id UNINDEXED, + tokenize = "unicode61 remove_diacritics 2" +); + +CREATE VIRTUAL TABLE IF NOT EXISTS search_trigram USING fts5( + text, + audio_id UNINDEXED, + segment_id UNINDEXED, + tokenize = "trigram" +); + +CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT +); +""" diff --git a/sjaudiosearch/search/context.py b/sjaudiosearch/search/context.py new file mode 100644 index 0000000..bcb3852 --- /dev/null +++ b/sjaudiosearch/search/context.py @@ -0,0 +1,186 @@ +"""Kontextaufbau fuer Suchtreffer. + +Aus den gespeicherten Segmenten wird eine durchgehende Wort-Timeline gebaut. +Dadurch gehen an Segmentgrenzen keine Informationen verloren, und der Kontext +kann mitten in einem Segment beginnen und enden. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Sequence + +from .fuzzy import normalize_text, tokenize_original + + +@dataclass +class TokenRef: + """Ein Wort im Transkript mit Zeitbezug und Herkunft.""" + + text: str + norm: str + start: float + end: float + segment_id: int + precision: str # "word" oder "segment" + + +@dataclass +class Timeline: + """Durchgehende Wortfolge einer Audiodatei.""" + + audio_id: int + tokens: list[TokenRef] = field(default_factory=list) + language: str | None = None + model: str | None = None + + @property + def normalized_tokens(self) -> list[str]: + return [token.norm for token in self.tokens] + + @property + def normalized_text(self) -> str: + return " ".join(token.norm for token in self.tokens) + + @property + def original_text(self) -> str: + return " ".join(token.text for token in self.tokens) + + def __len__(self) -> int: + return len(self.tokens) + + +def _distribute_times( + segment_start: float, segment_end: float, token_count: int +) -> list[tuple[float, float]]: + """Verteilt die Segmentzeit gleichmaessig auf die Woerter (Schaetzung).""" + if token_count <= 0: + return [] + span = max(0.0, float(segment_end) - float(segment_start)) + step = span / token_count if span else 0.0 + times: list[tuple[float, float]] = [] + for index in range(token_count): + start = float(segment_start) + step * index + end = float(segment_start) + step * (index + 1) + times.append((round(start, 3), round(end, 3))) + return times + + +def build_timeline( + audio_id: int, + segments: Sequence[Any], + words_by_segment: dict[int, Sequence[Any]] | None = None, + *, + language: str | None = None, + model: str | None = None, +) -> Timeline: + """Baut die Timeline aus Segmenten (sqlite3.Row oder Objekte mit Attributen).""" + timeline = Timeline(audio_id=audio_id, language=language, model=model) + words_by_segment = words_by_segment or {} + + for segment in segments: + if isinstance(segment, dict): + segment_id = int(segment["id"]) + start = float(segment["start_time"]) + end = float(segment["end_time"]) + text = segment["text"] + else: + segment_id = int(segment["id"]) + start = float(segment["start_time"]) + end = float(segment["end_time"]) + text = segment["text"] + + original_tokens = tokenize_original(text) + if not original_tokens: + continue + + words = list(words_by_segment.get(segment_id, [])) + use_words = len(words) == len(original_tokens) + + if use_words: + for token, word in zip(original_tokens, words): + word_start = word["start_time"] if not isinstance(word, dict) else word.get("start_time") + word_end = word["end_time"] if not isinstance(word, dict) else word.get("end_time") + if word_start is None or word_end is None: + use_words = False + break + + if use_words: + for token, word in zip(original_tokens, words): + word_start = word["start_time"] if not isinstance(word, dict) else word["start_time"] + word_end = word["end_time"] if not isinstance(word, dict) else word["end_time"] + timeline.tokens.append( + TokenRef( + text=token, + norm=normalize_text(token), + start=float(word_start), + end=float(word_end), + segment_id=segment_id, + precision="word", + ) + ) + else: + times = _distribute_times(start, end, len(original_tokens)) + for token, (token_start, token_end) in zip(original_tokens, times): + timeline.tokens.append( + TokenRef( + text=token, + norm=normalize_text(token), + start=token_start, + end=token_end, + segment_id=segment_id, + precision="segment", + ) + ) + + return timeline + + +@dataclass +class SearchContext: + """Die drei Kontextabschnitte eines Treffers.""" + + before: str + match: str + after: str + start_time: float + end_time: float + precision: str + + def as_block(self, quote: str = '"') -> str: + """Formatiert den Kontext als zusammenhaengenden Textblock.""" + before = f"\u2026{self.before} " if self.before else "" + after = f" {self.after}\u2026" if self.after else "" + return f"{quote}{before}{self.match}{after}{quote}" + + +def build_context( + timeline: Timeline, start_index: int, end_index: int, context_words: int = 50 +) -> SearchContext: + """Schneidet Kontext um den Treffer ``[start_index, end_index)`` aus.""" + tokens = timeline.tokens + if not tokens: + return SearchContext("", "", "", 0.0, 0.0, "segment") + + start_index = max(0, min(start_index, len(tokens))) + end_index = max(start_index + 1, min(end_index, len(tokens))) + + before_start = max(0, start_index - max(0, context_words)) + after_end = min(len(tokens), end_index + max(0, context_words)) + + before_tokens = tokens[before_start:start_index] + match_tokens = tokens[start_index:end_index] + after_tokens = tokens[end_index:after_end] + + precision = "word" if all(token.precision == "word" for token in match_tokens) else "segment" + start_time = match_tokens[0].start + end_time = match_tokens[-1].end + + return SearchContext( + before=" ".join(token.text for token in before_tokens), + match=" ".join(token.text for token in match_tokens), + after=" ".join(token.text for token in after_tokens), + start_time=start_time, + end_time=end_time, + precision=precision, + ) diff --git a/sjaudiosearch/search/engine.py b/sjaudiosearch/search/engine.py new file mode 100644 index 0000000..b15421d --- /dev/null +++ b/sjaudiosearch/search/engine.py @@ -0,0 +1,303 @@ +"""Suchmaschine fuer SjAudioSearch. + +Ablauf einer Suche: + +1. Anfrage normalisieren. +2. Kandidaten ueber FTS5 und den Trigramm-Index sammeln (schnelle Vorauswahl). +3. Sliding-Window-Vergleich mit RapidFuzz auf der Wort-Timeline der Kandidaten. +4. Optionaler, vollstaendiger Fallback-Lauf ueber alle verbleibenden Dateien, + damit auch komplett falsch erkannte Woerter gefunden werden. +5. Treffer entduplizieren, sortieren, begrenzen. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Callable + +from rapidfuzz import fuzz + +from ..config import SearchConfig +from ..database.repository import Repository +from ..utils.logging import get_logger +from .context import SearchContext, Timeline, build_context, build_timeline +from .fuzzy import ( + build_fts_query, + normalize_text, + score_window, + significant_overlap, + trigram_terms, +) +from .ranking import dedupe_overlapping, rank_hits + +logger = get_logger("search.engine") + +ProgressCallback = Callable[[int, int], None] + + +class SearchError(RuntimeError): + """Fehler bei der Suche (z. B. leere Anfrage).""" + + +@dataclass +class SearchHit: + """Ein einzelner Suchtreffer.""" + + audio_id: int + filename: str + path: str + score: float + start_time: float + end_time: float + start_token: int + end_token: int + context: SearchContext + language: str | None = None + model: str | None = None + source: str = "fts" + + @property + def matched_text(self) -> str: + return self.context.match + + def to_dict(self, context_words: int = 50) -> dict[str, object]: + return { + "audio_id": self.audio_id, + "filename": self.filename, + "path": self.path, + "similarity": round(self.score, 2), + "timestamp": self.start_time, + "timestamp_end": self.end_time, + "timestamp_precision": self.context.precision, + "match": self.context.match, + "context_before": self.context.before, + "context_after": self.context.after, + "context_words": context_words, + "language": self.language, + "model": self.model, + "source": self.source, + } + + +class SearchEngine: + """Kombinierte Suche aus FTS5, Trigrammen und Fuzzy-Sliding-Window.""" + + def __init__(self, repository: Repository, config: SearchConfig): + self.repo = repository + self.config = config + + # ------------------------------------------------------------------ + def search( + self, + query: str, + *, + limit: int | None = None, + min_score: float | None = None, + context_words: int | None = None, + language: str | None = None, + progress: ProgressCallback | None = None, + ) -> list[SearchHit]: + query_normalized = normalize_text(query) + if not query_normalized: + raise SearchError("Die Suchanfrage enthaelt keine verwertbaren Woerter.") + query_tokens = query_normalized.split() + + limit = self.config.default_limit if limit is None else int(limit) + min_score = self.config.default_min_score if min_score is None else float(min_score) + context_words = self.config.context_words if context_words is None else int(context_words) + if limit <= 0: + raise SearchError("--limit muss groesser als 0 sein.") + + candidate_audio, candidate_source = self._collect_candidates(query_tokens) + hits: list[SearchHit] = [] + swept: set[int] = set() + + total_candidates = len(candidate_audio) + for index, audio_id in enumerate(candidate_audio, start=1): + swept.add(audio_id) + hits.extend( + self._scan_audio( + audio_id, + query_normalized, + query_tokens, + min_score=min_score, + context_words=context_words, + language=language, + source="fts", + ) + ) + if progress: + progress(index, total_candidates) + + # Wichtig: FTS5 ist nur eine Vorauswahl. Wenn wenige Treffer gefunden + # wurden, werden alle uebrigen Dateien vollstaendig durchsucht. + fallback_used = False + if self.config.exhaustive_fallback and len(hits) < limit: + remaining = [ + audio_id + for audio_id in self.repo.iter_audio_ids_with_transcripts(language) + if audio_id not in swept + ] + fallback_used = bool(remaining) + for index, audio_id in enumerate(remaining, start=1): + hits.extend( + self._scan_audio( + audio_id, + query_normalized, + query_tokens, + min_score=min_score, + context_words=context_words, + language=language, + source="fallback", + ) + ) + if progress: + progress(index, len(remaining)) + + if fallback_used: + logger.info( + "Fallback-Suche aktiv: %d Dateien ohne FTS-Treffer durchsucht", len(remaining) + ) + + deduped = dedupe_overlapping(hits) + return rank_hits(deduped, limit) + + # ------------------------------------------------------------------ + def _collect_candidates(self, query_tokens: list[str]) -> tuple[list[int], str]: + """Sammelt Kandidaten-Audiodateien (FTS5 + Trigramme).""" + limit = max(1, int(self.config.candidate_limit)) + matches: list[tuple[int, int]] = [] + + fts_query = build_fts_query(query_tokens) + if fts_query: + matches.extend(self.repo.fts_candidates(fts_query, limit)) + + remaining = limit - len(matches) + if remaining > 0: + terms = trigram_terms(query_tokens) + if terms: + matches.extend(self.repo.trigram_candidates(terms, remaining)) + + counts: dict[int, int] = {} + for audio_id, _segment_id in matches: + counts[audio_id] = counts.get(audio_id, 0) + 1 + + ordered = sorted(counts, key=lambda audio_id: (-counts[audio_id], audio_id)) + return ordered, "fts" + + # ------------------------------------------------------------------ + def _scan_audio( + self, + audio_id: int, + query_normalized: str, + query_tokens: list[str], + *, + min_score: float, + context_words: int, + language: str | None, + source: str, + ) -> list[SearchHit]: + record = self.repo.get_audio(audio_id) + if record is None or record.duplicate_of is not None: + return [] + + transcript = self.repo.get_transcript(audio_id) + if transcript is None: + return [] + if language and (transcript["language"] or "").lower() != language.lower(): + return [] + + segments, words = self.repo.timeline(audio_id) + if not segments: + return [] + timeline = build_timeline( + audio_id, + segments, + words, + language=transcript["language"], + model=transcript["model"], + ) + if not timeline.tokens: + return [] + + windows = self._find_windows(timeline, query_normalized, query_tokens, min_score) + if not windows: + return [] + + hits: list[SearchHit] = [] + for score, start_index, end_index in windows: + context = build_context(timeline, start_index, end_index, context_words) + hits.append( + SearchHit( + audio_id=audio_id, + filename=record.filename, + path=record.path, + score=score, + start_time=context.start_time, + end_time=context.end_time, + start_token=start_index, + end_token=end_index, + context=context, + language=transcript["language"], + model=transcript["model"], + source=source, + ) + ) + + hits.sort(key=lambda hit: -hit.score) + return hits[: max(1, int(self.config.max_hits_per_file))] + + # ------------------------------------------------------------------ + def _find_windows( + self, + timeline: Timeline, + query_normalized: str, + query_tokens: list[str], + min_score: float, + ) -> list[tuple[float, int, int]]: + """Sliding-Window-Suche ueber die Wort-Timeline einer Datei.""" + norm_tokens = timeline.normalized_tokens + token_count = len(norm_tokens) + query_len = len(query_tokens) + if token_count == 0: + return [] + + min_size = max(1, query_len - max(0, self.config.window_before)) + max_size = max(min_size, query_len + max(0, self.config.window_after)) + overlap_threshold = max(0.0, min(1.0, self.config.min_token_overlap)) + + # Dateiebene: gibt es ueberhaupt Beruehrungspunkte? Das ist deutlich + # billiger als jede Datei Fenster fuer Fenster zu vergleichen. + file_overlap = significant_overlap(query_tokens, norm_tokens) + if file_overlap < overlap_threshold * 0.5: + partial = fuzz.partial_ratio(query_normalized, timeline.normalized_text) + if partial < 55.0: + return [] + + windows: list[tuple[float, int, int]] = [] + # Wenn auf Dateiebene kaum Wortueberschneidung besteht, kann der Treffer + # nur ueber einen falsch erkannten Ausdruck zustande kommen. Dann wird + # ohne Wort-Vorfilter verglichen (selten und durch die Dateipruefung + # oben bereits eingegrenzt). + use_overlap_filter = file_overlap >= overlap_threshold + for start in range(token_count): + pre_end = min(token_count, start + query_len + 2) + if use_overlap_filter and significant_overlap(query_tokens, norm_tokens[start:pre_end]) < overlap_threshold: + continue + + best_score = 0.0 + best_size = 0 + for size in range(min_size, max_size + 1): + end = start + size + if end > token_count: + break + window_normalized = " ".join(norm_tokens[start:end]) + candidate = score_window(query_normalized, window_normalized) + if candidate.score > best_score: + best_score = candidate.score + best_size = size + + if best_size and best_score >= min_score: + windows.append((best_score, start, start + best_size)) + + return windows diff --git a/sjaudiosearch/search/fuzzy.py b/sjaudiosearch/search/fuzzy.py new file mode 100644 index 0000000..39ba2cc --- /dev/null +++ b/sjaudiosearch/search/fuzzy.py @@ -0,0 +1,178 @@ +"""Textnormalisierung und Fuzzy-Vergleich (RapidFuzz). + +Wichtig: Die Normalisierung wird ausschliesslich fuer Vergleiche und fuer den +Suchindex verwendet. Der gespeicherte Originaltext bleibt unveraendert. +""" + +from __future__ import annotations + +import re +import unicodedata +from dataclasses import dataclass + +from rapidfuzz import fuzz + +# Satzzeichen und Symbole, die beim Vergleich stoeren. Wortzeichen (Buchstaben, +# Ziffern, Unterstrich) bleiben erhalten. +_PUNCT_RE = re.compile(r"[^\w\s]", re.UNICODE) +_WHITESPACE_RE = re.compile(r"\s+", re.UNICODE) + +# Sehr haeufige deutsche Fuellwoerter. Sie werden beim Vorfilter nicht als +# belastbare Uebereinstimmung gewertet, damit "der/die/das" allein keine +# Kandidaten erzeugt. +STOPWORDS = frozenset( + { + "der", "die", "das", "den", "dem", "des", "ein", "eine", "einer", "eines", + "einem", "einen", "und", "oder", "aber", "als", "auch", "auf", "aus", "bei", + "bin", "bis", "da", "dann", "dass", "doch", "dort", "du", "durch", "er", + "es", "hat", "ich", "ihm", "ihn", "ihr", "im", "in", "ist", "mit", "nach", + "nicht", "noch", "nun", "nur", "ob", "so", "sich", "sie", "sind", "um", + "und", "vom", "von", "vor", "war", "was", "wenn", "wie", "wir", "wo", "zu", + "zum", "zur", "the", "and", "of", "to", "in", "a", "is", + } +) + + +def normalize_text(text: str) -> str: + """Normalisiert Text fuer Vergleiche: NFKC, Kleinschreibung, ohne Satzzeichen.""" + if not text: + return "" + normalized = unicodedata.normalize("NFKC", text).casefold() + normalized = _PUNCT_RE.sub(" ", normalized) + return _WHITESPACE_RE.sub(" ", normalized).strip() + + +def tokenize(text: str) -> list[str]: + """Normalisierte Wortliste eines Textes.""" + normalized = normalize_text(text) + return normalized.split() if normalized else [] + + +def tokenize_original(text: str) -> list[str]: + """Wortliste mit Originaltext inklusive anhängender Satzzeichen (fuer die Ausgabe).""" + if not text: + return [] + return [token for token in _WHITESPACE_RE.split(text.strip()) if token] + + +def token_norm(token: str) -> str: + """Normalisiert ein einzelnes Originalwort.""" + return normalize_text(token) + + +def is_stopword(token: str) -> bool: + return token in STOPWORDS + + +@dataclass(frozen=True) +class WindowScore: + """Ergebnis eines Fenstervergleichs.""" + + score: float + exact: bool + + +def score_window(query_normalized: str, window_normalized: str) -> WindowScore: + """Vergleicht Suchanfrage und Textfenster und liefert einen Wert 0-100. + + Es werden mehrere RapidFuzz-Kennzahlen kombiniert, damit fehlende, + zusaetzliche, vertauschte oder falsch erkannte Woerter nicht sofort zum + Verlust des Treffers fuehren. + """ + if not query_normalized or not window_normalized: + return WindowScore(0.0, False) + + if query_normalized == window_normalized: + return WindowScore(100.0, True) + + # Hinweis: Eine reine Teilzeichenketten-Pruefung ist hier absichtlich NICHT + # enthalten. Sie wuerde z. B. das Fenster "Leider" fuer die Anfrage + # "Leider zu spaet" mit 100 bewerten und den Treffer auf ein Wort + # zusammenfallen lassen. Teiluebereinstimmungen werden stattdessen ueber + # die gewichteten RapidFuzz-Kennzahlen unten bewertet. + + ratio = fuzz.ratio(query_normalized, window_normalized) + token_set = fuzz.token_set_ratio(query_normalized, window_normalized) + weighted = fuzz.WRatio(query_normalized, window_normalized) + partial = fuzz.partial_ratio(query_normalized, window_normalized) + + # partial_ratio neigt bei kurzen Fenstern zu hohen Werten, daher nur + # gedämpft einbeziehen. + # token_set_ratio/WRatio/partial_ratio bewerten eine Teilmenge der Anfrage + # mit 100. Ohne Daempfung wuerde ein einzelnes Wort einer mehrwoertrigen + # Anfrage als perfekter Treffer gelten. Daher werden diese Kennzahlen mit + # dem Laengenverhaeltnis gewichtet, wenn das Fenster kuerzer ist. + length_factor = min(1.0, len(window_normalized) / max(1, len(query_normalized))) + + score = max( + ratio, + token_set * length_factor, + weighted * length_factor, + partial * 0.92 * length_factor, + ) + return WindowScore(min(100.0, round(score, 2)), False) + + +def token_overlap(query_tokens: list[str], window_tokens: list[str]) -> float: + """Anteil der Anfragewoerter, die im Fenster vorkommen (0..1).""" + if not query_tokens: + return 0.0 + window_set = set(window_tokens) + hits = sum(1 for token in query_tokens if token in window_set) + return hits / len(query_tokens) + + +def significant_overlap(query_tokens: list[str], window_tokens: list[str]) -> float: + """Wie :func:`token_overlap`, aber Fuellwoerter zaehlen nur halb.""" + if not query_tokens: + return 0.0 + window_set = set(window_tokens) + total = 0.0 + hits = 0.0 + for token in query_tokens: + weight = 0.5 if is_stopword(token) else 1.0 + total += weight + if token in window_set: + hits += weight + if total == 0: + return 0.0 + return hits / total + + +def _escape_fts_token(token: str) -> str: + return '"' + token.replace('"', '""') + '"' + + +def build_fts_query(tokens: list[str], prefix_min_length: int = 4) -> str: + """Baut eine FTS5-Anfrage aus ODER-verknuepften Woertern. + + Praefixsuche fuer laengere Woerter faengt deutsche Flexionsformen und + Zusammensetzungen ab. Die Anfrage ist bewusst weit gefasst: sie dient nur + der Kandidatensuche, die Bewertung erfolgt spaeter per Fuzzy-Vergleich. + """ + parts: list[str] = [] + seen: set[str] = set() + for token in tokens: + if len(token) < 2 or token in seen: + continue + seen.add(token) + escaped = _escape_fts_token(token) + if len(token) >= prefix_min_length: + parts.append(f"{escaped}*") + else: + parts.append(escaped) + return " OR ".join(parts) + + +def trigram_terms(tokens: list[str], max_terms: int = 32) -> list[str]: + """Sinnvolle Trigramm-Anfragen aus laengeren Anfragewoertern.""" + terms: list[str] = [] + seen: set[str] = set() + for token in tokens: + if len(token) < 4 or token in seen: + continue + seen.add(token) + terms.append(token) + if len(terms) >= max_terms: + break + return terms diff --git a/sjaudiosearch/search/ranking.py b/sjaudiosearch/search/ranking.py new file mode 100644 index 0000000..663b45c --- /dev/null +++ b/sjaudiosearch/search/ranking.py @@ -0,0 +1,53 @@ +"""Bewertung und Sortierung von Suchtreffern. + +Der Score ist eine Aehnlichkeitsbewertung (0-100) und ausdruecklich keine +statistische Wahrscheinlichkeit. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: # pragma: no cover + from .engine import SearchHit + + +def clamp_score(score: float) -> float: + return max(0.0, min(100.0, float(score))) + + +def dedupe_overlapping(hits: list["SearchHit"], gap_tokens: int = 3) -> list["SearchHit"]: + """Entfernt Treffer, die denselben Textabschnitt mehrfach abdecken.""" + if not hits: + return [] + + ordered = sorted(hits, key=lambda hit: (-hit.score, hit.start_token)) + kept: list["SearchHit"] = [] + for hit in ordered: + overlapping = False + for existing in kept: + if existing.audio_id != hit.audio_id: + continue + same_region = ( + abs(existing.start_token - hit.start_token) <= gap_tokens + or (hit.start_token < existing.end_token and existing.start_token < hit.end_token) + ) + if same_region: + overlapping = True + break + if not overlapping: + kept.append(hit) + return kept + + +def rank_hits(hits: list["SearchHit"], limit: int) -> list["SearchHit"]: + """Sortiert Treffer nach Relevanz und begrenzt die Anzahl.""" + ordered = sorted( + hits, + key=lambda hit: ( + -hit.score, + hit.filename.lower(), + hit.start_time, + ), + ) + return ordered[: max(0, int(limit))] diff --git a/sjaudiosearch/system/diagnostics.py b/sjaudiosearch/system/diagnostics.py new file mode 100644 index 0000000..0093f12 --- /dev/null +++ b/sjaudiosearch/system/diagnostics.py @@ -0,0 +1,233 @@ +"""Systemdiagnose und verstaendlicher Systembericht.""" + +from __future__ import annotations + +import os +import shutil +import sqlite3 +from dataclasses import dataclass, field +from pathlib import Path + +from rich.console import Console +from rich.panel import Panel +from rich.table import Table + +from ..config import Config +from ..utils.formatting import format_size +from ..utils.logging import get_logger +from .gpu import ( + GpuInfo, + ResolvedExecution, + RuntimeInfo, + collect_runtime_info, + detect_gpu, + is_pascal_or_older, + resolve_execution, +) + +logger = get_logger("system.diagnostics") + +CHECK = "[green]OK[/green]" +WARN = "[yellow]Warnung[/yellow]" +FAIL = "[red]nicht verfuegbar[/red]" + + +@dataclass +class SystemReport: + """Gesammelte Systeminformationen.""" + + gpu: GpuInfo | None + runtime: RuntimeInfo + execution: ResolvedExecution + db_path: Path + db_exists: bool + config_path: str | None + model_dir: Path + model_dir_writable: bool + ffmpeg: str | None + ffplay: str | None + notes: list[str] = field(default_factory=list) + open_tasks: list[str] = field(default_factory=list) + + @property + def gpu_ready(self) -> bool: + return self.runtime.cuda_runtime_usable and self.execution.device == "cuda" + + @property + def verdict(self) -> str: + if self.gpu_ready: + return ( + f"GPU-Transkription moeglich mit compute_type='{self.execution.compute_type}'." + ) + return "Keine GPU-Transkription moeglich - CPU-Fallback wird verwendet." + + +def _is_writable(path: Path) -> bool: + target = path + while not target.exists() and target != target.parent: + target = target.parent + return target.is_dir() and os.access(target, os.W_OK) + + +def collect_report(config: Config, *, requested_device: str | None = None, + requested_compute_type: str | None = None) -> SystemReport: + """Erstellt einen vollstaendigen Systembericht.""" + gpu = detect_gpu() + runtime = collect_runtime_info(check_fts=True) + + device = requested_device or config.transcription.device + compute_type = requested_compute_type or config.transcription.compute_type + execution = resolve_execution(device, compute_type, gpu, runtime) + + db_path = config.database_path + model_dir = config.model_path + notes: list[str] = [] + open_tasks: list[str] = [] + + if gpu is not None and is_pascal_or_older(gpu): + notes.append( + f"{gpu.name} (Compute Capability {gpu.compute_capability}) ist eine Pascal-Karte: " + "kein FP16/BF16, kein Tensor-Core-Betrieb. int8_float32 ist der passende Modus." + ) + if runtime.cudnn_libraries: + notes.append( + f"cuDNN gefunden ({len(runtime.cudnn_libraries)} Bibliothek(en)); " + "SjAudioSearch laedt die pip-nvidia-Bibliotheken beim Start automatisch vor." + ) + else: + notes.append( + "Kein cuDNN gefunden. Fuer GPU-Betrieb die CUDA-12-Wheels installieren: " + "pip install -e \".[cuda]\" (nvidia-cublas-cu12, nvidia-cudnn-cu12)." + ) + if not runtime.fts5_available: + open_tasks.append( + "SQLite ohne FTS5 - bitte Python mit FTS5-Unterstuetzung installieren." + ) + if not runtime.trigram_available: + open_tasks.append( + "Trigramm-Tokenizer nicht verfuegbar - die Kandidatensuche arbeitet nur ueber FTS5." + ) + + return SystemReport( + gpu=gpu, + runtime=runtime, + execution=execution, + db_path=db_path, + db_exists=db_path.exists(), + config_path=str(config.source_path) if config.source_path else None, + model_dir=model_dir, + model_dir_writable=_is_writable(model_dir.parent if not model_dir.exists() else model_dir), + ffmpeg=shutil.which("ffmpeg"), + ffplay=shutil.which("ffplay"), + notes=notes, + open_tasks=open_tasks, + ) + + +def render_report(report: SystemReport, console: Console) -> None: + """Gibt den Systembericht uebersichtlich im Terminal aus.""" + console.print(Panel.fit("[bold]SjAudioSearch - Systemdiagnose[/bold]", border_style="cyan")) + + system = Table(show_header=False, box=None, padding=(0, 1)) + system.add_column(style="bold cyan", no_wrap=True) + system.add_column() + system.add_row("Python", report.runtime.python_version) + system.add_row("System", report.runtime.platform) + system.add_row("SQLite", report.runtime.sqlite_version) + system.add_row("FTS5", CHECK if report.runtime.fts5_available else FAIL) + system.add_row("FTS5-Trigramme", CHECK if report.runtime.trigram_available else WARN) + system.add_row("CTranslate2", report.runtime.ctranslate2_version or FAIL) + system.add_row("FFmpeg", report.ffmpeg or WARN) + system.add_row("ffplay", report.ffplay or WARN) + system.add_row("Konfiguration", report.config_path or "eingebaute Defaults") + system.add_row("Datenbank", f"{report.db_path} ({'vorhanden' if report.db_exists else 'wird angelegt'})") + system.add_row("Modellordner", f"{report.model_dir} ({'beschreibbar' if report.model_dir_writable else 'NICHT beschreibbar'})") + console.print(system) + + console.print() + gpu_table = Table(title="GPU", show_header=False, box=None, padding=(0, 1)) + gpu_table.add_column(style="bold cyan", no_wrap=True) + gpu_table.add_column() + if report.gpu is None: + gpu_table.add_row("Status", FAIL + " (nvidia-smi lieferte keine Daten)") + else: + gpu_table.add_row("Modell", report.gpu.name) + gpu_table.add_row("Treiber", report.gpu.driver_version or "unbekannt") + gpu_table.add_row("CUDA (Treiber)", report.gpu.cuda_driver_version or "unbekannt") + gpu_table.add_row( + "VRAM", f"{report.gpu.memory_total_mb} MB" if report.gpu.memory_total_mb else "unbekannt" + ) + gpu_table.add_row("Compute Capability", report.gpu.compute_capability or "unbekannt") + gpu_table.add_row("Architektur", "Pascal oder aelter" if is_pascal_or_older(report.gpu) else "neuere Architektur") + gpu_table.add_row( + "CUDA-Geraete (CTranslate2)", str(report.runtime.cuda_device_count) + ) + gpu_table.add_row( + "Compute Types (CUDA)", + ", ".join(report.runtime.cuda_compute_types) or "keine", + ) + gpu_table.add_row( + "Compute Types (CPU)", + ", ".join(report.runtime.cpu_compute_types) or "keine", + ) + gpu_table.add_row( + "CUDA-Bibliotheken", + ", ".join(Path(lib).name for lib in report.runtime.cuda_libraries) or "keine im System", + ) + gpu_table.add_row( + "cuDNN-Bibliotheken", + ", ".join(Path(lib).name for lib in report.runtime.cudnn_libraries) or "keine im System", + ) + console.print(gpu_table) + + console.print() + plan = Table(title="Empfohlener Ausfuehrungsmodus", show_header=False, box=None, padding=(0, 1)) + plan.add_column(style="bold cyan", no_wrap=True) + plan.add_column() + plan.add_row("Angefragt", f"{report.execution.requested_device} / {report.execution.requested_compute_type}") + plan.add_row("Verwendet", f"{report.execution.device} / {report.execution.compute_type}") + plan.add_row("FP16/BF16", "nicht verwendet" if report.execution.device == "cuda" else "irrelevant (CPU)") + console.print(plan) + + for note in report.execution.notes: + console.print(f"[yellow]-[/yellow] {note}") + for note in report.notes: + console.print(f"[cyan]-[/cyan] {note}") + + console.print() + color = "green" if report.gpu_ready else "yellow" + console.print(Panel(report.verdict, border_style=color, title="Ergebnis")) + + if report.open_tasks: + console.print() + console.print("[bold red]Offene Punkte:[/bold red]") + for task in report.open_tasks: + console.print(f" [red]*[/red] {task}") + + +def database_summary(db_path: Path) -> dict[str, object] | None: + """Kurze Zusammenfassung einer vorhandenen Datenbank (fuer die Diagnose).""" + if not db_path.exists(): + return None + try: + conn = sqlite3.connect(str(db_path)) + try: + tables = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type IN ('table','view')" + ) + } + audio_count = 0 + if "audio_files" in tables: + audio_count = int(conn.execute("SELECT COUNT(*) FROM audio_files").fetchone()[0]) + return { + "tables": sorted(tables), + "audio_files": audio_count, + "size": format_size(db_path.stat().st_size), + } + finally: + conn.close() + except sqlite3.Error as exc: + logger.warning("Datenbank konnte nicht gelesen werden: %s", exc) + return None diff --git a/sjaudiosearch/system/gpu.py b/sjaudiosearch/system/gpu.py new file mode 100644 index 0000000..3dc0c92 --- /dev/null +++ b/sjaudiosearch/system/gpu.py @@ -0,0 +1,406 @@ +"""GPU-, CUDA- und CTranslate2-Diagnose. + +Die GTX 1070 Ti ist eine Pascal-Karte (Compute Capability 6.1). Sie besitzt +keine FP16-Tensor-Cores. Deshalb wird FP16/BF16 hier grundsaetzlich nicht als +GPU-Rechenmodus vorausgesetzt; bevorzugt wird ``int8_float32``. +""" + +from __future__ import annotations + +import ctypes.util +import glob +import os +import re +import subprocess +from dataclasses import dataclass, field +from pathlib import Path + +from ..utils.logging import get_logger + +logger = get_logger("system.gpu") + +# Pascal und aelter: kein FP16-Tensor-Betrieb auf Consumer-Karten. +PASCAL_CONSUMER_MAX_CC = 6.2 +PREFERRED_COMPUTE_TYPE = "int8_float32" +FORBIDDEN_GPU_COMPUTE_TYPES = ("float16", "bfloat16", "int8_bfloat16", "int8_float16") + + +@dataclass +class GpuInfo: + """Angaben aus ``nvidia-smi``.""" + + name: str + driver_version: str | None = None + cuda_driver_version: str | None = None + memory_total_mb: int | None = None + compute_capability: str | None = None + + +@dataclass +class RuntimeInfo: + """Laufzeitinformationen zu CTranslate2, CUDA und SQLite.""" + + ctranslate2_version: str | None = None + cuda_device_count: int = 0 + supported_compute_types: dict[str, list[str]] = field(default_factory=dict) + cuda_libraries: list[str] = field(default_factory=list) + cudnn_libraries: list[str] = field(default_factory=list) + python_version: str = "" + platform: str = "" + sqlite_version: str = "" + fts5_available: bool = False + trigram_available: bool = False + + @property + def cuda_runtime_usable(self) -> bool: + return self.cuda_device_count > 0 and bool(self.supported_compute_types.get("cuda")) + + @property + def cuda_compute_types(self) -> list[str]: + return sorted(self.supported_compute_types.get("cuda", [])) + + @property + def cpu_compute_types(self) -> list[str]: + return sorted(self.supported_compute_types.get("cpu", [])) + + +@dataclass +class ResolvedExecution: + """Tatsaechlich verwendeter Ausfuehrungsmodus.""" + + device: str + compute_type: str + requested_device: str + requested_compute_type: str + notes: list[str] = field(default_factory=list) + + @property + def used_fallback(self) -> bool: + return self.device != self.requested_device or self.compute_type != self.requested_compute_type + + +def _run_nvidia_smi(args: list[str]) -> str | None: + try: + completed = subprocess.run( + ["nvidia-smi", *args], + capture_output=True, + text=True, + timeout=15, + check=False, + ) + except (OSError, subprocess.SubprocessError) as exc: + logger.debug("nvidia-smi nicht ausfuehrbar: %s", exc) + return None + if completed.returncode != 0: + return None + return completed.stdout + + +def detect_gpu() -> GpuInfo | None: + """Ermittelt GPU-Modell, Treiber und Compute Capability.""" + output = _run_nvidia_smi( + [ + "--query-gpu=name,driver_version,memory.total,compute_cap", + "--format=csv,noheader", + ] + ) + gpu: GpuInfo | None = None + if output: + first_line = output.strip().splitlines()[0] if output.strip() else "" + parts = [part.strip() for part in first_line.split(",")] + if len(parts) >= 4: + memory_mb = None + match = re.search(r"(\d+)", parts[2]) + if match: + memory_mb = int(match.group(1)) + gpu = GpuInfo( + name=parts[0], + driver_version=parts[1] or None, + memory_total_mb=memory_mb, + compute_capability=parts[3] or None, + ) + if gpu is None: + return None + + header = _run_nvidia_smi([]) or "" + match = re.search(r"CUDA Version:\s*([0-9.]+)", header) + if match: + gpu.cuda_driver_version = match.group(1) + return gpu + + +_PRELOADED_LIBRARIES: list[str] = [] + + +def ensure_cuda_libraries() -> list[str]: + """Laedt CUDA-/cuDNN-Bibliotheken aus den pip-``nvidia``-Paketen. + + CTranslate2 linkt dynamisch gegen ``libcublas.so.12`` und ``libcudnn.so.9``. + Die PyPI-Wheels legen diese Bibliotheken unter ``site-packages/nvidia`` + ab, wo der dynamische Linker sie nicht von allein findet. Ein Preload mit + ``RTLD_GLOBAL`` macht sie verfuegbar, ohne systemweite Aenderungen an + ``/etc/ld.so.conf`` oder installierte CUDA-Versionen. + + Muss vor dem ersten CUDA-Aufruf von CTranslate2 erfolgen. + """ + global _PRELOADED_LIBRARIES + if _PRELOADED_LIBRARIES or os.name != "posix": + return _PRELOADED_LIBRARIES + + cuda_libs, cudnn_libs = detect_cuda_libraries() + if not cuda_libs and not cudnn_libs: + return _PRELOADED_LIBRARIES + + order = ("libcudart", "libcublasLt", "libcublas", "libcudnn") + + def sort_key(path: str) -> int: + name = Path(path).name + for index, prefix in enumerate(order): + if name.startswith(prefix): + return index + return len(order) + + import ctypes + + loaded: list[str] = [] + for path in sorted(cuda_libs + cudnn_libs, key=sort_key): + # cuDNN-Teilbibliotheken werden von libcudnn.so.9 selbst nachgeladen. + name = Path(path).name + if name.startswith("libcudnn") and not name.startswith("libcudnn.so"): + continue + try: + ctypes.CDLL(path, mode=ctypes.RTLD_GLOBAL) + loaded.append(path) + except OSError as exc: + logger.debug("Bibliothek konnte nicht vorgeladen werden (%s): %s", path, exc) + + if loaded: + directories = sorted({str(Path(path).parent) for path in loaded}) + existing = os.environ.get("LD_LIBRARY_PATH", "") + os.environ["LD_LIBRARY_PATH"] = os.pathsep.join( + directories + ([existing] if existing else []) + ) + logger.debug("CUDA-Bibliotheken vorgeladen: %s", ", ".join(Path(p).name for p in loaded)) + + _PRELOADED_LIBRARIES = loaded + return loaded + + +def _find_libraries(patterns: list[str], directories: list[str]) -> list[str]: + found: list[str] = [] + for pattern in patterns: + for directory in directories: + try: + found.extend(glob.glob(os.path.join(directory, pattern))) + except OSError: + continue + return sorted(set(found)) + + +def detect_cuda_libraries() -> tuple[list[str], list[str]]: + """Sucht CUDA- und cuDNN-Bibliotheken im System und in der venv.""" + directories = [ + "/usr/lib", + "/usr/lib64", + "/usr/local/cuda/lib64", + "/usr/local/lib", + "/opt/cuda/lib64", + str(Path.cwd() / ".venv" / "lib"), + ] + # nvidia-Wheels (falls installiert) liegen unter site-packages/nvidia//lib + import site + import sys + + site_roots: list[Path] = [] + for base in (os.environ.get("VIRTUAL_ENV"), os.environ.get("CONDA_PREFIX")): + if base: + site_roots.append(Path(base)) + site_roots.extend(Path(p) for p in site.getsitepackages()) + site_roots.append(Path(sys.prefix)) + + for base in site_roots: + directories.extend([str(base / "lib"), str(base / "lib64")]) + nvidia_root = base / "lib" / f"python{sys.version_info.major}.{sys.version_info.minor}" / "site-packages" / "nvidia" + directories.extend(str(path) for path in nvidia_root.glob("*/lib")) + directories.extend(str(path) for path in (base / "Lib" / "site-packages" / "nvidia").glob("*/bin")) + + cuda_patterns = ["libcudart.so*", "libcublas.so*", "libcublasLt.so*"] + cudnn_patterns = ["libcudnn*.so*"] + + cuda_libs = _find_libraries(cuda_patterns, directories) + cudnn_libs = _find_libraries(cudnn_patterns, directories) + + # Zusaetzlich ueber den dynamischen Linker suchen. + for name in ("cudart", "cublas", "cudnn"): + located = ctypes.util.find_library(name) + if located and located not in cuda_libs and name != "cudnn": + cuda_libs.append(located) + if located and name == "cudnn" and located not in cudnn_libs: + cudnn_libs.append(located) + + return sorted(set(cuda_libs)), sorted(set(cudnn_libs)) + + +def probe_ctranslate2() -> tuple[str | None, int, dict[str, list[str]], str | None]: + """Fragt CTranslate2 nach Geraeten und unterstuetzten Compute Types.""" + try: + import ctranslate2 + except ImportError as exc: + return None, 0, {}, f"CTranslate2 nicht installiert: {exc}" + + version = getattr(ctranslate2, "__version__", None) + error: str | None = None + + try: + device_count = int(ctranslate2.get_cuda_device_count()) + except Exception as exc: # pragma: no cover - hardwareabhaengig + device_count = 0 + error = f"CUDA-Geraete konnten nicht abgefragt werden: {exc}" + + supported: dict[str, list[str]] = {} + for device in ("cpu", "cuda"): + try: + supported[device] = sorted(ctranslate2.get_supported_compute_types(device)) + except Exception as exc: + logger.debug("Compute Types fuer %s nicht verfuegbar: %s", device, exc) + supported[device] = [] + + return version, device_count, supported, error + + +def collect_runtime_info(*, check_fts: bool = True) -> RuntimeInfo: + """Sammelt alle Laufzeitinformationen fuer den Systembericht.""" + import platform + import sqlite3 + import sys + + version, device_count, supported, error = probe_ctranslate2() + cuda_libs, cudnn_libs = detect_cuda_libraries() + + info = RuntimeInfo( + ctranslate2_version=version, + cuda_device_count=device_count, + supported_compute_types=supported, + cuda_libraries=cuda_libs, + cudnn_libraries=cudnn_libs, + python_version=sys.version.split()[0], + platform=f"{platform.system()} {platform.release()}", + sqlite_version=sqlite3.sqlite_version, + ) + if error: + logger.debug(error) + + if check_fts: + from ..database.connection import check_fts5, connect + + conn = connect(":memory:") + try: + info.fts5_available, info.trigram_available = check_fts5(conn) + finally: + conn.close() + + return info + + +def _cc_tuple(compute_capability: str | None) -> tuple[int, int] | None: + if not compute_capability: + return None + match = re.match(r"^\s*(\d+)\.(\d+)", compute_capability) + if not match: + return None + return int(match.group(1)), int(match.group(2)) + + +def is_pascal_or_older(gpu: GpuInfo | None) -> bool: + """True fuer Pascal und aeltere Architekturen (Compute Capability <= 6.x).""" + if gpu is None: + return False + cc = _cc_tuple(gpu.compute_capability) + if cc is None: + return False + return cc[0] <= 6 + + +def resolve_execution( + requested_device: str, + requested_compute_type: str, + gpu: GpuInfo | None, + runtime: RuntimeInfo, +) -> ResolvedExecution: + """Waehlt einen tatsaechlich unterstuetzten Ausfuehrungsmodus. + + Regeln: + * Nie einen Compute Type verwenden, den CTranslate2 fuer das Geraet nicht meldet. + * Auf Pascal kein FP16/BF16. + * Wenn CUDA nicht nutzbar ist, automatisch CPU mit einer verstaendlichen Notiz. + """ + device = (requested_device or "auto").strip().lower() + requested_compute = (requested_compute_type or "auto").strip().lower() + notes: list[str] = [] + + if device in {"auto", ""}: + device = "cuda" if runtime.cuda_runtime_usable else "cpu" + notes.append(f"device=auto auf '{device}' aufgeloest.") + + if device == "cuda" and not runtime.cuda_runtime_usable: + notes.append( + "CUDA ist nicht nutzbar (keine CUDA-Geraete oder keine CUDA-Compute-Types) - " + "es wird die CPU verwendet." + ) + device = "cpu" + + if device == "cuda" and gpu is None: + notes.append("nvidia-smi lieferte keine GPU-Informationen; CUDA wird trotzdem versucht.") + + cc = _cc_tuple(gpu.compute_capability) if gpu else None + if device == "cuda" and cc is not None and cc[0] <= 6: + if requested_compute in FORBIDDEN_GPU_COMPUTE_TYPES: + notes.append( + f"'{requested_compute}' ist auf Pascal (CC {gpu.compute_capability}) nicht sinnvoll - " + f"stattdessen {PREFERRED_COMPUTE_TYPE}." + ) + requested_compute = PREFERRED_COMPUTE_TYPE + + if device == "cpu": + supported = runtime.cpu_compute_types or ["int8", "float32"] + preference = [requested_compute, "int8", "int8_float32", "float32"] + notes.append("CPU-Fallback aktiv: Transkription laeuft ohne GPU und ist deutlich langsamer.") + else: + supported = runtime.cuda_compute_types or [] + if gpu is not None and gpu.compute_capability: + supported = [item for item in supported] + # Pascal: FP16/BF16 nie automatisch waehlen. + if cc is not None and cc[0] <= 6: + supported = [item for item in supported if item not in FORBIDDEN_GPU_COMPUTE_TYPES] + preference = [requested_compute, PREFERRED_COMPUTE_TYPE, "float32", "int8"] + + chosen = "" + for candidate in preference: + if candidate in {"", "auto"}: + continue + if candidate in supported: + chosen = candidate + break + + if not chosen: + # Letzter Ausweg: erster gemeldeter Typ. + chosen = supported[0] if supported else "float32" + notes.append( + f"Kein bevorzugter Compute Type verfuegbar; verwende '{chosen}'." + ) + elif chosen != requested_compute and requested_compute not in {"", "auto"}: + notes.append( + f"'{requested_compute}' ist nicht verfuegbar - verwende '{chosen}'." + ) + + if device == "cuda" and chosen in FORBIDDEN_GPU_COMPUTE_TYPES and cc is not None and cc[0] <= 6: + chosen = PREFERRED_COMPUTE_TYPE + notes.append("FP16/BF16 auf Pascal verhindert, int8_float32 erzwungen.") + + return ResolvedExecution( + device=device, + compute_type=chosen, + requested_device=requested_device, + requested_compute_type=requested_compute_type, + notes=notes, + ) diff --git a/sjaudiosearch/transcription/engine.py b/sjaudiosearch/transcription/engine.py new file mode 100644 index 0000000..578d298 --- /dev/null +++ b/sjaudiosearch/transcription/engine.py @@ -0,0 +1,179 @@ +"""faster-whisper-Engine. + +Das Modell wird genau einmal geladen und danach fuer alle Dateien +wiederverwendet. Die Audiodateien werden von faster-whisper gestreamt +dekodiert, es wird nie eine komplette Sammlung in den RAM geladen. +""" + +from __future__ import annotations + +import math +from pathlib import Path +from typing import Any + +from ..config import TranscriptionConfig +from ..utils.logging import get_logger +from .models import TranscriptResult, TranscriptSegment, WordToken + +logger = get_logger("transcription.engine") + + +class TranscriptionError(RuntimeError): + """Fehler beim Laden des Modells oder beim Transkribieren.""" + + +class FasterWhisperEngine: + """Duenner Wrapper um :class:`faster_whisper.WhisperModel`.""" + + def __init__( + self, + settings: TranscriptionConfig, + *, + device: str | None = None, + compute_type: str | None = None, + model_dir: str | Path | None = None, + ): + self.settings = settings + self.device = (device or settings.device).strip().lower() + self.compute_type = (compute_type or settings.compute_type).strip().lower() + self.model_dir = Path(model_dir) if model_dir else None + self._model: Any | None = None + + # ------------------------------------------------------------------ + @property + def model_name(self) -> str: + return self.settings.model + + @property + def is_loaded(self) -> bool: + return self._model is not None + + @property + def language(self) -> str | None: + return self.settings.language_for_whisper + + # ------------------------------------------------------------------ + def load(self) -> None: + """Laedt das Modell (idempotent).""" + if self._model is not None: + return + + # CUDA-Bibliotheken aus den pip-nvidia-Paketen verfuegbar machen, bevor + # CTranslate2 sie laedt (sonst: "libcublas.so.12 is not found"). + if self.device == "cuda": + from ..system.gpu import ensure_cuda_libraries + + ensure_cuda_libraries() + + try: + from faster_whisper import WhisperModel + except ImportError as exc: # pragma: no cover - Installationsthema + raise TranscriptionError( + "faster-whisper ist nicht installiert. Bitte die Projektabhaengigkeiten installieren." + ) from exc + + download_root = str(self.model_dir) if self.model_dir else None + if download_root: + Path(download_root).mkdir(parents=True, exist_ok=True) + + logger.info( + "Lade Whisper-Modell %s (device=%s, compute_type=%s)", + self.model_name, + self.device, + self.compute_type, + ) + try: + self._model = WhisperModel( + self.model_name, + device=self.device, + compute_type=self.compute_type, + download_root=download_root, + ) + except Exception as exc: # CTranslate2 wirft diverse RuntimeError + raise TranscriptionError( + f"Modell '{self.model_name}' konnte nicht geladen werden " + f"(device={self.device}, compute_type={self.compute_type}): {exc}" + ) from exc + + def unload(self) -> None: + self._model = None + + # ------------------------------------------------------------------ + def transcribe(self, path: str | Path) -> TranscriptResult: + """Transkribiert eine Audiodatei vollstaendig.""" + if self._model is None: + self.load() + assert self._model is not None + + settings = self.settings + transcribe_kwargs: dict[str, Any] = { + "language": settings.language_for_whisper, + "beam_size": max(1, int(settings.beam_size)), + "word_timestamps": bool(settings.word_timestamps), + "vad_filter": bool(settings.vad_filter), + # Dialogpassagen nicht durch aggressive Stille-Erkennung abschneiden. + "condition_on_previous_text": False, + } + if settings.vad_filter: + transcribe_kwargs["vad_parameters"] = { + "min_silence_duration_ms": 700, + "speech_pad_ms": 300, + } + + try: + segments_iter, info = self._model.transcribe(str(path), **transcribe_kwargs) + except Exception as exc: + raise TranscriptionError(f"Transkription fehlgeschlagen: {exc}") from exc + + segments: list[TranscriptSegment] = [] + text_parts: list[str] = [] + + try: + for segment in segments_iter: + words: list[WordToken] = [] + for word in getattr(segment, "words", None) or []: + words.append( + WordToken( + word=word.word, + start=float(word.start), + end=float(word.end), + probability=getattr(word, "probability", None), + ) + ) + confidence = None + avg_logprob = getattr(segment, "avg_logprob", None) + if avg_logprob is not None: + # Nur als interne Gueteinformation, keine Wahrscheinlichkeit + # im Sinne der Suche. + try: + confidence = float(math.exp(float(avg_logprob))) + except (OverflowError, ValueError): + confidence = None + + text = (segment.text or "").strip() + if text: + text_parts.append(text) + segments.append( + TranscriptSegment( + text=text, + start=float(segment.start), + end=float(segment.end), + confidence=confidence, + words=words, + ) + ) + except Exception as exc: + raise TranscriptionError(f"Fehler beim Lesen der Segmente: {exc}") from exc + + full_text = " ".join(part for part in text_parts if part).strip() + duration = getattr(info, "duration", None) + language = getattr(info, "language", None) or settings.language_for_whisper + + return TranscriptResult( + segments=segments, + full_text=full_text, + language=language, + language_probability=getattr(info, "language_probability", None), + duration=float(duration) if duration else None, + model=self.model_name, + ) diff --git a/sjaudiosearch/transcription/models.py b/sjaudiosearch/transcription/models.py new file mode 100644 index 0000000..143417f --- /dev/null +++ b/sjaudiosearch/transcription/models.py @@ -0,0 +1,41 @@ +"""Datenmodelle fuer Transkriptionsergebnisse.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class WordToken: + """Ein Wort mit Zeitstempel aus Whisper.""" + + word: str + start: float + end: float + probability: float | None = None + + +@dataclass +class TranscriptSegment: + """Ein Whisper-Segment.""" + + text: str + start: float + end: float + confidence: float | None = None + words: list[WordToken] = field(default_factory=list) + + +@dataclass +class TranscriptResult: + """Vollstaendiges Transkript einer Audiodatei.""" + + segments: list[TranscriptSegment] + full_text: str + language: str | None = None + language_probability: float | None = None + duration: float | None = None + model: str = "" + + def __len__(self) -> int: + return len(self.segments) diff --git a/sjaudiosearch/transcription/worker.py b/sjaudiosearch/transcription/worker.py new file mode 100644 index 0000000..9a24100 --- /dev/null +++ b/sjaudiosearch/transcription/worker.py @@ -0,0 +1,204 @@ +"""Transkriptionswarteschlange. + +Eine Datei nach der anderen, Modell nur einmal geladen, Fehler einer Datei +brechen den Lauf nicht ab. Der Status wird dauerhaft gespeichert, sodass ein +Neustart an der letzten Stelle fortsetzt. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Protocol + +from ..config import ProcessingConfig +from ..core.models import AudioFileRecord, FileStatus +from ..database.repository import Repository +from ..utils.logging import get_logger +from .models import TranscriptResult + +logger = get_logger("transcription.worker") + + +class EngineProtocol(Protocol): + """Minimal benoetigte Engine-Schnittstelle (fuer Tests austauschbar).""" + + model_name: str + + def transcribe(self, path: str | Path) -> TranscriptResult: ... + + +@dataclass +class WorkerEvent: + """Ereignis fuer die Fortschrittsanzeige.""" + + kind: str # start | done | error | skipped | finished + record: AudioFileRecord | None = None + message: str | None = None + elapsed: float = 0.0 + audio_duration: float | None = None + stats: "RunStats | None" = None + + +@dataclass +class RunStats: + """Zusammenfassung eines Transkriptionslaufs.""" + + queued: int = 0 + processed: int = 0 + failed: int = 0 + skipped: int = 0 + audio_seconds: float = 0.0 + elapsed: float = 0.0 + model: str = "" + device: str = "" + compute_type: str = "" + errors: list[tuple[str, str]] = field(default_factory=list) + + @property + def rate(self) -> float: + if self.elapsed <= 0: + return 0.0 + return self.audio_seconds / self.elapsed + + +class TranscriptionWorker: + """Verarbeitet die Transkriptionswarteschlange.""" + + def __init__( + self, + repository: Repository, + engine: EngineProtocol, + processing: ProcessingConfig | None = None, + *, + device: str = "", + compute_type: str = "", + ): + self.repo = repository + self.engine = engine + self.processing = processing or ProcessingConfig() + self.stats = RunStats( + model=getattr(engine, "model_name", ""), + device=device, + compute_type=compute_type, + ) + + # ------------------------------------------------------------------ + def run( + self, + *, + limit: int | None = None, + force: bool = False, + event_callback: Callable[[WorkerEvent], None] | None = None, + ) -> RunStats: + """Arbeitet die Warteschlange ab.""" + if self.processing.resume: + reset = self.repo.reset_stale_processing() + if reset: + logger.info("%d unterbrochene Eintraege zurueckgesetzt", reset) + + queue = self.repo.transcription_queue(force=force) + if limit is not None: + queue = queue[: max(0, int(limit))] + self.stats.queued = len(queue) + + started_at = time.monotonic() + try: + for record in queue: + self._process(record, event_callback) + except KeyboardInterrupt: + logger.warning("Transkription durch Benutzer abgebrochen") + raise + finally: + self.stats.elapsed = time.monotonic() - started_at + + if event_callback: + event_callback(WorkerEvent(kind="finished", stats=self.stats)) + + logger.info( + "Transkription beendet: %d verarbeitet, %d Fehler, %d uebersprungen, " + "%.1f s Audio in %.1f s (%.2fx Echtzeit)", + self.stats.processed, + self.stats.failed, + self.stats.skipped, + self.stats.audio_seconds, + self.stats.elapsed, + self.stats.rate, + ) + return self.stats + + # ------------------------------------------------------------------ + def _process( + self, record: AudioFileRecord, event_callback: Callable[[WorkerEvent], None] | None + ) -> None: + path = Path(record.path) + if not path.is_file(): + message = f"Datei nicht gefunden: {record.path}" + self.repo.set_status(record.id, FileStatus.ERROR.value, message) + self.stats.failed += 1 + self.stats.errors.append((record.path, message)) + if event_callback: + event_callback(WorkerEvent(kind="error", record=record, message=message)) + return + + self.repo.set_status(record.id, FileStatus.PROCESSING.value) + if event_callback: + event_callback(WorkerEvent(kind="start", record=record)) + + file_started = time.monotonic() + try: + result = self.engine.transcribe(path) + if not result.segments: + raise ValueError("Keine Sprache im Transkript erkannt") + + self.repo.save_transcript( + record.id, + language=result.language, + model=result.model or self.engine.model_name, + full_text=result.full_text, + segments=result.segments, + duration=result.duration or record.duration, + ) + except KeyboardInterrupt: + self.repo.set_status(record.id, FileStatus.PENDING.value) + raise + except Exception as exc: + elapsed = time.monotonic() - file_started + message = f"{type(exc).__name__}: {exc}" + logger.exception("Transkription fehlgeschlagen fuer %s", record.path) + self.repo.set_status(record.id, FileStatus.ERROR.value, message) + self.stats.failed += 1 + self.stats.errors.append((record.path, message)) + if event_callback: + event_callback( + WorkerEvent( + kind="error", + record=record, + message=message, + elapsed=elapsed, + audio_duration=record.duration, + ) + ) + return + + elapsed = time.monotonic() - file_started + audio_duration = result.duration or record.duration or 0.0 + self.stats.processed += 1 + self.stats.audio_seconds += float(audio_duration or 0.0) + logger.info( + "Fertig: %s (%.1f s Audio, %.1f s Laufzeit, %d Segmente)", + record.filename, + audio_duration or 0.0, + elapsed, + len(result.segments), + ) + if event_callback: + event_callback( + WorkerEvent( + kind="done", + record=record, + elapsed=elapsed, + audio_duration=audio_duration, + ) + ) diff --git a/sjaudiosearch/utils/formatting.py b/sjaudiosearch/utils/formatting.py new file mode 100644 index 0000000..23deaef --- /dev/null +++ b/sjaudiosearch/utils/formatting.py @@ -0,0 +1,95 @@ +"""Formatierungshelfer fuer Zeitangaben, Groessen und Pfade.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path + + +def format_timestamp(seconds: float | int | None) -> str: + """Formatiert Sekunden als HH:MM:SS bzw. MM:SS.""" + if seconds is None: + return "--:--" + seconds = max(0, int(round(float(seconds)))) + hours, remainder = divmod(seconds, 3600) + minutes, secs = divmod(remainder, 60) + if hours: + return f"{hours:02d}:{minutes:02d}:{secs:02d}" + return f"{minutes:02d}:{secs:02d}" + + +def format_timestamp_verbose(seconds: float | int | None) -> str: + """Formatiert Sekunden immer als HH:MM:SS (fuer Fundstellen in Hoerspielen).""" + if seconds is None: + return "--:--:--" + seconds = max(0, int(round(float(seconds)))) + hours, remainder = divmod(seconds, 3600) + minutes, secs = divmod(remainder, 60) + return f"{hours:02d}:{minutes:02d}:{secs:02d}" + + +def format_duration(seconds: float | int | None) -> str: + """Menschlich lesbare Dauer, z. B. '2 h 14 min'.""" + if seconds is None: + return "unbekannt" + total = max(0, int(round(float(seconds)))) + hours, remainder = divmod(total, 3600) + minutes, secs = divmod(remainder, 60) + if hours: + return f"{hours} h {minutes:02d} min" + if minutes: + return f"{minutes} min {secs:02d} s" + return f"{secs} s" + + +def format_size(num_bytes: int | float | None) -> str: + """Formatiert Bytes als B/KB/MB/GB.""" + if num_bytes is None: + return "?" + value = float(num_bytes) + for unit in ("B", "KB", "MB", "GB", "TB"): + if value < 1024 or unit == "TB": + if unit == "B": + return f"{int(value)} {unit}" + return f"{value:.1f} {unit}" + value /= 1024 + return f"{value:.1f} TB" + + +def format_score(score: float) -> str: + """Aehnlichkeitswert als Prozentangabe (keine statistische Wahrscheinlichkeit).""" + return f"{score:.1f}%" + + +def format_rate(audio_seconds: float, elapsed_seconds: float) -> str: + """Verarbeitungsgeschwindigkeit als Vielfaches der Echtzeit.""" + if elapsed_seconds <= 0: + return "-" + return f"{audio_seconds / elapsed_seconds:.2f}x Echtzeit" + + +def format_datetime(iso_timestamp: str | None) -> str: + """Formatiert einen ISO-String lesbar; unbekannte Formate werden durchgereicht.""" + if not iso_timestamp: + return "-" + try: + dt = datetime.fromisoformat(iso_timestamp) + except ValueError: + return iso_timestamp + if dt.tzinfo is not None: + dt = dt.astimezone() + return dt.strftime("%Y-%m-%d %H:%M") + + +def utc_now_iso() -> str: + """Aktueller Zeitpunkt als ISO-String in UTC (sekundengenau).""" + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def shorten_path(path: str | Path, max_length: int = 80) -> str: + """Kuerzt lange Pfade in der Mitte.""" + text = str(path) + if len(text) <= max_length: + return text + keep = max(8, (max_length - 3) // 2) + return f"{text[:keep]}...{text[-keep:]}" diff --git a/sjaudiosearch/utils/hashing.py b/sjaudiosearch/utils/hashing.py new file mode 100644 index 0000000..4c8db36 --- /dev/null +++ b/sjaudiosearch/utils/hashing.py @@ -0,0 +1,38 @@ +"""Hash-Helfer fuer die Duplikaterkennung.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +CHUNK_SIZE = 1024 * 1024 + + +def sha256_file(path: str | Path, chunk_size: int = CHUNK_SIZE) -> str: + """Berechnet den SHA256-Hash einer Datei, ohne sie komplett in den RAM zu laden.""" + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + while True: + chunk = handle.read(chunk_size) + if not chunk: + break + digest.update(chunk) + return digest.hexdigest() + + +def quick_signature(path: str | Path, sample_size: int = 64 * 1024) -> str: + """Schnelle Signatur aus Groesse, erster und letzter Datenportion. + + Nur fuer Vorpruefungen gedacht - fuer belastbare Duplikaterkennung muss + :func:`sha256_file` verwendet werden. + """ + path = Path(path) + size = path.stat().st_size + digest = hashlib.sha256() + digest.update(str(size).encode("ascii")) + with path.open("rb") as handle: + digest.update(handle.read(sample_size)) + if size > sample_size: + handle.seek(max(sample_size, size - sample_size)) + digest.update(handle.read(sample_size)) + return digest.hexdigest() diff --git a/sjaudiosearch/utils/logging.py b/sjaudiosearch/utils/logging.py new file mode 100644 index 0000000..267ada9 --- /dev/null +++ b/sjaudiosearch/utils/logging.py @@ -0,0 +1,62 @@ +"""Logging-Setup fuer SjAudioSearch. + +Protokolliert in eine rotierende Logdatei und optional zusaetzlich auf die +Konsole (Rich). Die CLI schaltet den Konsolen-Handler nur bei Bedarf ein. +""" + +from __future__ import annotations + +import logging +from logging.handlers import RotatingFileHandler +from pathlib import Path + +from rich.logging import RichHandler + +LOGGER_NAME = "sjaudiosearch" +_CONFIGURED = False + + +def setup_logging(log_path: Path, level: str = "INFO", console: bool = False) -> logging.Logger: + """Konfiguriert den Logger ``sjaudiosearch`` idempotent.""" + global _CONFIGURED + + logger = logging.getLogger(LOGGER_NAME) + numeric_level = getattr(logging, str(level).upper(), logging.INFO) + logger.setLevel(numeric_level) + + if _CONFIGURED: + return logger + + log_path = Path(log_path) + log_path.parent.mkdir(parents=True, exist_ok=True) + + formatter = logging.Formatter( + "%(asctime)s | %(levelname)-8s | %(name)s | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + file_handler = RotatingFileHandler( + log_path, + maxBytes=5 * 1024 * 1024, + backupCount=3, + encoding="utf-8", + ) + file_handler.setFormatter(formatter) + file_handler.setLevel(numeric_level) + logger.addHandler(file_handler) + + if console: + console_handler = RichHandler(show_path=False, rich_tracebacks=True, markup=False) + console_handler.setLevel(max(numeric_level, logging.INFO)) + logger.addHandler(console_handler) + + logger.propagate = False + _CONFIGURED = True + return logger + + +def get_logger(name: str | None = None) -> logging.Logger: + """Liefert einen Sub-Logger unterhalb von ``sjaudiosearch``.""" + if name: + return logging.getLogger(f"{LOGGER_NAME}.{name}") + return logging.getLogger(LOGGER_NAME) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..2a3fccc --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,168 @@ +"""Gemeinsame Fixtures fuer die SjAudioSearch-Tests. + +Die Tests erzeugen winzige echte WAV-Dateien und verwenden eine gefakte +Transkriptions-Engine. Es wird nie ein Whisper-Modell heruntergeladen. +""" + +from __future__ import annotations + +import os +import wave +from pathlib import Path +from typing import Callable, Iterable, Sequence + +import pytest + +from sjaudiosearch.config import ProcessingConfig, SearchConfig +from sjaudiosearch.core.models import AudioFileRecord, ScanCandidate +from sjaudiosearch.database.repository import Repository +from sjaudiosearch.transcription.models import TranscriptResult, TranscriptSegment, WordToken + + +def write_wav(path: Path, seconds: float = 1.0, sample_rate: int = 8000) -> Path: + """Schreibt eine stille WAV-Datei (gueltiges Audio, sehr klein).""" + path.parent.mkdir(parents=True, exist_ok=True) + frames = int(sample_rate * seconds) + with wave.open(str(path), "wb") as handle: + handle.setnchannels(1) + handle.setsampwidth(2) + handle.setframerate(sample_rate) + handle.writeframes(b"\x00\x00" * frames) + return path + + +@pytest.fixture +def repo(tmp_path: Path) -> Iterable[Repository]: + repository = Repository.open(tmp_path / "sjaudiosearch.db") + try: + yield repository + finally: + repository.close() + + +@pytest.fixture +def processing() -> ProcessingConfig: + return ProcessingConfig(compute_hash="auto") + + +@pytest.fixture +def search_config() -> SearchConfig: + return SearchConfig( + default_limit=50, + default_min_score=70.0, + context_words=50, + exhaustive_fallback=True, + ) + + +@pytest.fixture +def make_wav(tmp_path: Path) -> Callable[..., Path]: + def factory(name: str, seconds: float = 1.0, sample_rate: int = 8000) -> Path: + path = tmp_path / name + return write_wav(path, seconds=seconds, sample_rate=sample_rate) + + return factory + + +@pytest.fixture +def add_audio(repo: Repository) -> Callable[..., AudioFileRecord]: + def factory( + path: str | Path, + *, + duration: float | None = 60.0, + status: str = "pending", + content_hash: str | None = None, + duplicate_of: int | None = None, + ) -> AudioFileRecord: + file_path = Path(path) + if not file_path.exists(): + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.write_bytes(b"\x00") + candidate = ScanCandidate( + path=str(file_path), + filename=file_path.name, + file_size=file_path.stat().st_size, + modified_at=file_path.stat().st_mtime, + duration=duration, + content_hash=content_hash, + ) + audio_id = repo.insert_audio( + candidate, status=status, duplicate_of=duplicate_of + ) + record = repo.get_audio(audio_id) + assert record is not None + return record + + return factory + + +@pytest.fixture +def store_transcript(repo: Repository) -> Callable[..., int]: + """Speichert ein Transkript mit optionalen Wort-Zeitstempeln.""" + + def factory( + audio_id: int, + segments: Sequence[str | tuple[str, float, float]], + *, + language: str = "de", + model: str = "test-model", + with_words: bool = False, + duration: float = 120.0, + ) -> int: + built: list[TranscriptSegment] = [] + cursor = 0.0 + for item in segments: + if isinstance(item, str): + text = item + start = cursor + end = cursor + 5.0 + else: + text, start, end = item + cursor = end + + words: list[WordToken] = [] + if with_words: + tokens = text.split() + step = (end - start) / max(1, len(tokens)) + words = [ + WordToken( + word=token, + start=start + index * step, + end=start + (index + 1) * step, + ) + for index, token in enumerate(tokens) + ] + + built.append( + TranscriptSegment( + text=text, + start=float(start), + end=float(end), + confidence=0.9, + words=words, + ) + ) + + full_text = " ".join(segment.text for segment in built) + result = TranscriptResult( + segments=built, + full_text=full_text, + language=language, + language_probability=0.99, + duration=duration, + model=model, + ) + return repo.save_transcript( + audio_id, + language=result.language, + model=result.model, + full_text=result.full_text, + segments=result.segments, + duration=result.duration, + ) + + return factory + + +def set_mtime(path: Path, seconds: float) -> None: + os.utime(path, (seconds, seconds)) diff --git a/tests/test_context.py b/tests/test_context.py new file mode 100644 index 0000000..4593f09 --- /dev/null +++ b/tests/test_context.py @@ -0,0 +1,101 @@ +"""Tests fuer Timeline, Kontextfenster und Zeitstempel.""" + +from __future__ import annotations + +import pytest + +from sjaudiosearch.search.context import build_context, build_timeline + + +def _segments(*specs): + return [ + { + "id": index + 1, + "start_time": start, + "end_time": end, + "text": text, + "confidence": 0.9, + } + for index, (text, start, end) in enumerate(specs) + ] + + +def test_context_contains_fifty_words_before_and_after(): + prefix = " ".join(f"wort{i}" for i in range(80)) + suffix = " ".join(f"danach{i}" for i in range(80)) + text = f"{prefix} der schlüssel liegt im alten haus {suffix}" + timeline = build_timeline(1, _segments((text, 0.0, 60.0))) + + start_index = len(prefix.split()) + context = build_context(timeline, start_index, start_index + 6, context_words=50) + + assert len(context.before.split()) == 50 + assert len(context.after.split()) == 50 + assert context.before.split()[-1] == "wort79" + assert context.after.split()[0] == "danach0" + assert context.match.split()[0] == "der" + assert "haus" in context.match + + +def test_context_spans_segment_boundaries_without_loss(): + first = "Der Professor betrat den Raum und sah sich suchend um" + second = "der Schlüssel lag im alten Haus auf dem Tisch" + timeline = build_timeline(1, _segments((first, 5.0, 10.0), (second, 10.0, 15.0))) + + assert len(timeline.tokens) == len(first.split()) + len(second.split()) + start_index = len(first.split()) + context = build_context(timeline, start_index, start_index + 4, context_words=50) + + assert context.before.endswith("suchend um") + assert context.match.startswith("der Schlüssel") + assert context.start_time == pytest.approx(10.0, abs=1.5) + + +def test_timestamps_use_word_precision_when_available(): + texts = ["Eins zwei drei vier fünf"] + segments = _segments((texts[0], 0.0, 5.0)) + words = { + 1: [ + {"word": "Eins", "start_time": 10.0, "end_time": 10.5}, + {"word": "zwei", "start_time": 10.5, "end_time": 11.0}, + {"word": "drei", "start_time": 11.0, "end_time": 11.5}, + {"word": "vier", "start_time": 11.5, "end_time": 12.0}, + {"word": "fünf", "start_time": 12.0, "end_time": 12.5}, + ] + } + timeline = build_timeline(1, segments, words) + context = build_context(timeline, 2, 4, context_words=10) + + assert context.precision == "word" + assert context.start_time == pytest.approx(11.0) + assert context.end_time == pytest.approx(12.0) + assert context.match == "drei vier" + + +def test_timestamps_fall_back_to_segment_precision(): + timeline = build_timeline(1, _segments(("Eins zwei drei vier", 30.0, 34.0))) + context = build_context(timeline, 0, 2, context_words=5) + + assert context.precision == "segment" + assert 30.0 <= context.start_time <= 32.0 + assert context.end_time <= 34.0 + + +def test_context_block_formatting(): + timeline = build_timeline(1, _segments(("vorher der Schlüssel nachher", 0.0, 5.0))) + context = build_context(timeline, 1, 2, context_words=10) + block = context.as_block() + + assert block.startswith('"\u2026vorher') + assert "der Schlüssel" in block + assert block.endswith('nachher\u2026"') + + alone = build_context(timeline, 0, 4, context_words=0) + assert alone.as_block() == '"vorher der Schlüssel nachher"' + + +def test_empty_timeline_is_safe(): + timeline = build_timeline(1, []) + context = build_context(timeline, 0, 1, 50) + assert context.match == "" + assert context.start_time == 0.0 diff --git a/tests/test_database.py b/tests/test_database.py new file mode 100644 index 0000000..97a9710 --- /dev/null +++ b/tests/test_database.py @@ -0,0 +1,167 @@ +"""Tests fuer Datenbank, Repository, Migrationen und Suchindex.""" + +from __future__ import annotations + +from pathlib import Path + +from sjaudiosearch.database.migrations import current_version, migrate +from sjaudiosearch.database.repository import Repository + + +def test_schema_and_migration_version(repo): + assert current_version(repo.conn) == 1 + # Migration ist idempotent. + assert migrate(repo.conn) == 1 + tables = { + row["name"] + for row in repo.conn.execute( + "SELECT name FROM sqlite_master WHERE type IN ('table', 'view')" + ) + } + for table in ( + "audio_files", + "transcripts", + "segments", + "words", + "search_index", + "search_trigram", + ): + assert table in tables + + +def test_transcript_stored_with_segments_and_words(repo, add_audio, store_transcript): + record = add_audio("/tmp/folge1.mp3", duration=125.0) + store_transcript( + record.id, + [("Der Professor sah sich um.", 0.0, 4.0), ("Der Schlüssel liegt im alten Haus.", 4.0, 9.5)], + with_words=True, + duration=9.5, + ) + + stored = repo.get_transcript(record.id) + assert stored is not None + assert stored["model"] == "test-model" + assert stored["language"] == "de" + assert "Schlüssel" in stored["full_text"] + + segments = repo.get_segments(record.id) + assert len(segments) == 2 + assert segments[1]["start_time"] == 4.0 + + words = repo.get_words([int(segment["id"]) for segment in segments]) + first_segment_words = words[int(segments[0]["id"])] + assert [row["word"] for row in first_segment_words][:2] == ["Der", "Professor"] + + updated = repo.get_audio(record.id) + assert updated.status == "done" + assert updated.duration == 125.0 + + +def test_status_management_and_retry(repo, add_audio): + ok = add_audio("/tmp/a.mp3") + failing = add_audio("/tmp/b.mp3") + + repo.set_status(ok.id, "processing") + repo.set_status(failing.id, "error", "Decoder kaputt") + + counts = repo.count_by_status() + assert counts["processing"] == 1 + assert counts["error"] == 1 + + # Abgestuerzte 'processing'-Eintraege werden zurueckgesetzt + assert repo.reset_stale_processing() == 1 + assert repo.get_audio(ok.id).status == "pending" + + assert repo.reset_errors() == 1 + reset = repo.get_audio(failing.id) + assert reset.status == "pending" + assert reset.error_message is None + + +def test_queue_skips_duplicates_and_done(repo, add_audio): + done = add_audio("/tmp/done.mp3", status="done") + pending = add_audio("/tmp/pending.mp3") + duplicate = add_audio("/tmp/dup.mp3", duplicate_of=done.id) + + queue = repo.transcription_queue() + assert [record.id for record in queue] == [pending.id] + + forced = repo.transcription_queue(force=True) + assert {record.id for record in forced} == {done.id, pending.id} + assert duplicate.id not in {record.id for record in forced} + + +def test_delete_audio_keeps_file_and_removes_index(repo, add_audio, store_transcript, tmp_path): + audio_file = tmp_path / "folge.mp3" + audio_file.write_bytes(b"audio") + record = add_audio(audio_file) + store_transcript(record.id, ["Der Schlüssel liegt im alten Haus."]) + + assert repo.fts_candidates('"schlüssel"', 10) + + assert repo.delete_audio(record.id) is True + assert audio_file.exists() + assert repo.get_audio(record.id) is None + assert repo.get_transcript(record.id) is None + assert repo.fts_candidates('"schlüssel"', 10) == [] + + +def test_foreign_key_cascade(repo, add_audio, store_transcript): + record = add_audio("/tmp/a.mp3") + store_transcript(record.id, ["Erster Satz.", "Zweiter Satz."]) + assert repo.get_segments(record.id) + + with repo.conn: + repo.conn.execute("DELETE FROM audio_files WHERE id = ?", (record.id,)) + + segments = repo.conn.execute("SELECT COUNT(*) AS amount FROM segments").fetchone() + words = repo.conn.execute("SELECT COUNT(*) AS amount FROM words").fetchone() + transcripts = repo.conn.execute("SELECT COUNT(*) AS amount FROM transcripts").fetchone() + assert segments["amount"] == 0 + assert words["amount"] == 0 + assert transcripts["amount"] == 0 + + +def test_reindex_preserves_transcripts(repo, add_audio, store_transcript): + record = add_audio("/tmp/a.mp3") + store_transcript(record.id, ["Der Schlüssel liegt im alten Haus.", "Niemand bemerkte es."]) + + before_transcript = dict(repo.get_transcript(record.id)) + before_segments = [dict(row) for row in repo.get_segments(record.id)] + assert repo.fts_candidates('"schlüssel"', 10) + + indexed = repo.rebuild_search_index() + assert indexed == 2 + + after_transcript = dict(repo.get_transcript(record.id)) + after_segments = [dict(row) for row in repo.get_segments(record.id)] + assert before_transcript == after_transcript + assert before_segments == after_segments + assert repo.fts_candidates('"schlüssel"', 10) + + +def test_saving_new_transcript_replaces_old(repo, add_audio, store_transcript): + record = add_audio("/tmp/a.mp3") + store_transcript(record.id, ["Alter Text mit Schlüssel."]) + store_transcript(record.id, ["Neuer Text ohne alles."]) + + transcript = repo.get_transcript(record.id) + assert transcript["full_text"] == "Neuer Text ohne alles." + assert len(repo.get_segments(record.id)) == 1 + assert repo.fts_candidates('"schlüssel"', 10) == [] + + +def test_database_size_and_stats(repo, add_audio, store_transcript): + first = add_audio("/tmp/a.mp3", duration=100.0) + second = add_audio("/tmp/b.mp3", duration=50.0, status="error") + store_transcript(first.id, ["Ein Satz."], duration=100.0) + + stats = repo.stats(repo.databases_size()) + assert stats.total_files == 2 + assert stats.done == 1 + assert stats.error == 1 + assert stats.total_duration == 150.0 + assert stats.transcripts == 1 + assert stats.segment_count == 1 + assert stats.database_size > 0 + assert "test-model" in stats.models diff --git a/tests/test_scanner.py b/tests/test_scanner.py new file mode 100644 index 0000000..1415119 --- /dev/null +++ b/tests/test_scanner.py @@ -0,0 +1,131 @@ +"""Tests fuer den rekursiven Audio-Scanner.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sjaudiosearch.core.metadata import AudioMetadataError, read_metadata +from sjaudiosearch.core.scanner import scan +from conftest import set_mtime + + +def test_scan_finds_multiple_audio_files(repo, make_wav, tmp_path, processing): + music = tmp_path / "music" + make_wav("music/a.mp3.wav", 1.0) + make_wav("music/sub/b.wav", 1.0) + make_wav("music/sub/deep/c.wav", 1.0) + (music / "notes.txt").write_text("kein Audio", encoding="utf-8") + + result = scan(music, repo, processing) + + assert result.files_found == 3 + assert result.added == 3 + assert result.changed == 0 + assert result.errors == [] + + records = repo.list_audio() + assert len(records) == 3 + assert {record.filename for record in records} == {"a.mp3.wav", "b.wav", "c.wav"} + for record in records: + assert record.status == "pending" + assert Path(record.path).is_absolute() + assert record.duration == pytest.approx(1.0, abs=0.05) + assert record.file_size > 0 + + +def test_scan_detects_duplicate_content(repo, make_wav, tmp_path, processing): + root = tmp_path / "music" + original = make_wav("music/original.wav", 2.0) + copy = root / "kopie.wav" + copy.parent.mkdir(parents=True, exist_ok=True) + copy.write_bytes(original.read_bytes()) + + result = scan(root, repo, processing) + + assert result.added == 2 + assert result.duplicates == 1 + + records = {record.filename: record for record in repo.list_audio()} + first, second = records["kopie.wav"], records["original.wav"] + # Der Scan ist alphabetisch sortiert, daher ist "kopie.wav" zuerst dran und + # "original.wav" wird als Duplikat markiert. Genau einer der beiden muss + # auf den anderen zeigen. + assert bool(first.duplicate_of) ^ bool(second.duplicate_of) + duplicate, keeper = (first, second) if first.duplicate_of else (second, first) + assert duplicate.duplicate_of == keeper.id + assert duplicate.content_hash == keeper.content_hash + # Duplikate werden nicht erneut transkribiert. + queue = [record.filename for record in repo.transcription_queue()] + assert queue == [keeper.filename] + + +def test_scan_detects_changed_file(repo, make_wav, tmp_path, processing): + root = tmp_path / "music" + root.mkdir() + first = scan(root, repo, processing) + assert first.added == 0 + + path = make_wav("music/folge1.wav", 1.0) + assert scan(root, repo, processing).added == 1 + record = repo.get_audio_by_path(str(path.resolve())) + assert record is not None and record.status == "pending" + + # Inhalt aendern (andere Laenge) und Aenderungszeit setzen + make_wav("music/folge1.wav", 3.0) + set_mtime(path, path.stat().st_mtime + 30) + + result = scan(root, repo, processing) + assert result.added == 0 + assert result.changed == 1 + assert result.unchanged == 0 + + updated = repo.get_audio(record.id) + assert updated is not None + assert updated.status == "pending" + assert updated.duration == pytest.approx(3.0, abs=0.05) + + +def test_scan_keeps_unchanged_files(repo, make_wav, tmp_path, processing): + root = tmp_path / "music" + make_wav("music/a.wav", 1.0) + make_wav("music/b.wav", 1.0) + + first = scan(root, repo, processing) + assert first.added == 2 + second = scan(root, repo, processing) + assert second.added == 0 + assert second.changed == 0 + assert second.unchanged == 2 + + +def test_scan_logs_corrupt_files(repo, tmp_path, processing): + root = tmp_path / "music" + root.mkdir() + broken = root / "kaputt.mp3" + broken.write_bytes(b"das ist kein audio") + + result = scan(root, repo, processing) + + assert result.files_found == 1 + assert result.added == 1 + assert len(result.errors) == 1 + assert "kaputt.mp3" in result.errors[0].path + + record = repo.get_audio_by_path(str(broken.resolve())) + assert record is not None + assert record.status == "error" + assert record.error_message + + +def test_scan_rejects_missing_path(repo, tmp_path, processing): + with pytest.raises(FileNotFoundError): + scan(tmp_path / "gibt-es-nicht", repo, processing) + + +def test_metadata_requires_real_audio(tmp_path): + fake = tmp_path / "fake.flac" + fake.write_bytes(b"nope") + with pytest.raises(AudioMetadataError): + read_metadata(fake) diff --git a/tests/test_search.py b/tests/test_search.py new file mode 100644 index 0000000..ad2bd33 --- /dev/null +++ b/tests/test_search.py @@ -0,0 +1,192 @@ +"""Tests fuer die Suchmaschine (Fuzzy, Sliding Window, Fallback).""" + +from __future__ import annotations + +import pytest + +from sjaudiosearch.search.engine import SearchEngine, SearchError +from sjaudiosearch.search.fuzzy import build_fts_query, normalize_text, score_window + + +def _engine(repo, search_config) -> SearchEngine: + return SearchEngine(repo, search_config) + + +def test_fuzzy_search_finds_slightly_different_sentence(repo, add_audio, store_transcript, search_config): + record = add_audio("/tmp/hoerspiel_042.mp3", duration=3000.0) + store_transcript( + record.id, + [("Der Professor sah sich um. Der Schlüssel befindet sich in dem alten Haus. " + "Niemand hatte das bisher bemerkt.", 42 * 60 + 10.0, 42 * 60 + 20.0)], + duration=3000.0, + ) + + hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus") + + assert len(hits) == 1 + hit = hits[0] + assert hit.filename == "hoerspiel_042.mp3" + assert hit.path == record.path + assert hit.score >= 85 + assert 42 * 60 <= hit.start_time <= 42 * 60 + 40 + assert "Schlüssel" in hit.matched_text + assert hit.context.before + + +def test_exact_phrase_scores_100(repo, add_audio, store_transcript, search_config): + record = add_audio("/tmp/exakt.mp3") + store_transcript(record.id, ["Der Schlüssel liegt im alten Haus."]) + + hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus") + + assert hits + assert hits[0].score == pytest.approx(100.0, abs=0.1) + + +def test_search_across_segment_boundaries(repo, add_audio, store_transcript, search_config): + record = add_audio("/tmp/krimi.mp3") + store_transcript( + record.id, + [ + ("Der Professor sah sich um. Der Schlüssel", 10.0, 15.0), + ("liegt in dem alten Haus. Niemand bemerkte es.", 15.0, 20.0), + ], + ) + + hits = _engine(repo, search_config).search("Der Schlüssel liegt in dem alten Haus") + + assert hits + hit = hits[0] + assert hit.score >= 90 + assert "Schlüssel" in hit.matched_text + assert "Haus" in hit.matched_text + # Der Treffer beginnt beim Wort "Schlüssel" innerhalb des ersten Segments. + assert 10.0 <= hit.start_time <= 16.0 + + +def test_fallback_finds_word_fts_cannot_match(repo, add_audio, store_transcript, search_config): + """Whisper-Transkripte koennen Woerter komplett falsch schreiben. + + Ohne FTS-Treffer muss der Fallback-Lauf die Datei trotzdem durchsuchen. + """ + record = add_audio("/tmp/misheard.mp3") + store_transcript(record.id, [("Die Sage vom Schluessel im alten Gemäuer.", 0.0, 6.0)]) + + # Keine exakte Uebereinstimmung fuer den Suchbegriff im FTS-Index + assert repo.fts_candidates(build_fts_query(["schlüssel"]), 10) == [] + assert repo.trigram_candidates(["schlüssel"], 10) == [] + + hits = _engine(repo, search_config).search("Schlüssel") + + assert len(hits) == 1 + assert hits[0].audio_id == record.id + assert hits[0].source == "fallback" + assert hits[0].score >= 70 + + +def test_search_returns_nothing_for_unrelated_text(repo, add_audio, store_transcript, search_config): + record = add_audio("/tmp/unrelated.mp3") + store_transcript(record.id, ["Es war einmal ein kleiner Drache im Wald."]) + + hits = _engine(repo, search_config).search("Quantenphysik Vorlesung") + assert hits == [] + + +def test_min_score_and_limit(repo, add_audio, store_transcript, search_config): + for index in range(5): + record = add_audio(f"/tmp/folge{index}.mp3") + store_transcript( + record.id, + [f"Der Schlüssel liegt im alten Haus Nummer {index}.", ], + ) + + engine = _engine(repo, search_config) + all_hits = engine.search("Der Schlüssel liegt im alten Haus", limit=10, min_score=50) + assert len(all_hits) >= 1 + + limited = engine.search("Der Schlüssel liegt im alten Haus", limit=2, min_score=50) + assert len(limited) == 2 + + strict = engine.search("Der Schlüssel liegt im alten Haus", limit=10, min_score=99.5) + assert len(strict) <= len(all_hits) + + +def test_results_sorted_by_score(repo, add_audio, store_transcript, search_config): + exact = add_audio("/tmp/exact.mp3") + store_transcript(exact.id, ["Der Schlüssel liegt im alten Haus."]) + fuzzy = add_audio("/tmp/fuzzy.mp3") + store_transcript(fuzzy.id, ["Der Schlüssel ist wohl im alten Haus zu finden, oder?"]) + + hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus", min_score=60) + + scores = [hit.score for hit in hits] + assert scores == sorted(scores, reverse=True) + assert hits[0].filename == "exact.mp3" + + +def test_language_filter(repo, add_audio, store_transcript, search_config): + german = add_audio("/tmp/de.mp3") + store_transcript(german.id, ["Der Schlüssel liegt im alten Haus."], language="de") + english = add_audio("/tmp/en.mp3") + store_transcript(english.id, ["The key is in the old house."], language="en") + + hits = _engine(repo, search_config).search( + "The key is in the old house", language="en", min_score=60 + ) + assert [hit.audio_id for hit in hits] == [english.id] + + german_hits = _engine(repo, search_config).search( + "Der Schlüssel liegt im alten Haus", language="de", min_score=60 + ) + assert [hit.audio_id for hit in german_hits] == [german.id] + + +def test_duplicate_files_are_not_searched(repo, add_audio, store_transcript, search_config): + original = add_audio("/tmp/original.mp3") + store_transcript(original.id, ["Der Schlüssel liegt im alten Haus."]) + duplicate = add_audio("/tmp/kopie.mp3", duplicate_of=original.id) + + hits = _engine(repo, search_config).search("Der Schlüssel liegt im alten Haus") + assert {hit.audio_id for hit in hits} == {original.id} + + +def test_empty_query_is_rejected(repo, search_config): + with pytest.raises(SearchError): + _engine(repo, search_config).search(" ... ") + + +def test_word_insertions_and_deletions(repo, add_audio, store_transcript, search_config): + record = add_audio("/tmp/insert.mp3") + store_transcript( + record.id, + ["Also, der Schlüssel liegt wirklich direkt im alten Haus, nicht wahr?"], + ) + + hits = _engine(repo, search_config).search("Schlüssel liegt im alten Haus") + assert hits + assert hits[0].score >= 80 + + +def test_multiword_query_matches_whole_phrase_not_single_word(repo, add_audio, store_transcript, search_config): + """Regression: token_set_ratio darf ein Einzelwort nicht als 100% werten.""" + record = add_audio("/tmp/kurz.mp3") + store_transcript(record.id, [("Leider zu spät.", 0.0, 2.0)]) + + hits = _engine(repo, search_config).search("Leider zu spät", limit=5, min_score=70) + + assert hits + assert hits[0].context.match == "Leider zu spät." + assert hits[0].score == pytest.approx(100.0, abs=0.1) + + +def test_normalization_and_scoring_helpers(): + assert normalize_text("Der Schlüssel, liegt im alten Haus!") == "der schlüssel liegt im alten haus" + assert score_window("der schlüssel liegt im alten haus", "der schlüssel liegt im alten haus").score == 100.0 + assert score_window("der schlüssel liegt im alten haus", "").score == 0.0 + assert build_fts_query(["haus"]).startswith('"haus"') + + +def test_partial_window_is_not_a_perfect_match(): + # Ein einzelnes Wort einer mehrwoertrigen Anfrage darf nicht 100 ergeben. + assert score_window("leider zu spät", "leider").score < 70 + assert score_window("der schlüssel liegt im alten haus", "haus").score < 70 diff --git a/tests/test_transcription.py b/tests/test_transcription.py new file mode 100644 index 0000000..25c9c01 --- /dev/null +++ b/tests/test_transcription.py @@ -0,0 +1,247 @@ +"""Tests fuer Transkriptionswarteschlange, Wiederaufnahme und GPU-Diagnose.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sjaudiosearch.config import ProcessingConfig +from sjaudiosearch.core.models import FileStatus +from sjaudiosearch.system.gpu import ( + GpuInfo, + RuntimeInfo, + resolve_execution, +) +from sjaudiosearch.transcription.models import TranscriptResult, TranscriptSegment, WordToken +from sjaudiosearch.transcription.worker import TranscriptionWorker + + +class FakeEngine: + """Ersetzt faster-whisper; laedt nie ein Modell und braucht kein Netzwerk.""" + + model_name = "fake-model" + + def __init__(self, fail_for: set[str] | None = None): + self.fail_for = fail_for or set() + self.calls: list[str] = [] + + def transcribe(self, path: str | Path) -> TranscriptResult: + name = Path(path).name + self.calls.append(name) + if name in self.fail_for: + raise RuntimeError("Simulierter Decoderfehler") + + words = [ + WordToken(word="Der", start=0.0, end=0.4), + WordToken(word="Schlüssel", start=0.4, end=1.0), + WordToken(word="liegt", start=1.0, end=1.4), + WordToken(word="im", start=1.4, end=1.6), + WordToken(word="alten", start=1.6, end=2.0), + WordToken(word="Haus.", start=2.0, end=2.6), + ] + segment = TranscriptSegment( + text="Der Schlüssel liegt im alten Haus.", + start=0.0, + end=2.6, + confidence=0.95, + words=words, + ) + return TranscriptResult( + segments=[segment], + full_text=segment.text, + language="de", + language_probability=0.99, + duration=10.0, + model=self.model_name, + ) + + +def test_worker_transcribes_and_indexes(repo, add_audio, make_wav, search_config): + record = add_audio(make_wav("folge1.wav")) + engine = FakeEngine() + worker = TranscriptionWorker(repo, engine, ProcessingConfig()) + + stats = worker.run() + + assert stats.queued == 1 + assert stats.processed == 1 + assert stats.failed == 0 + assert engine.calls == ["folge1.wav"] + + updated = repo.get_audio(record.id) + assert updated.status == FileStatus.DONE.value + transcript = repo.get_transcript(record.id) + assert transcript is not None + assert transcript["model"] == "fake-model" + + from sjaudiosearch.search.engine import SearchEngine + + hits = SearchEngine(repo, search_config).search("Schlüssel liegt im alten Haus") + assert hits and hits[0].audio_id == record.id + + +def test_worker_isolates_errors(repo, add_audio, make_wav): + good = add_audio(make_wav("good.wav")) + bad = add_audio(make_wav("bad.wav")) + engine = FakeEngine(fail_for={"bad.wav"}) + worker = TranscriptionWorker(repo, engine, ProcessingConfig()) + + stats = worker.run() + + assert stats.queued == 2 + assert stats.processed == 1 + assert stats.failed == 1 + assert stats.errors[0][0].endswith("bad.wav") + + assert repo.get_audio(good.id).status == "done" + failed = repo.get_audio(bad.id) + assert failed.status == "error" + assert "Simulierter Decoderfehler" in failed.error_message + + +def test_worker_skips_done_and_resumes(repo, add_audio, make_wav): + record = add_audio(make_wav("done.wav"), status="done") + engine = FakeEngine() + worker = TranscriptionWorker(repo, engine, ProcessingConfig()) + + stats = worker.run() + assert stats.queued == 0 + assert engine.calls == [] + + stats_forced = worker.run(force=True) + assert stats_forced.queued == 1 + assert engine.calls == ["done.wav"] + assert repo.get_audio(record.id).status == "done" + + +def test_worker_resets_stale_processing(repo, add_audio, make_wav): + record = add_audio(make_wav("haengt.wav"), status="processing") + worker = TranscriptionWorker(repo, FakeEngine(), ProcessingConfig(resume=True)) + + stats = worker.run() + + assert stats.processed == 1 + assert repo.get_audio(record.id).status == "done" + + +def test_worker_limits_queue(repo, add_audio, make_wav): + for index in range(3): + add_audio(make_wav(f"folge{index}.wav")) + engine = FakeEngine() + worker = TranscriptionWorker(repo, engine, ProcessingConfig()) + + stats = worker.run(limit=2) + + assert stats.queued == 2 + assert len(engine.calls) == 2 + + +def test_failed_transcription_can_be_retried(repo, add_audio, make_wav): + record = add_audio(make_wav("retry.wav")) + failing = TranscriptionWorker(repo, FakeEngine(fail_for={"retry.wav"}), ProcessingConfig()) + failing.run() + assert repo.get_audio(record.id).status == "error" + + assert repo.reset_errors() == 1 + retrying = TranscriptionWorker(repo, FakeEngine(), ProcessingConfig()) + stats = retrying.run() + + assert stats.processed == 1 + assert repo.get_audio(record.id).status == "done" + + +def test_worker_marks_missing_file(repo, add_audio): + record = add_audio("/tmp/gibt-es-nicht-mehr.mp3") + Path(record.path).unlink(missing_ok=True) + stats = TranscriptionWorker(repo, FakeEngine(), ProcessingConfig()).run() + + assert stats.failed == 1 + assert repo.get_audio(record.id).status == "error" + + +def test_worker_events_are_emitted(repo, add_audio, make_wav): + add_audio(make_wav("event.wav")) + events = [] + + TranscriptionWorker(repo, FakeEngine(), ProcessingConfig()).run(event_callback=events.append) + + kinds = [event.kind for event in events] + assert kinds[0] == "start" + assert "done" in kinds + assert kinds[-1] == "finished" + + +# ---------------------------------------------------------------------- +# GPU-/Ausfuehrungsdiagnose (reine Funktionen, keine echte GPU noetig) +# ---------------------------------------------------------------------- +def _pascal_gpu() -> GpuInfo: + return GpuInfo( + name="NVIDIA GeForce GTX 1070 Ti", + driver_version="580.196.07", + cuda_driver_version="13.0", + memory_total_mb=8192, + compute_capability="6.1", + ) + + +def _runtime(**overrides) -> RuntimeInfo: + runtime = RuntimeInfo( + ctranslate2_version="4.9.0", + cuda_device_count=1, + supported_compute_types={ + "cpu": ["float32", "int16", "int8", "int8_float32"], + "cuda": ["float32", "int8", "int8_float32"], + }, + cuda_libraries=["/usr/lib/libcudart.so.12"], + cudnn_libraries=["/usr/lib/libcudnn.so.9"], + python_version="3.13.15", + platform="Linux", + sqlite_version="3.50.0", + fts5_available=True, + trigram_available=True, + ) + for key, value in overrides.items(): + setattr(runtime, key, value) + return runtime + + +def test_pascal_uses_int8_float32(): + resolved = resolve_execution("cuda", "int8_float32", _pascal_gpu(), _runtime()) + assert resolved.device == "cuda" + assert resolved.compute_type == "int8_float32" + assert not resolved.used_fallback + + +def test_diagnostics_rejects_float16_on_incompatible_gpu(): + resolved = resolve_execution("cuda", "float16", _pascal_gpu(), _runtime()) + assert resolved.compute_type != "float16" + assert resolved.compute_type == "int8_float32" + assert any("Pascal" in note for note in resolved.notes) + + +def test_diagnostics_falls_back_when_int8_float32_unsupported(): + runtime = _runtime( + supported_compute_types={"cpu": ["float32", "int8"], "cuda": ["float32"]} + ) + resolved = resolve_execution("cuda", "int8_float32", _pascal_gpu(), runtime) + assert resolved.compute_type == "float32" + assert resolved.device == "cuda" + + +def test_cpu_fallback_when_cuda_unavailable(): + runtime = _runtime( + cuda_device_count=0, + supported_compute_types={"cpu": ["float32", "int8"], "cuda": []}, + ) + resolved = resolve_execution("cuda", "int8_float32", None, runtime) + assert resolved.device == "cpu" + assert resolved.used_fallback + assert resolved.compute_type in {"int8", "int8_float32", "float32"} + assert any("CPU" in note for note in resolved.notes) + + +def test_cpu_mode_never_returns_gpu_compute_type(): + resolved = resolve_execution("cpu", "float16", _pascal_gpu(), _runtime()) + assert resolved.device == "cpu" + assert resolved.compute_type not in {"float16", "bfloat16"}