Initial commit

This commit is contained in:
2026-11-16 16:29:32 +01:00
commit 97e5c9ef4d
37 changed files with 5932 additions and 0 deletions
+5
View File
@@ -0,0 +1,5 @@
"""SjAudioSearch - lokale Suchmaschine fuer transkribierte Audiosammlungen."""
__version__ = "1.0.0"
__all__ = ["__version__"]
+6
View File
@@ -0,0 +1,6 @@
"""Ermoeglicht ``python -m sjaudiosearch``."""
from .cli import app
if __name__ == "__main__":
app()
+715
View File
@@ -0,0 +1,715 @@
"""Kommandozeilenschnittstelle von SjAudioSearch."""
from __future__ import annotations
import json
import sqlite3
import sys
from dataclasses import dataclass, replace
from pathlib import Path
from typing import Optional
import typer
from rich.console import Console
from rich.markup import escape
from rich.panel import Panel
from rich.progress import (
BarColumn,
MofNCompleteColumn,
Progress,
SpinnerColumn,
TextColumn,
TimeElapsedColumn,
)
from rich.table import Table
from . import __version__
from .config import Config, load_config
from .core.audio import PlaybackError, play as play_audio
from .core.metadata import AudioMetadataError, read_metadata
from .core.models import FileStatus
from .core.scanner import scan as run_scan
from .database.connection import check_fts5
from .database.repository import Repository
from .search.engine import SearchEngine, SearchError
from .system.diagnostics import collect_report, render_report
from .system.gpu import collect_runtime_info, detect_gpu, resolve_execution
from .transcription.engine import FasterWhisperEngine, TranscriptionError
from .transcription.worker import RunStats, TranscriptionWorker, WorkerEvent
from .utils.formatting import (
format_datetime,
format_duration,
format_rate,
format_score,
format_size,
format_timestamp_verbose,
shorten_path,
)
from .utils.logging import get_logger, setup_logging
app = typer.Typer(
add_completion=False,
no_args_is_help=True,
help="Lokale Suchmaschine fuer transkribierte Hoerspiele und Hoerbuecher.",
)
console = Console()
logger = get_logger("cli")
@dataclass
class AppContext:
"""Globaler Zustand der CLI."""
config: Config
console: Console
_repository: Repository | None = None
def repository(self) -> Repository:
if self._repository is None:
self._repository = Repository.open(self.config.database_path)
return self._repository
def close(self) -> None:
if self._repository is not None:
self._repository.close()
self._repository = None
def _fail(message: str, code: int = 1) -> None:
console.print(f"[red]Fehler:[/red] {escape(message)}")
raise typer.Exit(code)
def _get_context(ctx: typer.Context) -> AppContext:
if not isinstance(ctx.obj, AppContext):
_fail("Interner Fehler: CLI-Kontext fehlt.")
return ctx.obj
@app.callback()
def main(
ctx: typer.Context,
config: Optional[Path] = typer.Option(
None, "--config", "-c", help="Pfad zur config.toml", show_default=False
),
version: bool = typer.Option(False, "--version", help="Version anzeigen", is_eager=True),
) -> None:
"""SjAudioSearch initialisieren."""
if version:
console.print(f"SjAudioSearch {__version__}")
raise typer.Exit()
cfg = load_config(config)
setup_logging(cfg.log_path, cfg.logging.level)
app_context = AppContext(config=cfg, console=console)
ctx.obj = app_context
ctx.call_on_close(app_context.close)
for warning in cfg.load_warnings:
console.print(f"[yellow]Hinweis:[/yellow] {escape(warning)}")
for key in cfg.unknown_keys:
console.print(f"[yellow]Hinweis:[/yellow] unbekannter Konfigurationsschluessel '{escape(key)}'")
# ----------------------------------------------------------------------
# diagnose
# ----------------------------------------------------------------------
@app.command()
def diagnose(
ctx: typer.Context,
device: Optional[str] = typer.Option(None, help="Zu pruefendes Geraet: cuda oder cpu"),
compute_type: Optional[str] = typer.Option(None, "--compute-type", help="Zu pruefender Compute Type"),
) -> None:
"""Zeigt einen vollstaendigen Systembericht (GPU, CUDA, Compute Types)."""
app_context = _get_context(ctx)
report = collect_report(
app_context.config,
requested_device=device,
requested_compute_type=compute_type,
)
render_report(report, app_context.console)
# ----------------------------------------------------------------------
# scan
# ----------------------------------------------------------------------
@app.command()
def scan(
ctx: typer.Context,
path: Path = typer.Argument(..., help="Ordner mit Audiodateien"),
) -> None:
"""Durchsucht einen Ordner rekursiv und nimmt neue Dateien in den Index auf."""
app_context = _get_context(ctx)
repository = app_context.repository()
with Progress(
SpinnerColumn(),
TextColumn("[bold]{task.description}"),
BarColumn(),
MofNCompleteColumn(),
TimeElapsedColumn(),
console=app_context.console,
transient=True,
) as progress:
task = progress.add_task("Scanne Dateien", total=None)
def on_progress(current: int, current_path: Path) -> None:
progress.update(
task,
total=max(current, progress.tasks[0].total or 0),
description=f"Scanne {escape(current_path.name)}",
)
try:
result = run_scan(path, repository, app_context.config.processing, progress=on_progress)
except (FileNotFoundError, NotADirectoryError) as exc:
_fail(str(exc))
table = Table(title="Scan-Ergebnis", show_header=False, box=None)
table.add_column(style="bold cyan")
table.add_column()
table.add_row("Ordner", result.root)
table.add_row("Gefundene Audiodateien", str(result.files_found))
table.add_row("Neu aufgenommen", str(result.added))
table.add_row("Geaendert", str(result.changed))
table.add_row("Unveraendert", str(result.unchanged))
table.add_row("Duplikate", str(result.duplicates))
table.add_row("Fehler", str(len(result.errors)))
console.print(table)
if result.errors:
error_table = Table(title="Dateien mit Problemen")
error_table.add_column("Datei", overflow="fold")
error_table.add_column("Grund", overflow="fold")
for item in result.errors[:15]:
error_table.add_row(shorten_path(item.path, 70), escape(item.message))
console.print(error_table)
console.print(
"[yellow]Hinweis:[/yellow] Diese Dateien wurden mit Status 'error' gespeichert. "
"Details stehen im Log."
)
# ----------------------------------------------------------------------
# transcribe
# ----------------------------------------------------------------------
def _resolve_transcription(ctx_app: AppContext, model: str | None, device: str | None,
compute_type: str | None, language: str | None):
config = ctx_app.config
settings = config.transcription
if model:
settings = replace(settings, model=model)
if language is not None:
settings = replace(settings, language=language)
requested_device = device or settings.device
requested_compute = compute_type or settings.compute_type
gpu = detect_gpu()
runtime = collect_runtime_info(check_fts=False)
resolved = resolve_execution(requested_device, requested_compute, gpu, runtime)
return settings, resolved, gpu, runtime
@app.command()
def transcribe(
ctx: typer.Context,
model: Optional[str] = typer.Option(None, help="Modell, z. B. large-v3, medium, small"),
device: Optional[str] = typer.Option(None, help="cuda oder cpu"),
compute_type: Optional[str] = typer.Option(None, "--compute-type", help="z. B. int8_float32, float32"),
language: Optional[str] = typer.Option(None, help="Sprache, z. B. de oder auto"),
limit: Optional[int] = typer.Option(None, help="Maximale Anzahl Dateien in diesem Lauf"),
force: bool = typer.Option(False, "--force", help="Bereits transkribierte Dateien erneut verarbeiten"),
) -> None:
"""Arbeitet die Transkriptionswarteschlange ab."""
app_context = _get_context(ctx)
repository = app_context.repository()
config = app_context.config
settings, resolved, gpu, runtime = _resolve_transcription(
app_context, model, device, compute_type, language
)
mode_table = Table(show_header=False, box=None, padding=(0, 1))
mode_table.add_column(style="bold cyan", no_wrap=True)
mode_table.add_column()
mode_table.add_row("Modell", settings.model)
mode_table.add_row("Geraet", f"{resolved.device}" + (" (GPU)" if resolved.device == "cuda" else " (CPU)"))
mode_table.add_row("Compute Type", resolved.compute_type)
mode_table.add_row("Sprache", settings.language_for_whisper or "automatisch")
if gpu is not None:
mode_table.add_row(
"GPU", f"{gpu.name} (CC {gpu.compute_capability or '?'}, {gpu.memory_total_mb or '?'} MB)"
)
console.print(Panel(mode_table, title="Transkription", border_style="cyan"))
for note in resolved.notes:
console.print(f"[yellow]Hinweis:[/yellow] {escape(note)}")
if not runtime.cuda_runtime_usable and resolved.device == "cpu":
console.print(
"[yellow]Hinweis:[/yellow] CPU-Modus aktiv. Die Verarbeitung wird deutlich langsamer sein."
)
queue = repository.transcription_queue(force=force)
if limit is not None:
queue = queue[: max(0, limit)]
if not queue:
console.print("Keine Dateien in der Warteschlange. Nutze zuerst 'sjaudiosearch scan <ordner>'.")
return
engine = FasterWhisperEngine(
settings,
device=resolved.device,
compute_type=resolved.compute_type,
model_dir=config.model_path,
)
with console.status(f"Lade Modell {settings.model} ...", spinner="dots"):
try:
engine.load()
except TranscriptionError as exc:
if resolved.device == "cuda":
console.print(f"[yellow]GPU-Modus fehlgeschlagen:[/yellow] {escape(str(exc))}")
console.print("[yellow]Wechsle automatisch auf CPU (int8).[/yellow]")
engine = FasterWhisperEngine(
settings,
device="cpu",
compute_type="int8",
model_dir=config.model_path,
)
resolved = replace(resolved, device="cpu", compute_type="int8")
try:
engine.load()
except TranscriptionError as cpu_exc:
_fail(f"Auch der CPU-Modus ist fehlgeschlagen: {cpu_exc}")
else:
_fail(str(exc))
worker = TranscriptionWorker(
repository,
engine,
config.processing,
device=resolved.device,
compute_type=resolved.compute_type,
)
columns = [
SpinnerColumn(),
TextColumn("[bold]{task.description}"),
BarColumn(),
MofNCompleteColumn(),
TimeElapsedColumn(),
TextColumn("{task.fields[rate]}"),
]
errors: list[tuple[str, str]] = []
with Progress(*columns, console=console) as progress:
task = progress.add_task("Starte Transkription", total=len(queue), rate="")
def on_event(event: WorkerEvent) -> None:
if event.kind == "start" and event.record:
progress.update(
task, description=escape(event.record.filename), rate=""
)
elif event.kind == "done":
stats = worker.stats
progress.update(
task,
advance=1,
rate=format_rate(stats.audio_seconds, stats.elapsed) if stats.elapsed else "",
)
elif event.kind == "error" and event.record:
errors.append((event.record.path, event.message or "unbekannter Fehler"))
progress.update(task, advance=1, rate="[red]Fehler[/red]")
try:
stats = worker.run(limit=limit, force=force, event_callback=on_event)
except KeyboardInterrupt:
console.print()
console.print("[yellow]Abgebrochen.[/yellow] Der aktuelle Stand ist gespeichert, "
"die Unterbrechung wird beim naechsten Lauf fortgesetzt.")
raise typer.Exit(130)
except TranscriptionError as exc:
_fail(str(exc))
_render_run_summary(stats, settings.model, resolved.device, resolved.compute_type, errors)
def _render_run_summary(
stats: RunStats, model: str, device: str, compute_type: str, errors: list[tuple[str, str]]
) -> None:
table = Table(title="Transkription abgeschlossen", show_header=False, box=None)
table.add_column(style="bold cyan", no_wrap=True)
table.add_column()
table.add_row("Modell", model)
table.add_row("Ausfuehrung", f"{device} / {compute_type}")
table.add_row("Warteschlange", str(stats.queued))
table.add_row("Erfolgreich", str(stats.processed))
table.add_row("Fehlgeschlagen", str(stats.failed))
table.add_row("Verarbeitete Audiodauer", format_duration(stats.audio_seconds))
table.add_row("Laufzeit", format_duration(stats.elapsed))
table.add_row("Durchschnitt", f"{format_rate(stats.audio_seconds, stats.elapsed)}")
console.print(table)
if errors:
error_table = Table(title="Fehlerhafte Dateien")
error_table.add_column("Datei", overflow="fold")
error_table.add_column("Grund", overflow="fold")
for path, message in errors[:15]:
error_table.add_row(shorten_path(path, 70), escape(message))
console.print(error_table)
console.print("Mit [bold]sjaudiosearch retry[/bold] koennen fehlgeschlagene Dateien erneut versucht werden.")
# ----------------------------------------------------------------------
# search
# ----------------------------------------------------------------------
@app.command()
def search(
ctx: typer.Context,
query: str = typer.Argument(..., help="Gesuchter Text (auch ungenau)"),
limit: Optional[int] = typer.Option(None, help="Maximale Anzahl Treffer"),
min_score: Optional[float] = typer.Option(None, "--min-score", help="Mindestaehnlichkeit 0-100"),
context_words: Optional[int] = typer.Option(None, "--context-words", help="Woerter vor/nach dem Treffer"),
language: Optional[str] = typer.Option(None, help="Nur Transkripte dieser Sprache"),
json_output: bool = typer.Option(False, "--json", help="Ergebnisse als JSON ausgeben"),
verbose: bool = typer.Option(False, "--verbose", help="Zusaetzliche Details anzeigen"),
) -> None:
"""Sucht in allen Transkripten - auch bei ungenauen Erinnerungen."""
app_context = _get_context(ctx)
repository = app_context.repository()
config = app_context.config
engine = SearchEngine(repository, config.search)
effective_limit = limit if limit is not None else config.search.default_limit
effective_min = min_score if min_score is not None else config.search.default_min_score
effective_context = context_words if context_words is not None else config.search.context_words
progress_state: dict[str, int] = {}
with console.status("[cyan]Suche ...[/cyan]", spinner="dots") as status:
def on_progress(current: int, total: int) -> None:
progress_state["current"] = current
progress_state["total"] = total
status.update(f"[cyan]Suche ... {current}/{total} Dateien[/cyan]")
try:
hits = engine.search(
query,
limit=effective_limit,
min_score=effective_min,
context_words=effective_context,
language=language,
progress=on_progress,
)
except SearchError as exc:
_fail(str(exc))
except sqlite3.Error as exc:
_fail(f"Datenbankfehler bei der Suche: {exc}")
if json_output:
payload = {
"query": query,
"limit": effective_limit,
"min_score": effective_min,
"context_words": effective_context,
"count": len(hits),
"results": [hit.to_dict(effective_context) for hit in hits],
}
print(json.dumps(payload, ensure_ascii=False, indent=2))
return
console.print(Panel.fit("[bold]SjAudioSearch[/bold]", border_style="cyan"))
console.print(f'Query: [bold]"{escape(query)}"[/bold]')
console.print(
f"[bold]{len(hits)}[/bold] Treffer (Mindestaehnlichkeit {effective_min:.0f}%, "
f"Kontext {effective_context} Woerter)"
)
if not hits:
console.print()
console.print(
"[yellow]Keine Treffer.[/yellow] Versuche eine hoehere Kontextlaenge, "
"eine niedrigere Mindestaehnlichkeit oder ein anderes Modell."
)
for index, hit in enumerate(hits, start=1):
header = f"[bold][{index}] {escape(hit.filename)}[/bold]"
details = [
f"Similarity: [bold green]{format_score(hit.score)}[/bold green]",
f"Timestamp: [bold]{format_timestamp_verbose(hit.start_time)}[/bold] "
f"(bis {format_timestamp_verbose(hit.end_time)}, "
f"{'Wort' if hit.context.precision == 'word' else 'Segment'}-Genauigkeit)",
]
if verbose:
details.append(
f"Audio-ID: {hit.audio_id} | Sprache: {hit.language or '?'} | Modell: {escape(hit.model or '?')} | "
f"Quelle: {hit.source}"
)
body = "\n".join(details)
console.print()
console.print(header)
console.print(body)
console.print()
console.print(Panel(hit.context.as_block(), title="Context", border_style="dim"))
console.print(f"Path: {escape(hit.path)}")
console.print("[dim]" + "-" * 40 + "[/dim]")
# ----------------------------------------------------------------------
# status
# ----------------------------------------------------------------------
@app.command()
def status(ctx: typer.Context) -> None:
"""Zeigt Kennzahlen der Sammlung und den Datenbankzustand."""
app_context = _get_context(ctx)
repository = app_context.repository()
stats = repository.stats(repository.databases_size())
config = app_context.config
table = Table(title="SjAudioSearch Status", show_header=False, box=None)
table.add_column(style="bold cyan", no_wrap=True)
table.add_column()
table.add_row("Audiodateien gesamt", str(stats.total_files))
table.add_row("Transkribiert", str(stats.done))
table.add_row("Ausstehend", str(stats.pending))
table.add_row("In Bearbeitung", str(stats.processing))
table.add_row("Fehlgeschlagen", str(stats.error))
table.add_row("Duplikate", str(stats.duplicates))
table.add_row("Gesamtspieldauer", format_duration(stats.total_duration))
table.add_row("Transkripte", str(stats.transcripts))
table.add_row("Segmente", str(stats.segment_count))
table.add_row("Woerter", str(stats.word_count))
table.add_row("Datenbank", f"{config.database_path} ({format_size(stats.database_size)})")
model_text = ", ".join(stats.models) if stats.models else f"{config.transcription.model} (noch nicht verwendet)"
table.add_row("Transkriptionsmodell", model_text)
console.print(table)
if stats.error:
console.print("[yellow]Hinweis:[/yellow] 'sjaudiosearch retry' setzt fehlgeschlagene Dateien zurueck.")
if stats.pending or stats.processing:
console.print("[yellow]Hinweis:[/yellow] 'sjaudiosearch transcribe' arbeitet die Warteschlange ab.")
# ----------------------------------------------------------------------
# list
# ----------------------------------------------------------------------
@app.command("list")
def list_files(
ctx: typer.Context,
limit: Optional[int] = typer.Option(None, help="Maximale Anzahl Eintraege"),
status_filter: Optional[str] = typer.Option(
None, "--status", help="pending, processing, done oder error"
),
search: Optional[str] = typer.Option(None, "--search", help="Dateiname/Pfad filtern"),
) -> None:
"""Listet bekannte Audiodateien auf."""
app_context = _get_context(ctx)
repository = app_context.repository()
if status_filter and status_filter not in FileStatus.values():
_fail(f"Unbekannter Status '{status_filter}'. Erlaubt: {', '.join(FileStatus.values())}")
records = repository.list_audio(status=status_filter, limit=limit, search=search)
if not records:
console.print("Keine Dateien gefunden. Nutze 'sjaudiosearch scan <ordner>'.")
return
table = Table(title=f"Audiodateien ({len(records)})", show_lines=False)
table.add_column("ID", justify="right", style="cyan")
table.add_column("Status")
table.add_column("Dauer", justify="right")
table.add_column("Groesse", justify="right")
table.add_column("Datei", overflow="fold")
table.add_column("Pfad", overflow="fold")
colors = {
FileStatus.DONE.value: "green",
FileStatus.PENDING.value: "yellow",
FileStatus.PROCESSING.value: "cyan",
FileStatus.ERROR.value: "red",
}
for record in records:
status_text = record.status
if record.duplicate_of:
status_text = f"duplikat von #{record.duplicate_of}"
color = colors.get(record.status, "white")
table.add_row(
str(record.id),
f"[{color}]{status_text}[/{color}]",
format_duration(record.duration),
format_size(record.file_size),
escape(record.filename),
escape(shorten_path(record.path, 70)),
)
console.print(table)
# ----------------------------------------------------------------------
# info
# ----------------------------------------------------------------------
@app.command()
def info(
ctx: typer.Context,
audio_id: int = typer.Argument(..., help="ID der Audiodatei"),
) -> None:
"""Zeigt Details zu einer Audiodatei und ihrem Transkript."""
app_context = _get_context(ctx)
repository = app_context.repository()
record = repository.get_audio(audio_id)
if record is None:
_fail(f"Keine Audiodatei mit ID {audio_id} gefunden.")
table = Table(show_header=False, box=None)
table.add_column(style="bold cyan", no_wrap=True)
table.add_column(overflow="fold")
table.add_row("ID", str(record.id))
table.add_row("Dateiname", escape(record.filename))
table.add_row("Pfad", escape(record.path))
table.add_row("Status", record.status)
table.add_row("Dauer", format_duration(record.duration))
table.add_row("Groesse", format_size(record.file_size))
table.add_row("Geaendert", format_datetime(record.updated_at))
table.add_row("Index-Hash", record.content_hash or "-")
if record.duplicate_of:
table.add_row("Duplikat von", f"#{record.duplicate_of}")
if record.error_message:
table.add_row("Fehler", escape(record.error_message))
try:
meta = read_metadata(record.path)
if meta.tags:
tags = ", ".join(f"{key}={value}" for key, value in list(meta.tags.items())[:6])
table.add_row("Tags", escape(tags))
except AudioMetadataError:
pass
console.print(Panel(table, title=f"Audiodatei #{record.id}", border_style="cyan"))
transcript = repository.get_transcript(record.id)
if transcript is None:
console.print("[yellow]Kein Transkript vorhanden.[/yellow]")
return
segments = repository.get_segments(record.id)
console.print(
f"Transkript: {len(segments)} Segmente, Modell [bold]{escape(transcript['model'])}[/bold], "
f"Sprache {transcript['language'] or '?'}, erstellt {format_datetime(transcript['created_at'])}"
)
preview = Table(title="Segmente (Auszug)")
preview.add_column("Start", justify="right")
preview.add_column("Ende", justify="right")
preview.add_column("Text", overflow="fold")
for segment in segments[:20]:
preview.add_row(
format_timestamp_verbose(segment["start_time"]),
format_timestamp_verbose(segment["end_time"]),
escape(segment["text"]),
)
console.print(preview)
if len(segments) > 20:
console.print(f"[dim]... {len(segments) - 20} weitere Segmente[/dim]")
# ----------------------------------------------------------------------
# play
# ----------------------------------------------------------------------
@app.command()
def play(
ctx: typer.Context,
audio_id: int = typer.Argument(..., help="ID der Audiodatei"),
time: float = typer.Option(0.0, "--time", help="Startzeit in Sekunden"),
) -> None:
"""Spielt eine Audiodatei ab einer Zeitposition ab."""
app_context = _get_context(ctx)
repository = app_context.repository()
record = repository.get_audio(audio_id)
if record is None:
_fail(f"Keine Audiodatei mit ID {audio_id} gefunden.")
console.print(
f"Spiele [bold]{escape(record.filename)}[/bold] ab {format_timestamp_verbose(time)} ..."
)
try:
play_audio(record.path, time)
except PlaybackError as exc:
_fail(str(exc), code=127)
# ----------------------------------------------------------------------
# retry / reindex / remove
# ----------------------------------------------------------------------
@app.command()
def retry(ctx: typer.Context) -> None:
"""Setzt fehlgeschlagene Transkriptionen zurueck auf 'pending'."""
app_context = _get_context(ctx)
repository = app_context.repository()
count = repository.reset_errors()
console.print(f"{count} Datei(en) zurueckgesetzt. Starte 'sjaudiosearch transcribe'.")
@app.command()
def reindex(ctx: typer.Context) -> None:
"""Baut den Suchindex aus den vorhandenen Transkripten neu auf."""
app_context = _get_context(ctx)
repository = app_context.repository()
before = repository.stats().transcripts
with console.status("Baue Suchindex neu ...", spinner="dots"):
indexed = repository.rebuild_search_index()
after = repository.stats().transcripts
if before != after:
_fail(f"Transkriptanzahl hat sich geaendert ({before} -> {after}) - bitte pruefen.")
console.print(
f"Suchindex neu aufgebaut: {indexed} Segmente indexiert. "
f"Transkripte unveraendert ({after})."
)
@app.command()
def remove(
ctx: typer.Context,
audio_id: int = typer.Argument(..., help="ID der Audiodatei"),
) -> None:
"""Entfernt eine Datei aus dem Index (die Audiodatei selbst bleibt erhalten)."""
app_context = _get_context(ctx)
repository = app_context.repository()
record = repository.get_audio(audio_id)
if record is None:
_fail(f"Keine Audiodatei mit ID {audio_id} gefunden.")
if not repository.delete_audio(audio_id):
_fail(f"Datei #{audio_id} konnte nicht entfernt werden.")
console.print(
f"[green]Entfernt:[/green] #{audio_id} {escape(record.filename)} "
"(Transkript und Suchindex geloescht)"
)
console.print(f"[dim]Die Originaldatei bleibt erhalten: {escape(record.path)}[/dim]")
# ----------------------------------------------------------------------
# search-index Diagnose
# ----------------------------------------------------------------------
@app.command("check")
def check(ctx: typer.Context) -> None:
"""Prueft Datenbank, FTS5 und Trigramm-Unterstuetzung."""
app_context = _get_context(ctx)
config = app_context.config
conn = app_context.repository().conn
fts5, trigram = check_fts5(conn)
console.print(f"Datenbank: {config.database_path}")
console.print(f"FTS5: {'OK' if fts5 else 'nicht verfuegbar'}")
console.print(f"Trigramm-Tokenizer: {'OK' if trigram else 'nicht verfuegbar'}")
console.print(f"Logdatei: {config.log_path}")
console.print(f"Konfiguration: {config.source_path or 'eingebaute Defaults'}")
def run() -> None:
"""Einstiegspunkt fuer ``python -m sjaudiosearch``."""
try:
app()
except KeyboardInterrupt:
console.print("\n[yellow]Abgebrochen.[/yellow]")
sys.exit(130)
if __name__ == "__main__": # pragma: no cover
run()
+225
View File
@@ -0,0 +1,225 @@
"""Konfigurationsverwaltung fuer SjAudioSearch.
Die Konfiguration wird aus einer TOML-Datei gelesen. Fehlende Werte werden
durch Defaults ersetzt, unbekannte Schluessel werden gesammelt und gemeldet,
statt stillschweigend ignoriert zu werden.
"""
from __future__ import annotations
import os
import tomllib
from dataclasses import dataclass, field, fields
from pathlib import Path
from typing import Any
PROJECT_ROOT = Path(__file__).resolve().parent.parent
DEFAULT_CONFIG_FILENAME = "config.toml"
ENV_CONFIG_PATH = "SJAUDIOSEARCH_CONFIG"
def _xdg_dir(env_var: str, fallback: str) -> Path:
"""Ermittelt ein XDG-Verzeichnis (Konfiguration bzw. Daten)."""
base = os.environ.get(env_var)
root = Path(base).expanduser() if base else Path.home() / fallback
return root / "sjaudiosearch"
# ~/.config/sjaudiosearch/config.toml (bzw. $XDG_CONFIG_HOME/...)
USER_CONFIG_DIR = _xdg_dir("XDG_CONFIG_HOME", ".config")
# Ablage fuer Datenbank und Logs, wenn das Programm systemweit installiert ist.
USER_DATA_DIR = _xdg_dir("XDG_DATA_HOME", ".local/share")
SYSTEM_CONFIG_PATH = Path("/etc/sjaudiosearch/config.toml")
@dataclass
class TranscriptionConfig:
model: str = "large-v3"
device: str = "cuda"
compute_type: str = "int8_float32"
language: str | None = "de"
beam_size: int = 5
word_timestamps: bool = True
vad_filter: bool = True
model_dir: str = "./data/models"
@property
def language_for_whisper(self) -> str | None:
"""Gibt ``None`` zurueck, wenn die Sprache automatisch erkannt werden soll."""
if self.language is None:
return None
value = self.language.strip().lower()
if value in {"", "auto", "none"}:
return None
return value
@dataclass
class SearchConfig:
default_limit: int = 50
default_min_score: float = 70.0
context_words: int = 50
window_before: int = 2
window_after: int = 4
candidate_limit: int = 4000
min_token_overlap: float = 0.34
exhaustive_fallback: bool = True
max_hits_per_file: int = 5
@dataclass
class ProcessingConfig:
workers: int = 1
resume: bool = True
compute_hash: str = "auto"
hash_max_size_mb: int = 4096
@dataclass
class DatabaseConfig:
path: str = "./data/sjaudiosearch.db"
@dataclass
class LoggingConfig:
level: str = "INFO"
path: str = "./logs/sjaudiosearch.log"
@dataclass
class Config:
transcription: TranscriptionConfig = field(default_factory=TranscriptionConfig)
search: SearchConfig = field(default_factory=SearchConfig)
processing: ProcessingConfig = field(default_factory=ProcessingConfig)
database: DatabaseConfig = field(default_factory=DatabaseConfig)
logging: LoggingConfig = field(default_factory=LoggingConfig)
# Metadaten zur Herkunft der Konfiguration
source_path: Path | None = None
base_dir: Path = field(default_factory=lambda: PROJECT_ROOT)
unknown_keys: list[str] = field(default_factory=list)
load_warnings: list[str] = field(default_factory=list)
def resolve_path(self, value: str) -> Path:
"""Loest einen (moeglicherweise relativen) Pfad gegen das Config-Verzeichnis auf."""
path = Path(value).expanduser()
if path.is_absolute():
return path
return (self.base_dir / path).resolve()
@property
def database_path(self) -> Path:
return self.resolve_path(self.database.path)
@property
def log_path(self) -> Path:
return self.resolve_path(self.logging.path)
@property
def model_path(self) -> Path:
return self.resolve_path(self.transcription.model_dir)
def _coerce(current: Any, value: Any) -> Any:
"""Konvertiert TOML-Werte tolerant auf den Typ des Defaults."""
if current is None:
return value
if isinstance(current, bool):
if isinstance(value, bool):
return value
if isinstance(value, str):
return value.strip().lower() in {"1", "true", "yes", "on"}
return bool(value)
if isinstance(current, int) and not isinstance(current, bool):
try:
return int(value)
except (TypeError, ValueError):
return current
if isinstance(current, float):
try:
return float(value)
except (TypeError, ValueError):
return current
if isinstance(current, str):
return str(value)
return value
def _apply_section(target: Any, table: dict[str, Any], section: str, config: Config) -> None:
known = {f.name: f for f in fields(target)}
for key, value in table.items():
if key not in known:
config.unknown_keys.append(f"{section}.{key}")
continue
current = getattr(target, key)
setattr(target, key, _coerce(current, value))
def _is_writable_dir(path: Path) -> bool:
try:
return path.is_dir() and os.access(path, os.W_OK)
except OSError:
return False
def find_config_path(explicit: str | Path | None = None) -> Path | None:
"""Ermittelt die zu verwendende Konfigurationsdatei."""
candidates: list[Path] = []
if explicit:
candidates.append(Path(explicit).expanduser())
env_value = os.environ.get(ENV_CONFIG_PATH)
if env_value:
candidates.append(Path(env_value).expanduser())
candidates.append(Path.cwd() / DEFAULT_CONFIG_FILENAME)
candidates.append(PROJECT_ROOT / DEFAULT_CONFIG_FILENAME)
candidates.append(USER_CONFIG_DIR / DEFAULT_CONFIG_FILENAME)
candidates.append(SYSTEM_CONFIG_PATH)
for candidate in candidates:
if candidate.is_file():
return candidate.resolve()
return None
def load_config(explicit: str | Path | None = None) -> Config:
"""Laedt die Konfiguration; ohne Datei werden die Defaults verwendet."""
path = find_config_path(explicit)
config = Config()
if path is None:
# Im Quellbaum (beschreibbar) bleiben Daten im Projekt, nach einer
# systemweiten Installation liegen sie im Benutzerverzeichnis.
config.base_dir = PROJECT_ROOT if _is_writable_dir(PROJECT_ROOT) else USER_DATA_DIR
if explicit:
config.load_warnings.append(
f"Konfigurationsdatei nicht gefunden: {explicit} - es gelten die Defaults."
)
return config
config.source_path = path
config.base_dir = path.parent
try:
with path.open("rb") as handle:
raw = tomllib.load(handle)
except (OSError, tomllib.TOMLDecodeError) as exc:
config.load_warnings.append(f"Konfiguration konnte nicht gelesen werden ({path}): {exc}")
return config
section_map = {
"transcription": config.transcription,
"search": config.search,
"processing": config.processing,
"database": config.database,
"logging": config.logging,
}
for section_name, section_obj in section_map.items():
table = raw.get(section_name)
if isinstance(table, dict):
_apply_section(section_obj, table, section_name, config)
for key in raw:
if key not in section_map:
config.unknown_keys.append(key)
return config
+74
View File
@@ -0,0 +1,74 @@
"""Audio-Wiedergabe und Dekodierpruefungen."""
from __future__ import annotations
import shutil
import subprocess
from pathlib import Path
from .metadata import AudioMetadataError, read_metadata
class PlaybackError(RuntimeError):
"""Wiedergabe nicht moeglich."""
def find_player() -> str | None:
"""Sucht einen installierten Player (bevorzugt ffplay)."""
for candidate in ("ffplay", "mpv", "vlc"):
executable = shutil.which(candidate)
if executable:
return executable
return None
def build_play_command(path: str | Path, start_time: float = 0.0) -> list[str]:
"""Baut den Kommandozeilenaufruf zum Abspielen ab einer Zeitposition."""
player = find_player()
if player is None:
raise PlaybackError(
"Kein Player gefunden. Installiere ffmpeg (ffplay), mpv oder vlc."
)
name = Path(player).name
start = max(0.0, float(start_time))
if name.startswith("ffplay"):
# -nodisp: reine Audiowiedergabe ohne zusaetzliches Fenster.
command = [player, "-nodisp", "-autoexit", "-loglevel", "error"]
if start > 0:
command += ["-ss", f"{start:.3f}"]
command.append(str(path))
return command
if name.startswith("mpv"):
command = [player]
if start > 0:
command += [f"--start={start:.3f}"]
command.append(str(path))
return command
# vlc
command = [player]
if start > 0:
command += [f"--start-time={start:.3f}"]
command.append(str(path))
return command
def play(path: str | Path, start_time: float = 0.0) -> None:
"""Startet die Wiedergabe und wartet, bis der Player beendet wird."""
file_path = Path(path)
if not file_path.is_file():
raise PlaybackError(f"Audiodatei nicht gefunden: {file_path}")
command = build_play_command(file_path, start_time)
try:
subprocess.run(command, check=False)
except OSError as exc:
raise PlaybackError(f"Player konnte nicht gestartet werden: {exc}") from exc
def can_decode(path: str | Path) -> bool:
"""Prueft grob, ob die Datei als Audio dekodierbar ist."""
try:
read_metadata(path)
return True
except AudioMetadataError:
return False
+143
View File
@@ -0,0 +1,143 @@
"""Audio-Metadaten (Dauer, Tags, Format).
Primaer wird Mutagen verwendet; wenn Mutagen eine Datei nicht lesen kann, wird
PyAV als Fallback genutzt. Beide Wege lesen nur Header bzw. Container, nicht
die komplette Audiodatei.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from ..utils.logging import get_logger
logger = get_logger("core.metadata")
SUPPORTED_EXTENSIONS = frozenset(
{".mp3", ".wav", ".flac", ".m4a", ".ogg", ".opus", ".wma"}
)
# WMA ist nur zuverlaessig dekodierbar, wenn ffmpeg/PyAV den Codec kennt.
CONDITIONAL_EXTENSIONS = frozenset({".wma"})
class AudioMetadataError(RuntimeError):
"""Die Datei konnte nicht als Audio gelesen werden."""
@dataclass
class AudioInfo:
"""Ergebnis der Metadatenanalyse."""
path: str
duration: float | None = None
codec: str | None = None
sample_rate: int | None = None
channels: int | None = None
bitrate: int | None = None
tags: dict[str, str] = field(default_factory=dict)
source: str = "mutagen"
def is_supported_audio(path: str | Path) -> bool:
return Path(path).suffix.lower() in SUPPORTED_EXTENSIONS
def _clean_tags(raw: dict[str, Any]) -> dict[str, str]:
tags: dict[str, str] = {}
for key, value in raw.items():
try:
if isinstance(value, (list, tuple)):
text = ", ".join(str(item) for item in value if item is not None)
else:
text = str(value)
except Exception: # defekte Tag-Objekte duerfen den Scan nicht stoppen
continue
if text:
tags[str(key).lower()] = text
return tags
def _read_with_mutagen(path: Path) -> AudioInfo:
import mutagen
audio = mutagen.File(str(path))
if audio is None:
raise AudioMetadataError("Format wurde von Mutagen nicht erkannt")
info = getattr(audio, "info", None)
duration = getattr(info, "length", None)
if duration is None:
raise AudioMetadataError("Mutagen konnte keine Dauer ermitteln")
return AudioInfo(
path=str(path),
duration=float(duration),
codec=type(audio).__name__,
sample_rate=getattr(info, "sample_rate", None),
channels=getattr(info, "channels", None),
bitrate=getattr(info, "bitrate", None),
tags=_clean_tags(dict(audio.tags or {})),
source="mutagen",
)
def _read_with_pyav(path: Path) -> AudioInfo:
import av
try:
with av.open(str(path)) as container:
if not container.streams.audio:
raise AudioMetadataError("Keine Audiospur gefunden")
stream = container.streams.audio[0]
duration = None
if stream.duration is not None and stream.time_base is not None:
duration = float(stream.duration * stream.time_base)
elif container.duration is not None:
duration = float(container.duration / 1_000_000)
if duration is None or duration <= 0:
raise AudioMetadataError("Keine gueltige Dauer ermittelbar")
return AudioInfo(
path=str(path),
duration=duration,
codec=stream.codec_context.name,
sample_rate=stream.codec_context.sample_rate,
channels=stream.codec_context.channels,
bitrate=stream.bit_rate,
source="pyav",
)
except AudioMetadataError:
raise
except Exception as exc: # PyAV wirft verschiedene Fehlertypen
raise AudioMetadataError(f"PyAV konnte die Datei nicht lesen: {exc}") from exc
def read_metadata(path: str | Path) -> AudioInfo:
"""Liest Metadaten ueber Mutagen, bei Bedarf ueber PyAV."""
file_path = Path(path)
if not file_path.is_file():
raise AudioMetadataError(f"Datei nicht gefunden: {file_path}")
errors: list[str] = []
try:
return _read_with_mutagen(file_path)
except Exception as exc:
errors.append(f"mutagen: {exc}")
logger.debug("Mutagen fehlgeschlagen fuer %s: %s", file_path, exc)
try:
return _read_with_pyav(file_path)
except Exception as exc:
errors.append(f"pyav: {exc}")
logger.debug("PyAV fehlgeschlagen fuer %s: %s", file_path, exc)
raise AudioMetadataError("; ".join(errors))
def read_duration(path: str | Path) -> float | None:
"""Dauer einer Datei oder ``None``, wenn sie nicht ermittelbar ist."""
try:
return read_metadata(path).duration
except AudioMetadataError:
return None
+89
View File
@@ -0,0 +1,89 @@
"""Datenmodelle fuer Audiodateien und Scan-Ergebnisse."""
from __future__ import annotations
from dataclasses import dataclass, field
from enum import Enum
class FileStatus(str, Enum):
"""Transkriptionsstatus einer Audiodatei."""
PENDING = "pending"
PROCESSING = "processing"
DONE = "done"
ERROR = "error"
@classmethod
def values(cls) -> tuple[str, ...]:
return tuple(item.value for item in cls)
@dataclass
class AudioFileRecord:
"""Repraesentiert eine Zeile aus ``audio_files``."""
id: int
path: str
filename: str
file_size: int
modified_at: float
status: str
duration: float | None = None
content_hash: str | None = None
duplicate_of: int | None = None
error_message: str | None = None
created_at: str | None = None
updated_at: str | None = None
@dataclass
class ScanCandidate:
"""Beim Scan gefundene Datei (noch nicht in der Datenbank)."""
path: str
filename: str
file_size: int
modified_at: float
duration: float | None = None
content_hash: str | None = None
@dataclass
class ScanError:
"""Fehler, der beim Einlesen einer Datei aufgetreten ist."""
path: str
message: str
@dataclass
class ScanResult:
"""Zusammenfassung eines Scan-Laufs."""
root: str
files_found: int = 0
added: int = 0
changed: int = 0
unchanged: int = 0
duplicates: int = 0
errors: list[ScanError] = field(default_factory=list)
skipped_extensions: int = 0
@dataclass
class RepositoryStats:
"""Kennzahlen fuer ``sjaudiosearch status``."""
total_files: int = 0
done: int = 0
pending: int = 0
processing: int = 0
error: int = 0
duplicates: int = 0
total_duration: float = 0.0
database_size: int = 0
segment_count: int = 0
word_count: int = 0
transcripts: int = 0
models: list[str] = field(default_factory=list)
+254
View File
@@ -0,0 +1,254 @@
"""Rekursiver Audio-Scanner.
Der Scanner nutzt Dateigroesse und Aenderungszeit fuer eine schnelle
Aenderungspruefung. SHA256-Hashes werden nur berechnet, wenn sie fuer die
Duplikaterkennung tatsaechlich gebraucht werden (Modus ``auto``).
"""
from __future__ import annotations
import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import Callable
from ..config import ProcessingConfig
from ..database.repository import Repository
from ..utils.hashing import sha256_file
from ..utils.logging import get_logger
from .metadata import AudioMetadataError, is_supported_audio, read_metadata
from .models import FileStatus, ScanCandidate, ScanError, ScanResult
logger = get_logger("core.scanner")
ScanProgress = Callable[[int, Path], None]
# Toleranz fuer Dateisystem-Zeitstempel (Sekunden).
MTIME_TOLERANCE = 1.0
@dataclass
class _DiscoveredFile:
path: Path
size: int
mtime: float
duration: float | None = None
content_hash: str | None = None
error: str | None = None
existing_id: int | None = None
changed: bool = False
@dataclass
class _ScanState:
new_files: list[_DiscoveredFile] = field(default_factory=list)
changed_files: list[_DiscoveredFile] = field(default_factory=list)
def _iter_audio_files(root: Path) -> list[Path]:
found: list[Path] = []
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
dirnames.sort()
for filename in sorted(filenames):
if filename.startswith("."):
continue
path = Path(dirpath) / filename
if is_supported_audio(path):
found.append(path)
return found
def _should_hash(mode: str) -> bool:
return mode.strip().lower() == "always"
def scan(
root: str | Path,
repository: Repository,
processing: ProcessingConfig | None = None,
*,
progress: ScanProgress | None = None,
) -> ScanResult:
"""Scannt ``root`` rekursiv und aktualisiert die Datenbank."""
processing = processing or ProcessingConfig()
root_path = Path(root).expanduser().resolve()
if not root_path.exists():
raise FileNotFoundError(f"Pfad existiert nicht: {root_path}")
if not root_path.is_dir():
raise NotADirectoryError(f"Pfad ist kein Ordner: {root_path}")
result = ScanResult(root=str(root_path))
state = _ScanState()
files = _iter_audio_files(root_path)
result.files_found = len(files)
for index, path in enumerate(files, start=1):
if progress:
progress(index, path)
try:
stat = path.stat()
except OSError as exc:
result.errors.append(ScanError(str(path), f"Dateizugriff fehlgeschlagen: {exc}"))
continue
existing = repository.get_audio_by_path(str(path))
discovered = _DiscoveredFile(
path=path,
size=int(stat.st_size),
mtime=float(stat.st_mtime),
existing_id=existing.id if existing else None,
)
if existing is not None:
modified = abs(float(existing.modified_at) - discovered.mtime) > MTIME_TOLERANCE
if int(existing.file_size) != discovered.size or modified:
discovered.changed = True
state.changed_files.append(discovered)
else:
result.unchanged += 1
continue
else:
state.new_files.append(discovered)
try:
info = read_metadata(path)
discovered.duration = info.duration
except AudioMetadataError as exc:
discovered.error = str(exc)
result.errors.append(ScanError(str(path), str(exc)))
logger.warning("Metadaten konnten nicht gelesen werden: %s (%s)", path, exc)
_assign_hashes(state, repository, processing)
_insert_new_files(state.new_files, repository, processing, result)
_update_changed_files(state.changed_files, repository, processing, result)
logger.info(
"Scan abgeschlossen: %d gefunden, %d neu, %d geaendert, %d unveraendert, %d Duplikate, %d Fehler",
result.files_found,
result.added,
result.changed,
result.unchanged,
result.duplicates,
len(result.errors),
)
return result
def _assign_hashes(
state: _ScanState, repository: Repository, processing: ProcessingConfig
) -> None:
"""Berechnet Hashes nur dort, wo Duplikate moeglich sind."""
mode = (processing.compute_hash or "auto").strip().lower()
if mode == "never":
return
candidates = state.new_files + state.changed_files
if not candidates:
return
limit_bytes = max(0, int(processing.hash_max_size_mb)) * 1024 * 1024
if _should_hash(mode):
targets = [item for item in candidates if item.error is None]
else:
size_counts: dict[int, int] = {}
for item in candidates:
size_counts[item.size] = size_counts.get(item.size, 0) + 1
targets = []
for item in candidates:
if item.error is not None:
continue
needs_check = size_counts.get(item.size, 0) > 1
if not needs_check:
needs_check = bool(repository.paths_with_size(item.size))
if needs_check:
targets.append(item)
for item in targets:
if limit_bytes and item.size > limit_bytes:
logger.info("Datei zu gross fuer Hashing, wird uebersprungen: %s", item.path)
continue
try:
item.content_hash = sha256_file(item.path)
except OSError as exc:
logger.warning("Hashing fehlgeschlagen fuer %s: %s", item.path, exc)
def _insert_new_files(
files: list[_DiscoveredFile],
repository: Repository,
processing: ProcessingConfig,
result: ScanResult,
) -> None:
seen_hashes: dict[str, int] = {}
for item in files:
duplicate_of: int | None = None
status = FileStatus.PENDING.value
if item.content_hash:
existing = repository.find_by_hash(item.content_hash)
if existing is not None:
duplicate_of = existing.id
elif item.content_hash in seen_hashes:
duplicate_of = seen_hashes[item.content_hash]
if item.error is not None and duplicate_of is None:
status = FileStatus.ERROR.value
candidate = ScanCandidate(
path=str(item.path),
filename=item.path.name,
file_size=item.size,
modified_at=item.mtime,
duration=item.duration,
content_hash=item.content_hash,
)
audio_id = repository.insert_audio(
candidate,
status=status,
duplicate_of=duplicate_of,
error_message=item.error,
)
result.added += 1
if duplicate_of is not None:
result.duplicates += 1
elif item.content_hash:
seen_hashes[item.content_hash] = audio_id
def _update_changed_files(
files: list[_DiscoveredFile],
repository: Repository,
processing: ProcessingConfig,
result: ScanResult,
) -> None:
for item in files:
assert item.existing_id is not None
duplicate_of: int | None = None
status = FileStatus.PENDING.value
if item.content_hash:
existing = repository.find_by_hash(item.content_hash, exclude_id=item.existing_id)
if existing is not None:
duplicate_of = existing.id
if item.error is not None and duplicate_of is None:
status = FileStatus.ERROR.value
candidate = ScanCandidate(
path=str(item.path),
filename=item.path.name,
file_size=item.size,
modified_at=item.mtime,
duration=item.duration,
content_hash=item.content_hash,
)
repository.update_scan_data(
item.existing_id,
candidate,
status=status,
duplicate_of=duplicate_of,
error_message=item.error,
)
result.changed += 1
if duplicate_of is not None:
result.duplicates += 1
if files:
logger.info("%d geaenderte Dateien zur erneuten Verarbeitung vorgemerkt", len(files))
+63
View File
@@ -0,0 +1,63 @@
"""Verbindungsaufbau fuer die SQLite-Datenbank."""
from __future__ import annotations
import sqlite3
from pathlib import Path
from ..utils.logging import get_logger
logger = get_logger("database")
def connect(db_path: str | Path, *, create: bool = True) -> sqlite3.Connection:
"""Oeffnet die Datenbank mit sinnvollen PRAGMAs.
``create=False`` verhindert, dass versehentlich eine neue Datenbankdatei
angelegt wird (z. B. bei reiner Diagnose).
"""
path = Path(db_path)
if not create and not path.exists():
raise FileNotFoundError(f"Datenbank nicht gefunden: {path}")
path.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(str(path), timeout=30.0)
conn.row_factory = sqlite3.Row
conn.isolation_level = "" # explizite Transaktionen via "with conn:"
conn.execute("PRAGMA foreign_keys = ON")
conn.execute("PRAGMA journal_mode = WAL")
conn.execute("PRAGMA synchronous = NORMAL")
conn.execute("PRAGMA busy_timeout = 30000")
conn.execute("PRAGMA temp_store = MEMORY")
return conn
def check_fts5(conn: sqlite3.Connection) -> tuple[bool, bool]:
"""Prueft, ob FTS5 und der Trigram-Tokenizer verfuegbar sind."""
fts5 = False
trigram = False
try:
conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS _fts_check USING fts5(x)")
conn.execute("DROP TABLE IF EXISTS _fts_check")
fts5 = True
except sqlite3.OperationalError as exc:
logger.warning("FTS5 nicht verfuegbar: %s", exc)
return False, False
if fts5:
try:
conn.execute(
"CREATE VIRTUAL TABLE IF NOT EXISTS _trigram_check USING fts5(x, tokenize='trigram')"
)
conn.execute("DROP TABLE IF EXISTS _trigram_check")
trigram = True
except sqlite3.OperationalError as exc:
logger.warning("FTS5-Trigram-Tokenizer nicht verfuegbar: %s", exc)
return fts5, trigram
def sqlite_version() -> str:
return sqlite3.sqlite_version
+55
View File
@@ -0,0 +1,55 @@
"""Schema-Migrationen fuer SjAudioSearch.
Die Version wird in ``PRAGMA user_version`` gespeichert. Jede Migration laeuft
in einer eigenen Transaktion, damit ein Abbruch keinen halb migrierten Zustand
hinterlaesst.
"""
from __future__ import annotations
import sqlite3
from ..utils.logging import get_logger
from .schema import SCHEMA_VERSION, TABLES_SQL
logger = get_logger("database.migrations")
def current_version(conn: sqlite3.Connection) -> int:
row = conn.execute("PRAGMA user_version").fetchone()
return int(row[0]) if row else 0
def _migration_1(conn: sqlite3.Connection) -> None:
conn.executescript(TABLES_SQL)
MIGRATIONS: list[tuple[int, object]] = [
(1, _migration_1),
]
def migrate(conn: sqlite3.Connection, target: int = SCHEMA_VERSION) -> int:
"""Fuehrt alle ausstehenden Migrationen aus und liefert die neue Version."""
version = current_version(conn)
if version > target:
raise RuntimeError(
f"Datenbankversion {version} ist neuer als die unterstuetzte Version {target}."
)
if version == target:
return version
for migration_version, migration in MIGRATIONS:
if migration_version <= version:
continue
logger.info("Migration %s wird angewendet", migration_version)
with conn:
migration(conn) # type: ignore[operator]
conn.execute(f"PRAGMA user_version = {int(migration_version)}")
version = migration_version
if version != target:
raise RuntimeError(
f"Migration unvollstaendig: Version {version}, erwartet {target}."
)
return version
+547
View File
@@ -0,0 +1,547 @@
"""Datenbankzugriff fuer SjAudioSearch.
Alle Schreibvorgaenge laufen in expliziten Transaktionen, damit ein Abbruch
keine halben Transkripte hinterlaesst.
"""
from __future__ import annotations
import sqlite3
from collections.abc import Iterator, Sequence
from pathlib import Path
from typing import Any
from ..core.models import AudioFileRecord, FileStatus, RepositoryStats, ScanCandidate
from ..search.fuzzy import normalize_text
from ..utils.formatting import utc_now_iso
from ..utils.logging import get_logger
from .connection import connect
from .migrations import migrate
logger = get_logger("database.repository")
_AUDIO_COLUMNS = """
id, path, filename, duration, file_size, modified_at, content_hash,
status, duplicate_of, error_message, created_at, updated_at
"""
def _row_to_audio(row: sqlite3.Row) -> AudioFileRecord:
return AudioFileRecord(
id=row["id"],
path=row["path"],
filename=row["filename"],
duration=row["duration"],
file_size=row["file_size"],
modified_at=row["modified_at"],
content_hash=row["content_hash"],
status=row["status"],
duplicate_of=row["duplicate_of"],
error_message=row["error_message"],
created_at=row["created_at"],
updated_at=row["updated_at"],
)
class Repository:
"""Kapselt saemtliche SQL-Zugriffe."""
def __init__(self, connection: sqlite3.Connection):
self.conn = connection
@classmethod
def open(cls, db_path: str | Path) -> "Repository":
conn = connect(db_path)
migrate(conn)
return cls(conn)
def close(self) -> None:
self.conn.close()
def __enter__(self) -> "Repository":
return self
def __exit__(self, *exc_info: object) -> None:
self.close()
# ------------------------------------------------------------------
# audio_files
# ------------------------------------------------------------------
def get_audio(self, audio_id: int) -> AudioFileRecord | None:
row = self.conn.execute(
f"SELECT {_AUDIO_COLUMNS} FROM audio_files WHERE id = ?", (audio_id,)
).fetchone()
return _row_to_audio(row) if row else None
def get_audio_by_path(self, path: str) -> AudioFileRecord | None:
row = self.conn.execute(
f"SELECT {_AUDIO_COLUMNS} FROM audio_files WHERE path = ?", (str(path),)
).fetchone()
return _row_to_audio(row) if row else None
def list_audio(
self,
*,
status: str | None = None,
limit: int | None = None,
search: str | None = None,
include_duplicates: bool = True,
) -> list[AudioFileRecord]:
clauses: list[str] = []
params: list[Any] = []
if status:
clauses.append("status = ?")
params.append(status)
if not include_duplicates:
clauses.append("duplicate_of IS NULL")
if search:
clauses.append("(filename LIKE ? OR path LIKE ?)")
like = f"%{search}%"
params.extend([like, like])
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
sql = f"SELECT {_AUDIO_COLUMNS} FROM audio_files {where} ORDER BY filename COLLATE NOCASE, id"
if limit:
sql += " LIMIT ?"
params.append(int(limit))
rows = self.conn.execute(sql, params).fetchall()
return [_row_to_audio(row) for row in rows]
def insert_audio(
self,
candidate: ScanCandidate,
*,
status: str = FileStatus.PENDING.value,
duplicate_of: int | None = None,
error_message: str | None = None,
) -> int:
now = utc_now_iso()
with self.conn:
cursor = self.conn.execute(
"""
INSERT INTO audio_files
(path, filename, duration, file_size, modified_at, content_hash,
status, duplicate_of, error_message, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
candidate.path,
candidate.filename,
candidate.duration,
int(candidate.file_size),
float(candidate.modified_at),
candidate.content_hash,
status,
duplicate_of,
error_message,
now,
now,
),
)
return int(cursor.lastrowid)
def update_scan_data(
self,
audio_id: int,
candidate: ScanCandidate,
*,
status: str = FileStatus.PENDING.value,
duplicate_of: int | None = None,
error_message: str | None = None,
) -> None:
with self.conn:
self.conn.execute(
"""
UPDATE audio_files
SET duration = ?, file_size = ?, modified_at = ?, content_hash = ?,
status = ?, duplicate_of = ?, error_message = ?, updated_at = ?
WHERE id = ?
""",
(
candidate.duration,
int(candidate.file_size),
float(candidate.modified_at),
candidate.content_hash,
status,
duplicate_of,
error_message,
utc_now_iso(),
audio_id,
),
)
def set_duration(self, audio_id: int, duration: float | None) -> None:
if duration is None:
return
with self.conn:
self.conn.execute(
"UPDATE audio_files SET duration = ?, updated_at = ? WHERE id = ? AND duration IS NULL",
(float(duration), utc_now_iso(), audio_id),
)
def set_status(self, audio_id: int, status: str, error_message: str | None = None) -> None:
with self.conn:
self.conn.execute(
"UPDATE audio_files SET status = ?, error_message = ?, updated_at = ? WHERE id = ?",
(status, error_message, utc_now_iso(), audio_id),
)
def reset_stale_processing(self) -> int:
"""Setzt abgebrochene 'processing'-Eintraege nach einem Absturz zurueck."""
with self.conn:
cursor = self.conn.execute(
"UPDATE audio_files SET status = 'pending', updated_at = ? WHERE status = 'processing'",
(utc_now_iso(),),
)
return cursor.rowcount or 0
def reset_errors(self) -> int:
"""Setzt fehlgeschlagene Transkriptionen zurueck auf 'pending'."""
with self.conn:
cursor = self.conn.execute(
"""
UPDATE audio_files
SET status = 'pending', error_message = NULL, updated_at = ?
WHERE status = 'error'
""",
(utc_now_iso(),),
)
return cursor.rowcount or 0
def transcription_queue(self, *, force: bool = False) -> list[AudioFileRecord]:
if force:
sql = f"""
SELECT {_AUDIO_COLUMNS} FROM audio_files
WHERE duplicate_of IS NULL
ORDER BY id
"""
rows = self.conn.execute(sql).fetchall()
else:
sql = f"""
SELECT {_AUDIO_COLUMNS} FROM audio_files
WHERE duplicate_of IS NULL AND status IN ('pending', 'processing')
ORDER BY id
"""
rows = self.conn.execute(sql).fetchall()
return [_row_to_audio(row) for row in rows]
def find_by_hash(self, content_hash: str, *, exclude_id: int | None = None) -> AudioFileRecord | None:
sql = f"SELECT {_AUDIO_COLUMNS} FROM audio_files WHERE content_hash = ? AND duplicate_of IS NULL"
params: list[Any] = [content_hash]
if exclude_id is not None:
sql += " AND id != ?"
params.append(exclude_id)
row = self.conn.execute(sql + " ORDER BY id LIMIT 1", params).fetchone()
return _row_to_audio(row) if row else None
def paths_with_size(self, size: int) -> list[AudioFileRecord]:
rows = self.conn.execute(
f"SELECT {_AUDIO_COLUMNS} FROM audio_files WHERE file_size = ? AND content_hash IS NOT NULL",
(int(size),),
).fetchall()
return [_row_to_audio(row) for row in rows]
def count_by_status(self) -> dict[str, int]:
rows = self.conn.execute(
"SELECT status, COUNT(*) AS amount FROM audio_files GROUP BY status"
).fetchall()
return {row["status"]: int(row["amount"]) for row in rows}
def stats(self, db_size_bytes: int = 0) -> RepositoryStats:
counts = self.count_by_status()
total_row = self.conn.execute(
"SELECT COUNT(*) AS total, COALESCE(SUM(duration), 0) AS duration FROM audio_files"
).fetchone()
dup_row = self.conn.execute(
"SELECT COUNT(*) AS amount FROM audio_files WHERE duplicate_of IS NOT NULL"
).fetchone()
segment_row = self.conn.execute("SELECT COUNT(*) AS amount FROM segments").fetchone()
word_row = self.conn.execute("SELECT COUNT(*) AS amount FROM words").fetchone()
transcript_row = self.conn.execute("SELECT COUNT(*) AS amount FROM transcripts").fetchone()
model_rows = self.conn.execute("SELECT DISTINCT model FROM transcripts").fetchall()
return RepositoryStats(
total_files=int(total_row["total"]),
done=counts.get("done", 0),
pending=counts.get("pending", 0),
processing=counts.get("processing", 0),
error=counts.get("error", 0),
duplicates=int(dup_row["amount"]),
total_duration=float(total_row["duration"] or 0.0),
database_size=int(db_size_bytes),
segment_count=int(segment_row["amount"]),
word_count=int(word_row["amount"]),
transcripts=int(transcript_row["amount"]),
models=[row["model"] for row in model_rows],
)
def databases_size(self) -> int:
row = self.conn.execute("PRAGMA database_list").fetchone()
if not row:
return 0
path = Path(row["file"])
size = path.stat().st_size if path.exists() else 0
wal = path.with_name(path.name + "-wal")
if wal.exists():
size += wal.stat().st_size
return size
# ------------------------------------------------------------------
# Transkripte
# ------------------------------------------------------------------
def save_transcript(
self,
audio_id: int,
*,
language: str | None,
model: str,
full_text: str,
segments: Sequence[Any],
duration: float | None = None,
) -> int:
"""Speichert ein Transkript samt Segmenten, Woertern und Suchindex.
Vorhandene Daten der Datei werden ersetzt. Alles geschieht in einer
Transaktion, damit ein Abbruch keinen Teilzustand hinterlaesst.
"""
now = utc_now_iso()
with self.conn:
self.conn.execute("DELETE FROM transcripts WHERE audio_id = ?", (audio_id,))
self.conn.execute("DELETE FROM search_index WHERE audio_id = ?", (audio_id,))
self.conn.execute("DELETE FROM search_trigram WHERE audio_id = ?", (audio_id,))
cursor = self.conn.execute(
"""
INSERT INTO transcripts (audio_id, language, model, full_text, created_at)
VALUES (?, ?, ?, ?, ?)
""",
(audio_id, language, model, full_text, now),
)
transcript_id = int(cursor.lastrowid)
for segment in segments:
cursor = self.conn.execute(
"""
INSERT INTO segments (transcript_id, start_time, end_time, text, confidence)
VALUES (?, ?, ?, ?, ?)
""",
(
transcript_id,
float(segment.start),
float(segment.end),
segment.text,
getattr(segment, "confidence", None),
),
)
segment_id = int(cursor.lastrowid)
words = getattr(segment, "words", None) or []
if words:
self.conn.executemany(
"INSERT INTO words (segment_id, word, start_time, end_time) VALUES (?, ?, ?, ?)",
[
(
segment_id,
word.word,
getattr(word, "start", None),
getattr(word, "end", None),
)
for word in words
],
)
normalized = normalize_text(segment.text)
if normalized:
self.conn.execute(
"""
INSERT INTO search_index (text, audio_id, transcript_id, segment_id)
VALUES (?, ?, ?, ?)
""",
(normalized, audio_id, transcript_id, segment_id),
)
self.conn.execute(
"INSERT INTO search_trigram (text, audio_id, segment_id) VALUES (?, ?, ?)",
(normalized, audio_id, segment_id),
)
if duration is not None:
self.conn.execute(
"""
UPDATE audio_files
SET duration = COALESCE(duration, ?), status = 'done',
error_message = NULL, updated_at = ?
WHERE id = ?
""",
(float(duration), now, audio_id),
)
else:
self.conn.execute(
"""
UPDATE audio_files
SET status = 'done', error_message = NULL, updated_at = ?
WHERE id = ?
""",
(now, audio_id),
)
return transcript_id
def get_transcript(self, audio_id: int) -> sqlite3.Row | None:
return self.conn.execute(
"SELECT * FROM transcripts WHERE audio_id = ?", (audio_id,)
).fetchone()
def get_segments(self, audio_id: int) -> list[sqlite3.Row]:
return self.conn.execute(
"""
SELECT s.id, s.start_time, s.end_time, s.text, s.confidence
FROM segments s
JOIN transcripts t ON t.id = s.transcript_id
WHERE t.audio_id = ?
ORDER BY s.start_time, s.id
""",
(audio_id,),
).fetchall()
def get_words(self, segment_ids: Sequence[int]) -> dict[int, list[sqlite3.Row]]:
result: dict[int, list[sqlite3.Row]] = {}
ids = list(segment_ids)
for start in range(0, len(ids), 500):
chunk = ids[start : start + 500]
placeholders = ",".join("?" for _ in chunk)
rows = self.conn.execute(
f"""
SELECT id, segment_id, word, start_time, end_time
FROM words WHERE segment_id IN ({placeholders})
ORDER BY segment_id, start_time, id
""",
chunk,
).fetchall()
for row in rows:
result.setdefault(int(row["segment_id"]), []).append(row)
return result
def segment_texts(self, audio_id: int) -> list[str]:
return [row["text"] for row in self.get_segments(audio_id)]
def iter_audio_ids_with_transcripts(self, language: str | None = None) -> Iterator[int]:
sql = "SELECT DISTINCT audio_id FROM transcripts"
params: list[Any] = []
if language:
sql += " WHERE language = ?"
params.append(language)
sql += " ORDER BY audio_id"
cursor = self.conn.execute(sql, params)
for row in cursor:
yield int(row["audio_id"])
def transcript_languages(self) -> list[str]:
rows = self.conn.execute(
"SELECT DISTINCT language FROM transcripts WHERE language IS NOT NULL ORDER BY language"
).fetchall()
return [row["language"] for row in rows]
# ------------------------------------------------------------------
# Suche
# ------------------------------------------------------------------
def fts_candidates(self, match_query: str, limit: int) -> list[tuple[int, int]]:
"""Liefert (audio_id, segment_id) fuer einen FTS5-Ausdruck."""
if not match_query:
return []
try:
rows = self.conn.execute(
"""
SELECT audio_id, segment_id, bm25(search_index) AS rank
FROM search_index
WHERE search_index MATCH ?
ORDER BY rank
LIMIT ?
""",
(match_query, int(limit)),
).fetchall()
except sqlite3.OperationalError as exc:
logger.warning("FTS-Anfrage fehlgeschlagen (%s): %s", match_query, exc)
return []
return [(int(row["audio_id"]), int(row["segment_id"])) for row in rows]
def trigram_candidates(self, terms: Sequence[str], limit: int) -> list[tuple[int, int]]:
"""Teilstring-Kandidaten ueber den FTS5-Trigram-Index."""
results: list[tuple[int, int]] = []
seen: set[tuple[int, int]] = set()
for term in terms:
escaped = '"' + term.replace('"', '""') + '"'
try:
rows = self.conn.execute(
"""
SELECT audio_id, segment_id FROM search_trigram
WHERE search_trigram MATCH ?
LIMIT ?
""",
(escaped, int(limit)),
).fetchall()
except sqlite3.OperationalError as exc:
logger.warning("Trigramm-Anfrage fehlgeschlagen (%s): %s", term, exc)
continue
for row in rows:
key = (int(row["audio_id"]), int(row["segment_id"]))
if key not in seen:
seen.add(key)
results.append(key)
if len(results) >= limit:
return results
return results
def timeline(self, audio_id: int) -> tuple[list[sqlite3.Row], dict[int, list[sqlite3.Row]]]:
segments = self.get_segments(audio_id)
words = self.get_words([int(segment["id"]) for segment in segments])
return segments, words
def rebuild_search_index(self) -> int:
"""Baut den FTS-Index aus den gespeicherten Segmenten neu auf."""
with self.conn:
self.conn.execute("DELETE FROM search_index")
self.conn.execute("DELETE FROM search_trigram")
rows = self.conn.execute(
"""
SELECT s.id AS segment_id, t.audio_id AS audio_id, t.id AS transcript_id, s.text
FROM segments s
JOIN transcripts t ON t.id = s.transcript_id
ORDER BY s.id
"""
).fetchall()
index_rows = []
trigram_rows = []
for row in rows:
normalized = normalize_text(row["text"])
if not normalized:
continue
index_rows.append(
(normalized, int(row["audio_id"]), int(row["transcript_id"]), int(row["segment_id"]))
)
trigram_rows.append((normalized, int(row["audio_id"]), int(row["segment_id"])))
self.conn.executemany(
"INSERT INTO search_index (text, audio_id, transcript_id, segment_id) VALUES (?, ?, ?, ?)",
index_rows,
)
self.conn.executemany(
"INSERT INTO search_trigram (text, audio_id, segment_id) VALUES (?, ?, ?)",
trigram_rows,
)
return len(index_rows)
def delete_audio(self, audio_id: int) -> bool:
"""Entfernt eine Datei aus dem Index (die Audiodatei bleibt unberuehrt)."""
with self.conn:
self.conn.execute("DELETE FROM search_index WHERE audio_id = ?", (audio_id,))
self.conn.execute("DELETE FROM search_trigram WHERE audio_id = ?", (audio_id,))
cursor = self.conn.execute("DELETE FROM audio_files WHERE id = ?", (audio_id,))
return (cursor.rowcount or 0) > 0
def upsert_meta(self, key: str, value: str) -> None:
with self.conn:
self.conn.execute(
"INSERT INTO meta (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
(key, value),
)
def get_meta(self, key: str) -> str | None:
row = self.conn.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone()
return row["value"] if row else None
+80
View File
@@ -0,0 +1,80 @@
"""SQL-Schema fuer SjAudioSearch (SQLite + FTS5)."""
from __future__ import annotations
SCHEMA_VERSION = 1
TABLES_SQL = """
CREATE TABLE IF NOT EXISTS audio_files (
id INTEGER PRIMARY KEY AUTOINCREMENT,
path TEXT NOT NULL UNIQUE,
filename TEXT NOT NULL,
duration REAL,
file_size INTEGER NOT NULL,
modified_at REAL NOT NULL,
content_hash TEXT,
status TEXT NOT NULL DEFAULT 'pending'
CHECK (status IN ('pending', 'processing', 'done', 'error')),
duplicate_of INTEGER REFERENCES audio_files(id) ON DELETE SET NULL,
error_message TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_audio_status ON audio_files(status);
CREATE INDEX IF NOT EXISTS idx_audio_hash ON audio_files(content_hash);
CREATE INDEX IF NOT EXISTS idx_audio_size ON audio_files(file_size);
CREATE INDEX IF NOT EXISTS idx_audio_duplicate ON audio_files(duplicate_of);
CREATE TABLE IF NOT EXISTS transcripts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
audio_id INTEGER NOT NULL UNIQUE REFERENCES audio_files(id) ON DELETE CASCADE,
language TEXT,
model TEXT NOT NULL,
full_text TEXT NOT NULL,
created_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_transcripts_language ON transcripts(language);
CREATE TABLE IF NOT EXISTS segments (
id INTEGER PRIMARY KEY AUTOINCREMENT,
transcript_id INTEGER NOT NULL REFERENCES transcripts(id) ON DELETE CASCADE,
start_time REAL NOT NULL,
end_time REAL NOT NULL,
text TEXT NOT NULL,
confidence REAL
);
CREATE INDEX IF NOT EXISTS idx_segments_transcript ON segments(transcript_id, start_time);
CREATE TABLE IF NOT EXISTS words (
id INTEGER PRIMARY KEY AUTOINCREMENT,
segment_id INTEGER NOT NULL REFERENCES segments(id) ON DELETE CASCADE,
word TEXT NOT NULL,
start_time REAL,
end_time REAL
);
CREATE INDEX IF NOT EXISTS idx_words_segment ON words(segment_id);
CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
text,
audio_id UNINDEXED,
transcript_id UNINDEXED,
segment_id UNINDEXED,
tokenize = "unicode61 remove_diacritics 2"
);
CREATE VIRTUAL TABLE IF NOT EXISTS search_trigram USING fts5(
text,
audio_id UNINDEXED,
segment_id UNINDEXED,
tokenize = "trigram"
);
CREATE TABLE IF NOT EXISTS meta (
key TEXT PRIMARY KEY,
value TEXT
);
"""
+186
View File
@@ -0,0 +1,186 @@
"""Kontextaufbau fuer Suchtreffer.
Aus den gespeicherten Segmenten wird eine durchgehende Wort-Timeline gebaut.
Dadurch gehen an Segmentgrenzen keine Informationen verloren, und der Kontext
kann mitten in einem Segment beginnen und enden.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Sequence
from .fuzzy import normalize_text, tokenize_original
@dataclass
class TokenRef:
"""Ein Wort im Transkript mit Zeitbezug und Herkunft."""
text: str
norm: str
start: float
end: float
segment_id: int
precision: str # "word" oder "segment"
@dataclass
class Timeline:
"""Durchgehende Wortfolge einer Audiodatei."""
audio_id: int
tokens: list[TokenRef] = field(default_factory=list)
language: str | None = None
model: str | None = None
@property
def normalized_tokens(self) -> list[str]:
return [token.norm for token in self.tokens]
@property
def normalized_text(self) -> str:
return " ".join(token.norm for token in self.tokens)
@property
def original_text(self) -> str:
return " ".join(token.text for token in self.tokens)
def __len__(self) -> int:
return len(self.tokens)
def _distribute_times(
segment_start: float, segment_end: float, token_count: int
) -> list[tuple[float, float]]:
"""Verteilt die Segmentzeit gleichmaessig auf die Woerter (Schaetzung)."""
if token_count <= 0:
return []
span = max(0.0, float(segment_end) - float(segment_start))
step = span / token_count if span else 0.0
times: list[tuple[float, float]] = []
for index in range(token_count):
start = float(segment_start) + step * index
end = float(segment_start) + step * (index + 1)
times.append((round(start, 3), round(end, 3)))
return times
def build_timeline(
audio_id: int,
segments: Sequence[Any],
words_by_segment: dict[int, Sequence[Any]] | None = None,
*,
language: str | None = None,
model: str | None = None,
) -> Timeline:
"""Baut die Timeline aus Segmenten (sqlite3.Row oder Objekte mit Attributen)."""
timeline = Timeline(audio_id=audio_id, language=language, model=model)
words_by_segment = words_by_segment or {}
for segment in segments:
if isinstance(segment, dict):
segment_id = int(segment["id"])
start = float(segment["start_time"])
end = float(segment["end_time"])
text = segment["text"]
else:
segment_id = int(segment["id"])
start = float(segment["start_time"])
end = float(segment["end_time"])
text = segment["text"]
original_tokens = tokenize_original(text)
if not original_tokens:
continue
words = list(words_by_segment.get(segment_id, []))
use_words = len(words) == len(original_tokens)
if use_words:
for token, word in zip(original_tokens, words):
word_start = word["start_time"] if not isinstance(word, dict) else word.get("start_time")
word_end = word["end_time"] if not isinstance(word, dict) else word.get("end_time")
if word_start is None or word_end is None:
use_words = False
break
if use_words:
for token, word in zip(original_tokens, words):
word_start = word["start_time"] if not isinstance(word, dict) else word["start_time"]
word_end = word["end_time"] if not isinstance(word, dict) else word["end_time"]
timeline.tokens.append(
TokenRef(
text=token,
norm=normalize_text(token),
start=float(word_start),
end=float(word_end),
segment_id=segment_id,
precision="word",
)
)
else:
times = _distribute_times(start, end, len(original_tokens))
for token, (token_start, token_end) in zip(original_tokens, times):
timeline.tokens.append(
TokenRef(
text=token,
norm=normalize_text(token),
start=token_start,
end=token_end,
segment_id=segment_id,
precision="segment",
)
)
return timeline
@dataclass
class SearchContext:
"""Die drei Kontextabschnitte eines Treffers."""
before: str
match: str
after: str
start_time: float
end_time: float
precision: str
def as_block(self, quote: str = '"') -> str:
"""Formatiert den Kontext als zusammenhaengenden Textblock."""
before = f"\u2026{self.before} " if self.before else ""
after = f" {self.after}\u2026" if self.after else ""
return f"{quote}{before}{self.match}{after}{quote}"
def build_context(
timeline: Timeline, start_index: int, end_index: int, context_words: int = 50
) -> SearchContext:
"""Schneidet Kontext um den Treffer ``[start_index, end_index)`` aus."""
tokens = timeline.tokens
if not tokens:
return SearchContext("", "", "", 0.0, 0.0, "segment")
start_index = max(0, min(start_index, len(tokens)))
end_index = max(start_index + 1, min(end_index, len(tokens)))
before_start = max(0, start_index - max(0, context_words))
after_end = min(len(tokens), end_index + max(0, context_words))
before_tokens = tokens[before_start:start_index]
match_tokens = tokens[start_index:end_index]
after_tokens = tokens[end_index:after_end]
precision = "word" if all(token.precision == "word" for token in match_tokens) else "segment"
start_time = match_tokens[0].start
end_time = match_tokens[-1].end
return SearchContext(
before=" ".join(token.text for token in before_tokens),
match=" ".join(token.text for token in match_tokens),
after=" ".join(token.text for token in after_tokens),
start_time=start_time,
end_time=end_time,
precision=precision,
)
+303
View File
@@ -0,0 +1,303 @@
"""Suchmaschine fuer SjAudioSearch.
Ablauf einer Suche:
1. Anfrage normalisieren.
2. Kandidaten ueber FTS5 und den Trigramm-Index sammeln (schnelle Vorauswahl).
3. Sliding-Window-Vergleich mit RapidFuzz auf der Wort-Timeline der Kandidaten.
4. Optionaler, vollstaendiger Fallback-Lauf ueber alle verbleibenden Dateien,
damit auch komplett falsch erkannte Woerter gefunden werden.
5. Treffer entduplizieren, sortieren, begrenzen.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Callable
from rapidfuzz import fuzz
from ..config import SearchConfig
from ..database.repository import Repository
from ..utils.logging import get_logger
from .context import SearchContext, Timeline, build_context, build_timeline
from .fuzzy import (
build_fts_query,
normalize_text,
score_window,
significant_overlap,
trigram_terms,
)
from .ranking import dedupe_overlapping, rank_hits
logger = get_logger("search.engine")
ProgressCallback = Callable[[int, int], None]
class SearchError(RuntimeError):
"""Fehler bei der Suche (z. B. leere Anfrage)."""
@dataclass
class SearchHit:
"""Ein einzelner Suchtreffer."""
audio_id: int
filename: str
path: str
score: float
start_time: float
end_time: float
start_token: int
end_token: int
context: SearchContext
language: str | None = None
model: str | None = None
source: str = "fts"
@property
def matched_text(self) -> str:
return self.context.match
def to_dict(self, context_words: int = 50) -> dict[str, object]:
return {
"audio_id": self.audio_id,
"filename": self.filename,
"path": self.path,
"similarity": round(self.score, 2),
"timestamp": self.start_time,
"timestamp_end": self.end_time,
"timestamp_precision": self.context.precision,
"match": self.context.match,
"context_before": self.context.before,
"context_after": self.context.after,
"context_words": context_words,
"language": self.language,
"model": self.model,
"source": self.source,
}
class SearchEngine:
"""Kombinierte Suche aus FTS5, Trigrammen und Fuzzy-Sliding-Window."""
def __init__(self, repository: Repository, config: SearchConfig):
self.repo = repository
self.config = config
# ------------------------------------------------------------------
def search(
self,
query: str,
*,
limit: int | None = None,
min_score: float | None = None,
context_words: int | None = None,
language: str | None = None,
progress: ProgressCallback | None = None,
) -> list[SearchHit]:
query_normalized = normalize_text(query)
if not query_normalized:
raise SearchError("Die Suchanfrage enthaelt keine verwertbaren Woerter.")
query_tokens = query_normalized.split()
limit = self.config.default_limit if limit is None else int(limit)
min_score = self.config.default_min_score if min_score is None else float(min_score)
context_words = self.config.context_words if context_words is None else int(context_words)
if limit <= 0:
raise SearchError("--limit muss groesser als 0 sein.")
candidate_audio, candidate_source = self._collect_candidates(query_tokens)
hits: list[SearchHit] = []
swept: set[int] = set()
total_candidates = len(candidate_audio)
for index, audio_id in enumerate(candidate_audio, start=1):
swept.add(audio_id)
hits.extend(
self._scan_audio(
audio_id,
query_normalized,
query_tokens,
min_score=min_score,
context_words=context_words,
language=language,
source="fts",
)
)
if progress:
progress(index, total_candidates)
# Wichtig: FTS5 ist nur eine Vorauswahl. Wenn wenige Treffer gefunden
# wurden, werden alle uebrigen Dateien vollstaendig durchsucht.
fallback_used = False
if self.config.exhaustive_fallback and len(hits) < limit:
remaining = [
audio_id
for audio_id in self.repo.iter_audio_ids_with_transcripts(language)
if audio_id not in swept
]
fallback_used = bool(remaining)
for index, audio_id in enumerate(remaining, start=1):
hits.extend(
self._scan_audio(
audio_id,
query_normalized,
query_tokens,
min_score=min_score,
context_words=context_words,
language=language,
source="fallback",
)
)
if progress:
progress(index, len(remaining))
if fallback_used:
logger.info(
"Fallback-Suche aktiv: %d Dateien ohne FTS-Treffer durchsucht", len(remaining)
)
deduped = dedupe_overlapping(hits)
return rank_hits(deduped, limit)
# ------------------------------------------------------------------
def _collect_candidates(self, query_tokens: list[str]) -> tuple[list[int], str]:
"""Sammelt Kandidaten-Audiodateien (FTS5 + Trigramme)."""
limit = max(1, int(self.config.candidate_limit))
matches: list[tuple[int, int]] = []
fts_query = build_fts_query(query_tokens)
if fts_query:
matches.extend(self.repo.fts_candidates(fts_query, limit))
remaining = limit - len(matches)
if remaining > 0:
terms = trigram_terms(query_tokens)
if terms:
matches.extend(self.repo.trigram_candidates(terms, remaining))
counts: dict[int, int] = {}
for audio_id, _segment_id in matches:
counts[audio_id] = counts.get(audio_id, 0) + 1
ordered = sorted(counts, key=lambda audio_id: (-counts[audio_id], audio_id))
return ordered, "fts"
# ------------------------------------------------------------------
def _scan_audio(
self,
audio_id: int,
query_normalized: str,
query_tokens: list[str],
*,
min_score: float,
context_words: int,
language: str | None,
source: str,
) -> list[SearchHit]:
record = self.repo.get_audio(audio_id)
if record is None or record.duplicate_of is not None:
return []
transcript = self.repo.get_transcript(audio_id)
if transcript is None:
return []
if language and (transcript["language"] or "").lower() != language.lower():
return []
segments, words = self.repo.timeline(audio_id)
if not segments:
return []
timeline = build_timeline(
audio_id,
segments,
words,
language=transcript["language"],
model=transcript["model"],
)
if not timeline.tokens:
return []
windows = self._find_windows(timeline, query_normalized, query_tokens, min_score)
if not windows:
return []
hits: list[SearchHit] = []
for score, start_index, end_index in windows:
context = build_context(timeline, start_index, end_index, context_words)
hits.append(
SearchHit(
audio_id=audio_id,
filename=record.filename,
path=record.path,
score=score,
start_time=context.start_time,
end_time=context.end_time,
start_token=start_index,
end_token=end_index,
context=context,
language=transcript["language"],
model=transcript["model"],
source=source,
)
)
hits.sort(key=lambda hit: -hit.score)
return hits[: max(1, int(self.config.max_hits_per_file))]
# ------------------------------------------------------------------
def _find_windows(
self,
timeline: Timeline,
query_normalized: str,
query_tokens: list[str],
min_score: float,
) -> list[tuple[float, int, int]]:
"""Sliding-Window-Suche ueber die Wort-Timeline einer Datei."""
norm_tokens = timeline.normalized_tokens
token_count = len(norm_tokens)
query_len = len(query_tokens)
if token_count == 0:
return []
min_size = max(1, query_len - max(0, self.config.window_before))
max_size = max(min_size, query_len + max(0, self.config.window_after))
overlap_threshold = max(0.0, min(1.0, self.config.min_token_overlap))
# Dateiebene: gibt es ueberhaupt Beruehrungspunkte? Das ist deutlich
# billiger als jede Datei Fenster fuer Fenster zu vergleichen.
file_overlap = significant_overlap(query_tokens, norm_tokens)
if file_overlap < overlap_threshold * 0.5:
partial = fuzz.partial_ratio(query_normalized, timeline.normalized_text)
if partial < 55.0:
return []
windows: list[tuple[float, int, int]] = []
# Wenn auf Dateiebene kaum Wortueberschneidung besteht, kann der Treffer
# nur ueber einen falsch erkannten Ausdruck zustande kommen. Dann wird
# ohne Wort-Vorfilter verglichen (selten und durch die Dateipruefung
# oben bereits eingegrenzt).
use_overlap_filter = file_overlap >= overlap_threshold
for start in range(token_count):
pre_end = min(token_count, start + query_len + 2)
if use_overlap_filter and significant_overlap(query_tokens, norm_tokens[start:pre_end]) < overlap_threshold:
continue
best_score = 0.0
best_size = 0
for size in range(min_size, max_size + 1):
end = start + size
if end > token_count:
break
window_normalized = " ".join(norm_tokens[start:end])
candidate = score_window(query_normalized, window_normalized)
if candidate.score > best_score:
best_score = candidate.score
best_size = size
if best_size and best_score >= min_score:
windows.append((best_score, start, start + best_size))
return windows
+178
View File
@@ -0,0 +1,178 @@
"""Textnormalisierung und Fuzzy-Vergleich (RapidFuzz).
Wichtig: Die Normalisierung wird ausschliesslich fuer Vergleiche und fuer den
Suchindex verwendet. Der gespeicherte Originaltext bleibt unveraendert.
"""
from __future__ import annotations
import re
import unicodedata
from dataclasses import dataclass
from rapidfuzz import fuzz
# Satzzeichen und Symbole, die beim Vergleich stoeren. Wortzeichen (Buchstaben,
# Ziffern, Unterstrich) bleiben erhalten.
_PUNCT_RE = re.compile(r"[^\w\s]", re.UNICODE)
_WHITESPACE_RE = re.compile(r"\s+", re.UNICODE)
# Sehr haeufige deutsche Fuellwoerter. Sie werden beim Vorfilter nicht als
# belastbare Uebereinstimmung gewertet, damit "der/die/das" allein keine
# Kandidaten erzeugt.
STOPWORDS = frozenset(
{
"der", "die", "das", "den", "dem", "des", "ein", "eine", "einer", "eines",
"einem", "einen", "und", "oder", "aber", "als", "auch", "auf", "aus", "bei",
"bin", "bis", "da", "dann", "dass", "doch", "dort", "du", "durch", "er",
"es", "hat", "ich", "ihm", "ihn", "ihr", "im", "in", "ist", "mit", "nach",
"nicht", "noch", "nun", "nur", "ob", "so", "sich", "sie", "sind", "um",
"und", "vom", "von", "vor", "war", "was", "wenn", "wie", "wir", "wo", "zu",
"zum", "zur", "the", "and", "of", "to", "in", "a", "is",
}
)
def normalize_text(text: str) -> str:
"""Normalisiert Text fuer Vergleiche: NFKC, Kleinschreibung, ohne Satzzeichen."""
if not text:
return ""
normalized = unicodedata.normalize("NFKC", text).casefold()
normalized = _PUNCT_RE.sub(" ", normalized)
return _WHITESPACE_RE.sub(" ", normalized).strip()
def tokenize(text: str) -> list[str]:
"""Normalisierte Wortliste eines Textes."""
normalized = normalize_text(text)
return normalized.split() if normalized else []
def tokenize_original(text: str) -> list[str]:
"""Wortliste mit Originaltext inklusive anhängender Satzzeichen (fuer die Ausgabe)."""
if not text:
return []
return [token for token in _WHITESPACE_RE.split(text.strip()) if token]
def token_norm(token: str) -> str:
"""Normalisiert ein einzelnes Originalwort."""
return normalize_text(token)
def is_stopword(token: str) -> bool:
return token in STOPWORDS
@dataclass(frozen=True)
class WindowScore:
"""Ergebnis eines Fenstervergleichs."""
score: float
exact: bool
def score_window(query_normalized: str, window_normalized: str) -> WindowScore:
"""Vergleicht Suchanfrage und Textfenster und liefert einen Wert 0-100.
Es werden mehrere RapidFuzz-Kennzahlen kombiniert, damit fehlende,
zusaetzliche, vertauschte oder falsch erkannte Woerter nicht sofort zum
Verlust des Treffers fuehren.
"""
if not query_normalized or not window_normalized:
return WindowScore(0.0, False)
if query_normalized == window_normalized:
return WindowScore(100.0, True)
# Hinweis: Eine reine Teilzeichenketten-Pruefung ist hier absichtlich NICHT
# enthalten. Sie wuerde z. B. das Fenster "Leider" fuer die Anfrage
# "Leider zu spaet" mit 100 bewerten und den Treffer auf ein Wort
# zusammenfallen lassen. Teiluebereinstimmungen werden stattdessen ueber
# die gewichteten RapidFuzz-Kennzahlen unten bewertet.
ratio = fuzz.ratio(query_normalized, window_normalized)
token_set = fuzz.token_set_ratio(query_normalized, window_normalized)
weighted = fuzz.WRatio(query_normalized, window_normalized)
partial = fuzz.partial_ratio(query_normalized, window_normalized)
# partial_ratio neigt bei kurzen Fenstern zu hohen Werten, daher nur
# gedämpft einbeziehen.
# token_set_ratio/WRatio/partial_ratio bewerten eine Teilmenge der Anfrage
# mit 100. Ohne Daempfung wuerde ein einzelnes Wort einer mehrwoertrigen
# Anfrage als perfekter Treffer gelten. Daher werden diese Kennzahlen mit
# dem Laengenverhaeltnis gewichtet, wenn das Fenster kuerzer ist.
length_factor = min(1.0, len(window_normalized) / max(1, len(query_normalized)))
score = max(
ratio,
token_set * length_factor,
weighted * length_factor,
partial * 0.92 * length_factor,
)
return WindowScore(min(100.0, round(score, 2)), False)
def token_overlap(query_tokens: list[str], window_tokens: list[str]) -> float:
"""Anteil der Anfragewoerter, die im Fenster vorkommen (0..1)."""
if not query_tokens:
return 0.0
window_set = set(window_tokens)
hits = sum(1 for token in query_tokens if token in window_set)
return hits / len(query_tokens)
def significant_overlap(query_tokens: list[str], window_tokens: list[str]) -> float:
"""Wie :func:`token_overlap`, aber Fuellwoerter zaehlen nur halb."""
if not query_tokens:
return 0.0
window_set = set(window_tokens)
total = 0.0
hits = 0.0
for token in query_tokens:
weight = 0.5 if is_stopword(token) else 1.0
total += weight
if token in window_set:
hits += weight
if total == 0:
return 0.0
return hits / total
def _escape_fts_token(token: str) -> str:
return '"' + token.replace('"', '""') + '"'
def build_fts_query(tokens: list[str], prefix_min_length: int = 4) -> str:
"""Baut eine FTS5-Anfrage aus ODER-verknuepften Woertern.
Praefixsuche fuer laengere Woerter faengt deutsche Flexionsformen und
Zusammensetzungen ab. Die Anfrage ist bewusst weit gefasst: sie dient nur
der Kandidatensuche, die Bewertung erfolgt spaeter per Fuzzy-Vergleich.
"""
parts: list[str] = []
seen: set[str] = set()
for token in tokens:
if len(token) < 2 or token in seen:
continue
seen.add(token)
escaped = _escape_fts_token(token)
if len(token) >= prefix_min_length:
parts.append(f"{escaped}*")
else:
parts.append(escaped)
return " OR ".join(parts)
def trigram_terms(tokens: list[str], max_terms: int = 32) -> list[str]:
"""Sinnvolle Trigramm-Anfragen aus laengeren Anfragewoertern."""
terms: list[str] = []
seen: set[str] = set()
for token in tokens:
if len(token) < 4 or token in seen:
continue
seen.add(token)
terms.append(token)
if len(terms) >= max_terms:
break
return terms
+53
View File
@@ -0,0 +1,53 @@
"""Bewertung und Sortierung von Suchtreffern.
Der Score ist eine Aehnlichkeitsbewertung (0-100) und ausdruecklich keine
statistische Wahrscheinlichkeit.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING: # pragma: no cover
from .engine import SearchHit
def clamp_score(score: float) -> float:
return max(0.0, min(100.0, float(score)))
def dedupe_overlapping(hits: list["SearchHit"], gap_tokens: int = 3) -> list["SearchHit"]:
"""Entfernt Treffer, die denselben Textabschnitt mehrfach abdecken."""
if not hits:
return []
ordered = sorted(hits, key=lambda hit: (-hit.score, hit.start_token))
kept: list["SearchHit"] = []
for hit in ordered:
overlapping = False
for existing in kept:
if existing.audio_id != hit.audio_id:
continue
same_region = (
abs(existing.start_token - hit.start_token) <= gap_tokens
or (hit.start_token < existing.end_token and existing.start_token < hit.end_token)
)
if same_region:
overlapping = True
break
if not overlapping:
kept.append(hit)
return kept
def rank_hits(hits: list["SearchHit"], limit: int) -> list["SearchHit"]:
"""Sortiert Treffer nach Relevanz und begrenzt die Anzahl."""
ordered = sorted(
hits,
key=lambda hit: (
-hit.score,
hit.filename.lower(),
hit.start_time,
),
)
return ordered[: max(0, int(limit))]
+233
View File
@@ -0,0 +1,233 @@
"""Systemdiagnose und verstaendlicher Systembericht."""
from __future__ import annotations
import os
import shutil
import sqlite3
from dataclasses import dataclass, field
from pathlib import Path
from rich.console import Console
from rich.panel import Panel
from rich.table import Table
from ..config import Config
from ..utils.formatting import format_size
from ..utils.logging import get_logger
from .gpu import (
GpuInfo,
ResolvedExecution,
RuntimeInfo,
collect_runtime_info,
detect_gpu,
is_pascal_or_older,
resolve_execution,
)
logger = get_logger("system.diagnostics")
CHECK = "[green]OK[/green]"
WARN = "[yellow]Warnung[/yellow]"
FAIL = "[red]nicht verfuegbar[/red]"
@dataclass
class SystemReport:
"""Gesammelte Systeminformationen."""
gpu: GpuInfo | None
runtime: RuntimeInfo
execution: ResolvedExecution
db_path: Path
db_exists: bool
config_path: str | None
model_dir: Path
model_dir_writable: bool
ffmpeg: str | None
ffplay: str | None
notes: list[str] = field(default_factory=list)
open_tasks: list[str] = field(default_factory=list)
@property
def gpu_ready(self) -> bool:
return self.runtime.cuda_runtime_usable and self.execution.device == "cuda"
@property
def verdict(self) -> str:
if self.gpu_ready:
return (
f"GPU-Transkription moeglich mit compute_type='{self.execution.compute_type}'."
)
return "Keine GPU-Transkription moeglich - CPU-Fallback wird verwendet."
def _is_writable(path: Path) -> bool:
target = path
while not target.exists() and target != target.parent:
target = target.parent
return target.is_dir() and os.access(target, os.W_OK)
def collect_report(config: Config, *, requested_device: str | None = None,
requested_compute_type: str | None = None) -> SystemReport:
"""Erstellt einen vollstaendigen Systembericht."""
gpu = detect_gpu()
runtime = collect_runtime_info(check_fts=True)
device = requested_device or config.transcription.device
compute_type = requested_compute_type or config.transcription.compute_type
execution = resolve_execution(device, compute_type, gpu, runtime)
db_path = config.database_path
model_dir = config.model_path
notes: list[str] = []
open_tasks: list[str] = []
if gpu is not None and is_pascal_or_older(gpu):
notes.append(
f"{gpu.name} (Compute Capability {gpu.compute_capability}) ist eine Pascal-Karte: "
"kein FP16/BF16, kein Tensor-Core-Betrieb. int8_float32 ist der passende Modus."
)
if runtime.cudnn_libraries:
notes.append(
f"cuDNN gefunden ({len(runtime.cudnn_libraries)} Bibliothek(en)); "
"SjAudioSearch laedt die pip-nvidia-Bibliotheken beim Start automatisch vor."
)
else:
notes.append(
"Kein cuDNN gefunden. Fuer GPU-Betrieb die CUDA-12-Wheels installieren: "
"pip install -e \".[cuda]\" (nvidia-cublas-cu12, nvidia-cudnn-cu12)."
)
if not runtime.fts5_available:
open_tasks.append(
"SQLite ohne FTS5 - bitte Python mit FTS5-Unterstuetzung installieren."
)
if not runtime.trigram_available:
open_tasks.append(
"Trigramm-Tokenizer nicht verfuegbar - die Kandidatensuche arbeitet nur ueber FTS5."
)
return SystemReport(
gpu=gpu,
runtime=runtime,
execution=execution,
db_path=db_path,
db_exists=db_path.exists(),
config_path=str(config.source_path) if config.source_path else None,
model_dir=model_dir,
model_dir_writable=_is_writable(model_dir.parent if not model_dir.exists() else model_dir),
ffmpeg=shutil.which("ffmpeg"),
ffplay=shutil.which("ffplay"),
notes=notes,
open_tasks=open_tasks,
)
def render_report(report: SystemReport, console: Console) -> None:
"""Gibt den Systembericht uebersichtlich im Terminal aus."""
console.print(Panel.fit("[bold]SjAudioSearch - Systemdiagnose[/bold]", border_style="cyan"))
system = Table(show_header=False, box=None, padding=(0, 1))
system.add_column(style="bold cyan", no_wrap=True)
system.add_column()
system.add_row("Python", report.runtime.python_version)
system.add_row("System", report.runtime.platform)
system.add_row("SQLite", report.runtime.sqlite_version)
system.add_row("FTS5", CHECK if report.runtime.fts5_available else FAIL)
system.add_row("FTS5-Trigramme", CHECK if report.runtime.trigram_available else WARN)
system.add_row("CTranslate2", report.runtime.ctranslate2_version or FAIL)
system.add_row("FFmpeg", report.ffmpeg or WARN)
system.add_row("ffplay", report.ffplay or WARN)
system.add_row("Konfiguration", report.config_path or "eingebaute Defaults")
system.add_row("Datenbank", f"{report.db_path} ({'vorhanden' if report.db_exists else 'wird angelegt'})")
system.add_row("Modellordner", f"{report.model_dir} ({'beschreibbar' if report.model_dir_writable else 'NICHT beschreibbar'})")
console.print(system)
console.print()
gpu_table = Table(title="GPU", show_header=False, box=None, padding=(0, 1))
gpu_table.add_column(style="bold cyan", no_wrap=True)
gpu_table.add_column()
if report.gpu is None:
gpu_table.add_row("Status", FAIL + " (nvidia-smi lieferte keine Daten)")
else:
gpu_table.add_row("Modell", report.gpu.name)
gpu_table.add_row("Treiber", report.gpu.driver_version or "unbekannt")
gpu_table.add_row("CUDA (Treiber)", report.gpu.cuda_driver_version or "unbekannt")
gpu_table.add_row(
"VRAM", f"{report.gpu.memory_total_mb} MB" if report.gpu.memory_total_mb else "unbekannt"
)
gpu_table.add_row("Compute Capability", report.gpu.compute_capability or "unbekannt")
gpu_table.add_row("Architektur", "Pascal oder aelter" if is_pascal_or_older(report.gpu) else "neuere Architektur")
gpu_table.add_row(
"CUDA-Geraete (CTranslate2)", str(report.runtime.cuda_device_count)
)
gpu_table.add_row(
"Compute Types (CUDA)",
", ".join(report.runtime.cuda_compute_types) or "keine",
)
gpu_table.add_row(
"Compute Types (CPU)",
", ".join(report.runtime.cpu_compute_types) or "keine",
)
gpu_table.add_row(
"CUDA-Bibliotheken",
", ".join(Path(lib).name for lib in report.runtime.cuda_libraries) or "keine im System",
)
gpu_table.add_row(
"cuDNN-Bibliotheken",
", ".join(Path(lib).name for lib in report.runtime.cudnn_libraries) or "keine im System",
)
console.print(gpu_table)
console.print()
plan = Table(title="Empfohlener Ausfuehrungsmodus", show_header=False, box=None, padding=(0, 1))
plan.add_column(style="bold cyan", no_wrap=True)
plan.add_column()
plan.add_row("Angefragt", f"{report.execution.requested_device} / {report.execution.requested_compute_type}")
plan.add_row("Verwendet", f"{report.execution.device} / {report.execution.compute_type}")
plan.add_row("FP16/BF16", "nicht verwendet" if report.execution.device == "cuda" else "irrelevant (CPU)")
console.print(plan)
for note in report.execution.notes:
console.print(f"[yellow]-[/yellow] {note}")
for note in report.notes:
console.print(f"[cyan]-[/cyan] {note}")
console.print()
color = "green" if report.gpu_ready else "yellow"
console.print(Panel(report.verdict, border_style=color, title="Ergebnis"))
if report.open_tasks:
console.print()
console.print("[bold red]Offene Punkte:[/bold red]")
for task in report.open_tasks:
console.print(f" [red]*[/red] {task}")
def database_summary(db_path: Path) -> dict[str, object] | None:
"""Kurze Zusammenfassung einer vorhandenen Datenbank (fuer die Diagnose)."""
if not db_path.exists():
return None
try:
conn = sqlite3.connect(str(db_path))
try:
tables = {
row[0]
for row in conn.execute(
"SELECT name FROM sqlite_master WHERE type IN ('table','view')"
)
}
audio_count = 0
if "audio_files" in tables:
audio_count = int(conn.execute("SELECT COUNT(*) FROM audio_files").fetchone()[0])
return {
"tables": sorted(tables),
"audio_files": audio_count,
"size": format_size(db_path.stat().st_size),
}
finally:
conn.close()
except sqlite3.Error as exc:
logger.warning("Datenbank konnte nicht gelesen werden: %s", exc)
return None
+406
View File
@@ -0,0 +1,406 @@
"""GPU-, CUDA- und CTranslate2-Diagnose.
Die GTX 1070 Ti ist eine Pascal-Karte (Compute Capability 6.1). Sie besitzt
keine FP16-Tensor-Cores. Deshalb wird FP16/BF16 hier grundsaetzlich nicht als
GPU-Rechenmodus vorausgesetzt; bevorzugt wird ``int8_float32``.
"""
from __future__ import annotations
import ctypes.util
import glob
import os
import re
import subprocess
from dataclasses import dataclass, field
from pathlib import Path
from ..utils.logging import get_logger
logger = get_logger("system.gpu")
# Pascal und aelter: kein FP16-Tensor-Betrieb auf Consumer-Karten.
PASCAL_CONSUMER_MAX_CC = 6.2
PREFERRED_COMPUTE_TYPE = "int8_float32"
FORBIDDEN_GPU_COMPUTE_TYPES = ("float16", "bfloat16", "int8_bfloat16", "int8_float16")
@dataclass
class GpuInfo:
"""Angaben aus ``nvidia-smi``."""
name: str
driver_version: str | None = None
cuda_driver_version: str | None = None
memory_total_mb: int | None = None
compute_capability: str | None = None
@dataclass
class RuntimeInfo:
"""Laufzeitinformationen zu CTranslate2, CUDA und SQLite."""
ctranslate2_version: str | None = None
cuda_device_count: int = 0
supported_compute_types: dict[str, list[str]] = field(default_factory=dict)
cuda_libraries: list[str] = field(default_factory=list)
cudnn_libraries: list[str] = field(default_factory=list)
python_version: str = ""
platform: str = ""
sqlite_version: str = ""
fts5_available: bool = False
trigram_available: bool = False
@property
def cuda_runtime_usable(self) -> bool:
return self.cuda_device_count > 0 and bool(self.supported_compute_types.get("cuda"))
@property
def cuda_compute_types(self) -> list[str]:
return sorted(self.supported_compute_types.get("cuda", []))
@property
def cpu_compute_types(self) -> list[str]:
return sorted(self.supported_compute_types.get("cpu", []))
@dataclass
class ResolvedExecution:
"""Tatsaechlich verwendeter Ausfuehrungsmodus."""
device: str
compute_type: str
requested_device: str
requested_compute_type: str
notes: list[str] = field(default_factory=list)
@property
def used_fallback(self) -> bool:
return self.device != self.requested_device or self.compute_type != self.requested_compute_type
def _run_nvidia_smi(args: list[str]) -> str | None:
try:
completed = subprocess.run(
["nvidia-smi", *args],
capture_output=True,
text=True,
timeout=15,
check=False,
)
except (OSError, subprocess.SubprocessError) as exc:
logger.debug("nvidia-smi nicht ausfuehrbar: %s", exc)
return None
if completed.returncode != 0:
return None
return completed.stdout
def detect_gpu() -> GpuInfo | None:
"""Ermittelt GPU-Modell, Treiber und Compute Capability."""
output = _run_nvidia_smi(
[
"--query-gpu=name,driver_version,memory.total,compute_cap",
"--format=csv,noheader",
]
)
gpu: GpuInfo | None = None
if output:
first_line = output.strip().splitlines()[0] if output.strip() else ""
parts = [part.strip() for part in first_line.split(",")]
if len(parts) >= 4:
memory_mb = None
match = re.search(r"(\d+)", parts[2])
if match:
memory_mb = int(match.group(1))
gpu = GpuInfo(
name=parts[0],
driver_version=parts[1] or None,
memory_total_mb=memory_mb,
compute_capability=parts[3] or None,
)
if gpu is None:
return None
header = _run_nvidia_smi([]) or ""
match = re.search(r"CUDA Version:\s*([0-9.]+)", header)
if match:
gpu.cuda_driver_version = match.group(1)
return gpu
_PRELOADED_LIBRARIES: list[str] = []
def ensure_cuda_libraries() -> list[str]:
"""Laedt CUDA-/cuDNN-Bibliotheken aus den pip-``nvidia``-Paketen.
CTranslate2 linkt dynamisch gegen ``libcublas.so.12`` und ``libcudnn.so.9``.
Die PyPI-Wheels legen diese Bibliotheken unter ``site-packages/nvidia``
ab, wo der dynamische Linker sie nicht von allein findet. Ein Preload mit
``RTLD_GLOBAL`` macht sie verfuegbar, ohne systemweite Aenderungen an
``/etc/ld.so.conf`` oder installierte CUDA-Versionen.
Muss vor dem ersten CUDA-Aufruf von CTranslate2 erfolgen.
"""
global _PRELOADED_LIBRARIES
if _PRELOADED_LIBRARIES or os.name != "posix":
return _PRELOADED_LIBRARIES
cuda_libs, cudnn_libs = detect_cuda_libraries()
if not cuda_libs and not cudnn_libs:
return _PRELOADED_LIBRARIES
order = ("libcudart", "libcublasLt", "libcublas", "libcudnn")
def sort_key(path: str) -> int:
name = Path(path).name
for index, prefix in enumerate(order):
if name.startswith(prefix):
return index
return len(order)
import ctypes
loaded: list[str] = []
for path in sorted(cuda_libs + cudnn_libs, key=sort_key):
# cuDNN-Teilbibliotheken werden von libcudnn.so.9 selbst nachgeladen.
name = Path(path).name
if name.startswith("libcudnn") and not name.startswith("libcudnn.so"):
continue
try:
ctypes.CDLL(path, mode=ctypes.RTLD_GLOBAL)
loaded.append(path)
except OSError as exc:
logger.debug("Bibliothek konnte nicht vorgeladen werden (%s): %s", path, exc)
if loaded:
directories = sorted({str(Path(path).parent) for path in loaded})
existing = os.environ.get("LD_LIBRARY_PATH", "")
os.environ["LD_LIBRARY_PATH"] = os.pathsep.join(
directories + ([existing] if existing else [])
)
logger.debug("CUDA-Bibliotheken vorgeladen: %s", ", ".join(Path(p).name for p in loaded))
_PRELOADED_LIBRARIES = loaded
return loaded
def _find_libraries(patterns: list[str], directories: list[str]) -> list[str]:
found: list[str] = []
for pattern in patterns:
for directory in directories:
try:
found.extend(glob.glob(os.path.join(directory, pattern)))
except OSError:
continue
return sorted(set(found))
def detect_cuda_libraries() -> tuple[list[str], list[str]]:
"""Sucht CUDA- und cuDNN-Bibliotheken im System und in der venv."""
directories = [
"/usr/lib",
"/usr/lib64",
"/usr/local/cuda/lib64",
"/usr/local/lib",
"/opt/cuda/lib64",
str(Path.cwd() / ".venv" / "lib"),
]
# nvidia-Wheels (falls installiert) liegen unter site-packages/nvidia/<paket>/lib
import site
import sys
site_roots: list[Path] = []
for base in (os.environ.get("VIRTUAL_ENV"), os.environ.get("CONDA_PREFIX")):
if base:
site_roots.append(Path(base))
site_roots.extend(Path(p) for p in site.getsitepackages())
site_roots.append(Path(sys.prefix))
for base in site_roots:
directories.extend([str(base / "lib"), str(base / "lib64")])
nvidia_root = base / "lib" / f"python{sys.version_info.major}.{sys.version_info.minor}" / "site-packages" / "nvidia"
directories.extend(str(path) for path in nvidia_root.glob("*/lib"))
directories.extend(str(path) for path in (base / "Lib" / "site-packages" / "nvidia").glob("*/bin"))
cuda_patterns = ["libcudart.so*", "libcublas.so*", "libcublasLt.so*"]
cudnn_patterns = ["libcudnn*.so*"]
cuda_libs = _find_libraries(cuda_patterns, directories)
cudnn_libs = _find_libraries(cudnn_patterns, directories)
# Zusaetzlich ueber den dynamischen Linker suchen.
for name in ("cudart", "cublas", "cudnn"):
located = ctypes.util.find_library(name)
if located and located not in cuda_libs and name != "cudnn":
cuda_libs.append(located)
if located and name == "cudnn" and located not in cudnn_libs:
cudnn_libs.append(located)
return sorted(set(cuda_libs)), sorted(set(cudnn_libs))
def probe_ctranslate2() -> tuple[str | None, int, dict[str, list[str]], str | None]:
"""Fragt CTranslate2 nach Geraeten und unterstuetzten Compute Types."""
try:
import ctranslate2
except ImportError as exc:
return None, 0, {}, f"CTranslate2 nicht installiert: {exc}"
version = getattr(ctranslate2, "__version__", None)
error: str | None = None
try:
device_count = int(ctranslate2.get_cuda_device_count())
except Exception as exc: # pragma: no cover - hardwareabhaengig
device_count = 0
error = f"CUDA-Geraete konnten nicht abgefragt werden: {exc}"
supported: dict[str, list[str]] = {}
for device in ("cpu", "cuda"):
try:
supported[device] = sorted(ctranslate2.get_supported_compute_types(device))
except Exception as exc:
logger.debug("Compute Types fuer %s nicht verfuegbar: %s", device, exc)
supported[device] = []
return version, device_count, supported, error
def collect_runtime_info(*, check_fts: bool = True) -> RuntimeInfo:
"""Sammelt alle Laufzeitinformationen fuer den Systembericht."""
import platform
import sqlite3
import sys
version, device_count, supported, error = probe_ctranslate2()
cuda_libs, cudnn_libs = detect_cuda_libraries()
info = RuntimeInfo(
ctranslate2_version=version,
cuda_device_count=device_count,
supported_compute_types=supported,
cuda_libraries=cuda_libs,
cudnn_libraries=cudnn_libs,
python_version=sys.version.split()[0],
platform=f"{platform.system()} {platform.release()}",
sqlite_version=sqlite3.sqlite_version,
)
if error:
logger.debug(error)
if check_fts:
from ..database.connection import check_fts5, connect
conn = connect(":memory:")
try:
info.fts5_available, info.trigram_available = check_fts5(conn)
finally:
conn.close()
return info
def _cc_tuple(compute_capability: str | None) -> tuple[int, int] | None:
if not compute_capability:
return None
match = re.match(r"^\s*(\d+)\.(\d+)", compute_capability)
if not match:
return None
return int(match.group(1)), int(match.group(2))
def is_pascal_or_older(gpu: GpuInfo | None) -> bool:
"""True fuer Pascal und aeltere Architekturen (Compute Capability <= 6.x)."""
if gpu is None:
return False
cc = _cc_tuple(gpu.compute_capability)
if cc is None:
return False
return cc[0] <= 6
def resolve_execution(
requested_device: str,
requested_compute_type: str,
gpu: GpuInfo | None,
runtime: RuntimeInfo,
) -> ResolvedExecution:
"""Waehlt einen tatsaechlich unterstuetzten Ausfuehrungsmodus.
Regeln:
* Nie einen Compute Type verwenden, den CTranslate2 fuer das Geraet nicht meldet.
* Auf Pascal kein FP16/BF16.
* Wenn CUDA nicht nutzbar ist, automatisch CPU mit einer verstaendlichen Notiz.
"""
device = (requested_device or "auto").strip().lower()
requested_compute = (requested_compute_type or "auto").strip().lower()
notes: list[str] = []
if device in {"auto", ""}:
device = "cuda" if runtime.cuda_runtime_usable else "cpu"
notes.append(f"device=auto auf '{device}' aufgeloest.")
if device == "cuda" and not runtime.cuda_runtime_usable:
notes.append(
"CUDA ist nicht nutzbar (keine CUDA-Geraete oder keine CUDA-Compute-Types) - "
"es wird die CPU verwendet."
)
device = "cpu"
if device == "cuda" and gpu is None:
notes.append("nvidia-smi lieferte keine GPU-Informationen; CUDA wird trotzdem versucht.")
cc = _cc_tuple(gpu.compute_capability) if gpu else None
if device == "cuda" and cc is not None and cc[0] <= 6:
if requested_compute in FORBIDDEN_GPU_COMPUTE_TYPES:
notes.append(
f"'{requested_compute}' ist auf Pascal (CC {gpu.compute_capability}) nicht sinnvoll - "
f"stattdessen {PREFERRED_COMPUTE_TYPE}."
)
requested_compute = PREFERRED_COMPUTE_TYPE
if device == "cpu":
supported = runtime.cpu_compute_types or ["int8", "float32"]
preference = [requested_compute, "int8", "int8_float32", "float32"]
notes.append("CPU-Fallback aktiv: Transkription laeuft ohne GPU und ist deutlich langsamer.")
else:
supported = runtime.cuda_compute_types or []
if gpu is not None and gpu.compute_capability:
supported = [item for item in supported]
# Pascal: FP16/BF16 nie automatisch waehlen.
if cc is not None and cc[0] <= 6:
supported = [item for item in supported if item not in FORBIDDEN_GPU_COMPUTE_TYPES]
preference = [requested_compute, PREFERRED_COMPUTE_TYPE, "float32", "int8"]
chosen = ""
for candidate in preference:
if candidate in {"", "auto"}:
continue
if candidate in supported:
chosen = candidate
break
if not chosen:
# Letzter Ausweg: erster gemeldeter Typ.
chosen = supported[0] if supported else "float32"
notes.append(
f"Kein bevorzugter Compute Type verfuegbar; verwende '{chosen}'."
)
elif chosen != requested_compute and requested_compute not in {"", "auto"}:
notes.append(
f"'{requested_compute}' ist nicht verfuegbar - verwende '{chosen}'."
)
if device == "cuda" and chosen in FORBIDDEN_GPU_COMPUTE_TYPES and cc is not None and cc[0] <= 6:
chosen = PREFERRED_COMPUTE_TYPE
notes.append("FP16/BF16 auf Pascal verhindert, int8_float32 erzwungen.")
return ResolvedExecution(
device=device,
compute_type=chosen,
requested_device=requested_device,
requested_compute_type=requested_compute_type,
notes=notes,
)
+179
View File
@@ -0,0 +1,179 @@
"""faster-whisper-Engine.
Das Modell wird genau einmal geladen und danach fuer alle Dateien
wiederverwendet. Die Audiodateien werden von faster-whisper gestreamt
dekodiert, es wird nie eine komplette Sammlung in den RAM geladen.
"""
from __future__ import annotations
import math
from pathlib import Path
from typing import Any
from ..config import TranscriptionConfig
from ..utils.logging import get_logger
from .models import TranscriptResult, TranscriptSegment, WordToken
logger = get_logger("transcription.engine")
class TranscriptionError(RuntimeError):
"""Fehler beim Laden des Modells oder beim Transkribieren."""
class FasterWhisperEngine:
"""Duenner Wrapper um :class:`faster_whisper.WhisperModel`."""
def __init__(
self,
settings: TranscriptionConfig,
*,
device: str | None = None,
compute_type: str | None = None,
model_dir: str | Path | None = None,
):
self.settings = settings
self.device = (device or settings.device).strip().lower()
self.compute_type = (compute_type or settings.compute_type).strip().lower()
self.model_dir = Path(model_dir) if model_dir else None
self._model: Any | None = None
# ------------------------------------------------------------------
@property
def model_name(self) -> str:
return self.settings.model
@property
def is_loaded(self) -> bool:
return self._model is not None
@property
def language(self) -> str | None:
return self.settings.language_for_whisper
# ------------------------------------------------------------------
def load(self) -> None:
"""Laedt das Modell (idempotent)."""
if self._model is not None:
return
# CUDA-Bibliotheken aus den pip-nvidia-Paketen verfuegbar machen, bevor
# CTranslate2 sie laedt (sonst: "libcublas.so.12 is not found").
if self.device == "cuda":
from ..system.gpu import ensure_cuda_libraries
ensure_cuda_libraries()
try:
from faster_whisper import WhisperModel
except ImportError as exc: # pragma: no cover - Installationsthema
raise TranscriptionError(
"faster-whisper ist nicht installiert. Bitte die Projektabhaengigkeiten installieren."
) from exc
download_root = str(self.model_dir) if self.model_dir else None
if download_root:
Path(download_root).mkdir(parents=True, exist_ok=True)
logger.info(
"Lade Whisper-Modell %s (device=%s, compute_type=%s)",
self.model_name,
self.device,
self.compute_type,
)
try:
self._model = WhisperModel(
self.model_name,
device=self.device,
compute_type=self.compute_type,
download_root=download_root,
)
except Exception as exc: # CTranslate2 wirft diverse RuntimeError
raise TranscriptionError(
f"Modell '{self.model_name}' konnte nicht geladen werden "
f"(device={self.device}, compute_type={self.compute_type}): {exc}"
) from exc
def unload(self) -> None:
self._model = None
# ------------------------------------------------------------------
def transcribe(self, path: str | Path) -> TranscriptResult:
"""Transkribiert eine Audiodatei vollstaendig."""
if self._model is None:
self.load()
assert self._model is not None
settings = self.settings
transcribe_kwargs: dict[str, Any] = {
"language": settings.language_for_whisper,
"beam_size": max(1, int(settings.beam_size)),
"word_timestamps": bool(settings.word_timestamps),
"vad_filter": bool(settings.vad_filter),
# Dialogpassagen nicht durch aggressive Stille-Erkennung abschneiden.
"condition_on_previous_text": False,
}
if settings.vad_filter:
transcribe_kwargs["vad_parameters"] = {
"min_silence_duration_ms": 700,
"speech_pad_ms": 300,
}
try:
segments_iter, info = self._model.transcribe(str(path), **transcribe_kwargs)
except Exception as exc:
raise TranscriptionError(f"Transkription fehlgeschlagen: {exc}") from exc
segments: list[TranscriptSegment] = []
text_parts: list[str] = []
try:
for segment in segments_iter:
words: list[WordToken] = []
for word in getattr(segment, "words", None) or []:
words.append(
WordToken(
word=word.word,
start=float(word.start),
end=float(word.end),
probability=getattr(word, "probability", None),
)
)
confidence = None
avg_logprob = getattr(segment, "avg_logprob", None)
if avg_logprob is not None:
# Nur als interne Gueteinformation, keine Wahrscheinlichkeit
# im Sinne der Suche.
try:
confidence = float(math.exp(float(avg_logprob)))
except (OverflowError, ValueError):
confidence = None
text = (segment.text or "").strip()
if text:
text_parts.append(text)
segments.append(
TranscriptSegment(
text=text,
start=float(segment.start),
end=float(segment.end),
confidence=confidence,
words=words,
)
)
except Exception as exc:
raise TranscriptionError(f"Fehler beim Lesen der Segmente: {exc}") from exc
full_text = " ".join(part for part in text_parts if part).strip()
duration = getattr(info, "duration", None)
language = getattr(info, "language", None) or settings.language_for_whisper
return TranscriptResult(
segments=segments,
full_text=full_text,
language=language,
language_probability=getattr(info, "language_probability", None),
duration=float(duration) if duration else None,
model=self.model_name,
)
+41
View File
@@ -0,0 +1,41 @@
"""Datenmodelle fuer Transkriptionsergebnisse."""
from __future__ import annotations
from dataclasses import dataclass, field
@dataclass
class WordToken:
"""Ein Wort mit Zeitstempel aus Whisper."""
word: str
start: float
end: float
probability: float | None = None
@dataclass
class TranscriptSegment:
"""Ein Whisper-Segment."""
text: str
start: float
end: float
confidence: float | None = None
words: list[WordToken] = field(default_factory=list)
@dataclass
class TranscriptResult:
"""Vollstaendiges Transkript einer Audiodatei."""
segments: list[TranscriptSegment]
full_text: str
language: str | None = None
language_probability: float | None = None
duration: float | None = None
model: str = ""
def __len__(self) -> int:
return len(self.segments)
+204
View File
@@ -0,0 +1,204 @@
"""Transkriptionswarteschlange.
Eine Datei nach der anderen, Modell nur einmal geladen, Fehler einer Datei
brechen den Lauf nicht ab. Der Status wird dauerhaft gespeichert, sodass ein
Neustart an der letzten Stelle fortsetzt.
"""
from __future__ import annotations
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Callable, Protocol
from ..config import ProcessingConfig
from ..core.models import AudioFileRecord, FileStatus
from ..database.repository import Repository
from ..utils.logging import get_logger
from .models import TranscriptResult
logger = get_logger("transcription.worker")
class EngineProtocol(Protocol):
"""Minimal benoetigte Engine-Schnittstelle (fuer Tests austauschbar)."""
model_name: str
def transcribe(self, path: str | Path) -> TranscriptResult: ...
@dataclass
class WorkerEvent:
"""Ereignis fuer die Fortschrittsanzeige."""
kind: str # start | done | error | skipped | finished
record: AudioFileRecord | None = None
message: str | None = None
elapsed: float = 0.0
audio_duration: float | None = None
stats: "RunStats | None" = None
@dataclass
class RunStats:
"""Zusammenfassung eines Transkriptionslaufs."""
queued: int = 0
processed: int = 0
failed: int = 0
skipped: int = 0
audio_seconds: float = 0.0
elapsed: float = 0.0
model: str = ""
device: str = ""
compute_type: str = ""
errors: list[tuple[str, str]] = field(default_factory=list)
@property
def rate(self) -> float:
if self.elapsed <= 0:
return 0.0
return self.audio_seconds / self.elapsed
class TranscriptionWorker:
"""Verarbeitet die Transkriptionswarteschlange."""
def __init__(
self,
repository: Repository,
engine: EngineProtocol,
processing: ProcessingConfig | None = None,
*,
device: str = "",
compute_type: str = "",
):
self.repo = repository
self.engine = engine
self.processing = processing or ProcessingConfig()
self.stats = RunStats(
model=getattr(engine, "model_name", ""),
device=device,
compute_type=compute_type,
)
# ------------------------------------------------------------------
def run(
self,
*,
limit: int | None = None,
force: bool = False,
event_callback: Callable[[WorkerEvent], None] | None = None,
) -> RunStats:
"""Arbeitet die Warteschlange ab."""
if self.processing.resume:
reset = self.repo.reset_stale_processing()
if reset:
logger.info("%d unterbrochene Eintraege zurueckgesetzt", reset)
queue = self.repo.transcription_queue(force=force)
if limit is not None:
queue = queue[: max(0, int(limit))]
self.stats.queued = len(queue)
started_at = time.monotonic()
try:
for record in queue:
self._process(record, event_callback)
except KeyboardInterrupt:
logger.warning("Transkription durch Benutzer abgebrochen")
raise
finally:
self.stats.elapsed = time.monotonic() - started_at
if event_callback:
event_callback(WorkerEvent(kind="finished", stats=self.stats))
logger.info(
"Transkription beendet: %d verarbeitet, %d Fehler, %d uebersprungen, "
"%.1f s Audio in %.1f s (%.2fx Echtzeit)",
self.stats.processed,
self.stats.failed,
self.stats.skipped,
self.stats.audio_seconds,
self.stats.elapsed,
self.stats.rate,
)
return self.stats
# ------------------------------------------------------------------
def _process(
self, record: AudioFileRecord, event_callback: Callable[[WorkerEvent], None] | None
) -> None:
path = Path(record.path)
if not path.is_file():
message = f"Datei nicht gefunden: {record.path}"
self.repo.set_status(record.id, FileStatus.ERROR.value, message)
self.stats.failed += 1
self.stats.errors.append((record.path, message))
if event_callback:
event_callback(WorkerEvent(kind="error", record=record, message=message))
return
self.repo.set_status(record.id, FileStatus.PROCESSING.value)
if event_callback:
event_callback(WorkerEvent(kind="start", record=record))
file_started = time.monotonic()
try:
result = self.engine.transcribe(path)
if not result.segments:
raise ValueError("Keine Sprache im Transkript erkannt")
self.repo.save_transcript(
record.id,
language=result.language,
model=result.model or self.engine.model_name,
full_text=result.full_text,
segments=result.segments,
duration=result.duration or record.duration,
)
except KeyboardInterrupt:
self.repo.set_status(record.id, FileStatus.PENDING.value)
raise
except Exception as exc:
elapsed = time.monotonic() - file_started
message = f"{type(exc).__name__}: {exc}"
logger.exception("Transkription fehlgeschlagen fuer %s", record.path)
self.repo.set_status(record.id, FileStatus.ERROR.value, message)
self.stats.failed += 1
self.stats.errors.append((record.path, message))
if event_callback:
event_callback(
WorkerEvent(
kind="error",
record=record,
message=message,
elapsed=elapsed,
audio_duration=record.duration,
)
)
return
elapsed = time.monotonic() - file_started
audio_duration = result.duration or record.duration or 0.0
self.stats.processed += 1
self.stats.audio_seconds += float(audio_duration or 0.0)
logger.info(
"Fertig: %s (%.1f s Audio, %.1f s Laufzeit, %d Segmente)",
record.filename,
audio_duration or 0.0,
elapsed,
len(result.segments),
)
if event_callback:
event_callback(
WorkerEvent(
kind="done",
record=record,
elapsed=elapsed,
audio_duration=audio_duration,
)
)
+95
View File
@@ -0,0 +1,95 @@
"""Formatierungshelfer fuer Zeitangaben, Groessen und Pfade."""
from __future__ import annotations
from datetime import datetime, timezone
from pathlib import Path
def format_timestamp(seconds: float | int | None) -> str:
"""Formatiert Sekunden als HH:MM:SS bzw. MM:SS."""
if seconds is None:
return "--:--"
seconds = max(0, int(round(float(seconds))))
hours, remainder = divmod(seconds, 3600)
minutes, secs = divmod(remainder, 60)
if hours:
return f"{hours:02d}:{minutes:02d}:{secs:02d}"
return f"{minutes:02d}:{secs:02d}"
def format_timestamp_verbose(seconds: float | int | None) -> str:
"""Formatiert Sekunden immer als HH:MM:SS (fuer Fundstellen in Hoerspielen)."""
if seconds is None:
return "--:--:--"
seconds = max(0, int(round(float(seconds))))
hours, remainder = divmod(seconds, 3600)
minutes, secs = divmod(remainder, 60)
return f"{hours:02d}:{minutes:02d}:{secs:02d}"
def format_duration(seconds: float | int | None) -> str:
"""Menschlich lesbare Dauer, z. B. '2 h 14 min'."""
if seconds is None:
return "unbekannt"
total = max(0, int(round(float(seconds))))
hours, remainder = divmod(total, 3600)
minutes, secs = divmod(remainder, 60)
if hours:
return f"{hours} h {minutes:02d} min"
if minutes:
return f"{minutes} min {secs:02d} s"
return f"{secs} s"
def format_size(num_bytes: int | float | None) -> str:
"""Formatiert Bytes als B/KB/MB/GB."""
if num_bytes is None:
return "?"
value = float(num_bytes)
for unit in ("B", "KB", "MB", "GB", "TB"):
if value < 1024 or unit == "TB":
if unit == "B":
return f"{int(value)} {unit}"
return f"{value:.1f} {unit}"
value /= 1024
return f"{value:.1f} TB"
def format_score(score: float) -> str:
"""Aehnlichkeitswert als Prozentangabe (keine statistische Wahrscheinlichkeit)."""
return f"{score:.1f}%"
def format_rate(audio_seconds: float, elapsed_seconds: float) -> str:
"""Verarbeitungsgeschwindigkeit als Vielfaches der Echtzeit."""
if elapsed_seconds <= 0:
return "-"
return f"{audio_seconds / elapsed_seconds:.2f}x Echtzeit"
def format_datetime(iso_timestamp: str | None) -> str:
"""Formatiert einen ISO-String lesbar; unbekannte Formate werden durchgereicht."""
if not iso_timestamp:
return "-"
try:
dt = datetime.fromisoformat(iso_timestamp)
except ValueError:
return iso_timestamp
if dt.tzinfo is not None:
dt = dt.astimezone()
return dt.strftime("%Y-%m-%d %H:%M")
def utc_now_iso() -> str:
"""Aktueller Zeitpunkt als ISO-String in UTC (sekundengenau)."""
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
def shorten_path(path: str | Path, max_length: int = 80) -> str:
"""Kuerzt lange Pfade in der Mitte."""
text = str(path)
if len(text) <= max_length:
return text
keep = max(8, (max_length - 3) // 2)
return f"{text[:keep]}...{text[-keep:]}"
+38
View File
@@ -0,0 +1,38 @@
"""Hash-Helfer fuer die Duplikaterkennung."""
from __future__ import annotations
import hashlib
from pathlib import Path
CHUNK_SIZE = 1024 * 1024
def sha256_file(path: str | Path, chunk_size: int = CHUNK_SIZE) -> str:
"""Berechnet den SHA256-Hash einer Datei, ohne sie komplett in den RAM zu laden."""
digest = hashlib.sha256()
with Path(path).open("rb") as handle:
while True:
chunk = handle.read(chunk_size)
if not chunk:
break
digest.update(chunk)
return digest.hexdigest()
def quick_signature(path: str | Path, sample_size: int = 64 * 1024) -> str:
"""Schnelle Signatur aus Groesse, erster und letzter Datenportion.
Nur fuer Vorpruefungen gedacht - fuer belastbare Duplikaterkennung muss
:func:`sha256_file` verwendet werden.
"""
path = Path(path)
size = path.stat().st_size
digest = hashlib.sha256()
digest.update(str(size).encode("ascii"))
with path.open("rb") as handle:
digest.update(handle.read(sample_size))
if size > sample_size:
handle.seek(max(sample_size, size - sample_size))
digest.update(handle.read(sample_size))
return digest.hexdigest()
+62
View File
@@ -0,0 +1,62 @@
"""Logging-Setup fuer SjAudioSearch.
Protokolliert in eine rotierende Logdatei und optional zusaetzlich auf die
Konsole (Rich). Die CLI schaltet den Konsolen-Handler nur bei Bedarf ein.
"""
from __future__ import annotations
import logging
from logging.handlers import RotatingFileHandler
from pathlib import Path
from rich.logging import RichHandler
LOGGER_NAME = "sjaudiosearch"
_CONFIGURED = False
def setup_logging(log_path: Path, level: str = "INFO", console: bool = False) -> logging.Logger:
"""Konfiguriert den Logger ``sjaudiosearch`` idempotent."""
global _CONFIGURED
logger = logging.getLogger(LOGGER_NAME)
numeric_level = getattr(logging, str(level).upper(), logging.INFO)
logger.setLevel(numeric_level)
if _CONFIGURED:
return logger
log_path = Path(log_path)
log_path.parent.mkdir(parents=True, exist_ok=True)
formatter = logging.Formatter(
"%(asctime)s | %(levelname)-8s | %(name)s | %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
file_handler = RotatingFileHandler(
log_path,
maxBytes=5 * 1024 * 1024,
backupCount=3,
encoding="utf-8",
)
file_handler.setFormatter(formatter)
file_handler.setLevel(numeric_level)
logger.addHandler(file_handler)
if console:
console_handler = RichHandler(show_path=False, rich_tracebacks=True, markup=False)
console_handler.setLevel(max(numeric_level, logging.INFO))
logger.addHandler(console_handler)
logger.propagate = False
_CONFIGURED = True
return logger
def get_logger(name: str | None = None) -> logging.Logger:
"""Liefert einen Sub-Logger unterhalb von ``sjaudiosearch``."""
if name:
return logging.getLogger(f"{LOGGER_NAME}.{name}")
return logging.getLogger(LOGGER_NAME)