Initial commit
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
"""SjAudioSearch - lokale Suchmaschine fuer transkribierte Audiosammlungen."""
|
||||
|
||||
__version__ = "1.0.0"
|
||||
|
||||
__all__ = ["__version__"]
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Ermoeglicht ``python -m sjaudiosearch``."""
|
||||
|
||||
from .cli import app
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
@@ -0,0 +1,715 @@
|
||||
"""Kommandozeilenschnittstelle von SjAudioSearch."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import typer
|
||||
from rich.console import Console
|
||||
from rich.markup import escape
|
||||
from rich.panel import Panel
|
||||
from rich.progress import (
|
||||
BarColumn,
|
||||
MofNCompleteColumn,
|
||||
Progress,
|
||||
SpinnerColumn,
|
||||
TextColumn,
|
||||
TimeElapsedColumn,
|
||||
)
|
||||
from rich.table import Table
|
||||
|
||||
from . import __version__
|
||||
from .config import Config, load_config
|
||||
from .core.audio import PlaybackError, play as play_audio
|
||||
from .core.metadata import AudioMetadataError, read_metadata
|
||||
from .core.models import FileStatus
|
||||
from .core.scanner import scan as run_scan
|
||||
from .database.connection import check_fts5
|
||||
from .database.repository import Repository
|
||||
from .search.engine import SearchEngine, SearchError
|
||||
from .system.diagnostics import collect_report, render_report
|
||||
from .system.gpu import collect_runtime_info, detect_gpu, resolve_execution
|
||||
from .transcription.engine import FasterWhisperEngine, TranscriptionError
|
||||
from .transcription.worker import RunStats, TranscriptionWorker, WorkerEvent
|
||||
from .utils.formatting import (
|
||||
format_datetime,
|
||||
format_duration,
|
||||
format_rate,
|
||||
format_score,
|
||||
format_size,
|
||||
format_timestamp_verbose,
|
||||
shorten_path,
|
||||
)
|
||||
from .utils.logging import get_logger, setup_logging
|
||||
|
||||
app = typer.Typer(
|
||||
add_completion=False,
|
||||
no_args_is_help=True,
|
||||
help="Lokale Suchmaschine fuer transkribierte Hoerspiele und Hoerbuecher.",
|
||||
)
|
||||
|
||||
console = Console()
|
||||
logger = get_logger("cli")
|
||||
|
||||
|
||||
@dataclass
|
||||
class AppContext:
|
||||
"""Globaler Zustand der CLI."""
|
||||
|
||||
config: Config
|
||||
console: Console
|
||||
_repository: Repository | None = None
|
||||
|
||||
def repository(self) -> Repository:
|
||||
if self._repository is None:
|
||||
self._repository = Repository.open(self.config.database_path)
|
||||
return self._repository
|
||||
|
||||
def close(self) -> None:
|
||||
if self._repository is not None:
|
||||
self._repository.close()
|
||||
self._repository = None
|
||||
|
||||
|
||||
def _fail(message: str, code: int = 1) -> None:
|
||||
console.print(f"[red]Fehler:[/red] {escape(message)}")
|
||||
raise typer.Exit(code)
|
||||
|
||||
|
||||
def _get_context(ctx: typer.Context) -> AppContext:
|
||||
if not isinstance(ctx.obj, AppContext):
|
||||
_fail("Interner Fehler: CLI-Kontext fehlt.")
|
||||
return ctx.obj
|
||||
|
||||
|
||||
@app.callback()
|
||||
def main(
|
||||
ctx: typer.Context,
|
||||
config: Optional[Path] = typer.Option(
|
||||
None, "--config", "-c", help="Pfad zur config.toml", show_default=False
|
||||
),
|
||||
version: bool = typer.Option(False, "--version", help="Version anzeigen", is_eager=True),
|
||||
) -> None:
|
||||
"""SjAudioSearch initialisieren."""
|
||||
if version:
|
||||
console.print(f"SjAudioSearch {__version__}")
|
||||
raise typer.Exit()
|
||||
|
||||
cfg = load_config(config)
|
||||
setup_logging(cfg.log_path, cfg.logging.level)
|
||||
app_context = AppContext(config=cfg, console=console)
|
||||
ctx.obj = app_context
|
||||
ctx.call_on_close(app_context.close)
|
||||
|
||||
for warning in cfg.load_warnings:
|
||||
console.print(f"[yellow]Hinweis:[/yellow] {escape(warning)}")
|
||||
for key in cfg.unknown_keys:
|
||||
console.print(f"[yellow]Hinweis:[/yellow] unbekannter Konfigurationsschluessel '{escape(key)}'")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# diagnose
|
||||
# ----------------------------------------------------------------------
|
||||
@app.command()
|
||||
def diagnose(
|
||||
ctx: typer.Context,
|
||||
device: Optional[str] = typer.Option(None, help="Zu pruefendes Geraet: cuda oder cpu"),
|
||||
compute_type: Optional[str] = typer.Option(None, "--compute-type", help="Zu pruefender Compute Type"),
|
||||
) -> None:
|
||||
"""Zeigt einen vollstaendigen Systembericht (GPU, CUDA, Compute Types)."""
|
||||
app_context = _get_context(ctx)
|
||||
report = collect_report(
|
||||
app_context.config,
|
||||
requested_device=device,
|
||||
requested_compute_type=compute_type,
|
||||
)
|
||||
render_report(report, app_context.console)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# scan
|
||||
# ----------------------------------------------------------------------
|
||||
@app.command()
|
||||
def scan(
|
||||
ctx: typer.Context,
|
||||
path: Path = typer.Argument(..., help="Ordner mit Audiodateien"),
|
||||
) -> None:
|
||||
"""Durchsucht einen Ordner rekursiv und nimmt neue Dateien in den Index auf."""
|
||||
app_context = _get_context(ctx)
|
||||
repository = app_context.repository()
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[bold]{task.description}"),
|
||||
BarColumn(),
|
||||
MofNCompleteColumn(),
|
||||
TimeElapsedColumn(),
|
||||
console=app_context.console,
|
||||
transient=True,
|
||||
) as progress:
|
||||
task = progress.add_task("Scanne Dateien", total=None)
|
||||
|
||||
def on_progress(current: int, current_path: Path) -> None:
|
||||
progress.update(
|
||||
task,
|
||||
total=max(current, progress.tasks[0].total or 0),
|
||||
description=f"Scanne {escape(current_path.name)}",
|
||||
)
|
||||
|
||||
try:
|
||||
result = run_scan(path, repository, app_context.config.processing, progress=on_progress)
|
||||
except (FileNotFoundError, NotADirectoryError) as exc:
|
||||
_fail(str(exc))
|
||||
|
||||
table = Table(title="Scan-Ergebnis", show_header=False, box=None)
|
||||
table.add_column(style="bold cyan")
|
||||
table.add_column()
|
||||
table.add_row("Ordner", result.root)
|
||||
table.add_row("Gefundene Audiodateien", str(result.files_found))
|
||||
table.add_row("Neu aufgenommen", str(result.added))
|
||||
table.add_row("Geaendert", str(result.changed))
|
||||
table.add_row("Unveraendert", str(result.unchanged))
|
||||
table.add_row("Duplikate", str(result.duplicates))
|
||||
table.add_row("Fehler", str(len(result.errors)))
|
||||
console.print(table)
|
||||
|
||||
if result.errors:
|
||||
error_table = Table(title="Dateien mit Problemen")
|
||||
error_table.add_column("Datei", overflow="fold")
|
||||
error_table.add_column("Grund", overflow="fold")
|
||||
for item in result.errors[:15]:
|
||||
error_table.add_row(shorten_path(item.path, 70), escape(item.message))
|
||||
console.print(error_table)
|
||||
console.print(
|
||||
"[yellow]Hinweis:[/yellow] Diese Dateien wurden mit Status 'error' gespeichert. "
|
||||
"Details stehen im Log."
|
||||
)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# transcribe
|
||||
# ----------------------------------------------------------------------
|
||||
def _resolve_transcription(ctx_app: AppContext, model: str | None, device: str | None,
|
||||
compute_type: str | None, language: str | None):
|
||||
config = ctx_app.config
|
||||
settings = config.transcription
|
||||
if model:
|
||||
settings = replace(settings, model=model)
|
||||
if language is not None:
|
||||
settings = replace(settings, language=language)
|
||||
|
||||
requested_device = device or settings.device
|
||||
requested_compute = compute_type or settings.compute_type
|
||||
gpu = detect_gpu()
|
||||
runtime = collect_runtime_info(check_fts=False)
|
||||
resolved = resolve_execution(requested_device, requested_compute, gpu, runtime)
|
||||
return settings, resolved, gpu, runtime
|
||||
|
||||
|
||||
@app.command()
|
||||
def transcribe(
|
||||
ctx: typer.Context,
|
||||
model: Optional[str] = typer.Option(None, help="Modell, z. B. large-v3, medium, small"),
|
||||
device: Optional[str] = typer.Option(None, help="cuda oder cpu"),
|
||||
compute_type: Optional[str] = typer.Option(None, "--compute-type", help="z. B. int8_float32, float32"),
|
||||
language: Optional[str] = typer.Option(None, help="Sprache, z. B. de oder auto"),
|
||||
limit: Optional[int] = typer.Option(None, help="Maximale Anzahl Dateien in diesem Lauf"),
|
||||
force: bool = typer.Option(False, "--force", help="Bereits transkribierte Dateien erneut verarbeiten"),
|
||||
) -> None:
|
||||
"""Arbeitet die Transkriptionswarteschlange ab."""
|
||||
app_context = _get_context(ctx)
|
||||
repository = app_context.repository()
|
||||
config = app_context.config
|
||||
|
||||
settings, resolved, gpu, runtime = _resolve_transcription(
|
||||
app_context, model, device, compute_type, language
|
||||
)
|
||||
|
||||
mode_table = Table(show_header=False, box=None, padding=(0, 1))
|
||||
mode_table.add_column(style="bold cyan", no_wrap=True)
|
||||
mode_table.add_column()
|
||||
mode_table.add_row("Modell", settings.model)
|
||||
mode_table.add_row("Geraet", f"{resolved.device}" + (" (GPU)" if resolved.device == "cuda" else " (CPU)"))
|
||||
mode_table.add_row("Compute Type", resolved.compute_type)
|
||||
mode_table.add_row("Sprache", settings.language_for_whisper or "automatisch")
|
||||
if gpu is not None:
|
||||
mode_table.add_row(
|
||||
"GPU", f"{gpu.name} (CC {gpu.compute_capability or '?'}, {gpu.memory_total_mb or '?'} MB)"
|
||||
)
|
||||
console.print(Panel(mode_table, title="Transkription", border_style="cyan"))
|
||||
|
||||
for note in resolved.notes:
|
||||
console.print(f"[yellow]Hinweis:[/yellow] {escape(note)}")
|
||||
if not runtime.cuda_runtime_usable and resolved.device == "cpu":
|
||||
console.print(
|
||||
"[yellow]Hinweis:[/yellow] CPU-Modus aktiv. Die Verarbeitung wird deutlich langsamer sein."
|
||||
)
|
||||
|
||||
queue = repository.transcription_queue(force=force)
|
||||
if limit is not None:
|
||||
queue = queue[: max(0, limit)]
|
||||
if not queue:
|
||||
console.print("Keine Dateien in der Warteschlange. Nutze zuerst 'sjaudiosearch scan <ordner>'.")
|
||||
return
|
||||
|
||||
engine = FasterWhisperEngine(
|
||||
settings,
|
||||
device=resolved.device,
|
||||
compute_type=resolved.compute_type,
|
||||
model_dir=config.model_path,
|
||||
)
|
||||
with console.status(f"Lade Modell {settings.model} ...", spinner="dots"):
|
||||
try:
|
||||
engine.load()
|
||||
except TranscriptionError as exc:
|
||||
if resolved.device == "cuda":
|
||||
console.print(f"[yellow]GPU-Modus fehlgeschlagen:[/yellow] {escape(str(exc))}")
|
||||
console.print("[yellow]Wechsle automatisch auf CPU (int8).[/yellow]")
|
||||
engine = FasterWhisperEngine(
|
||||
settings,
|
||||
device="cpu",
|
||||
compute_type="int8",
|
||||
model_dir=config.model_path,
|
||||
)
|
||||
resolved = replace(resolved, device="cpu", compute_type="int8")
|
||||
try:
|
||||
engine.load()
|
||||
except TranscriptionError as cpu_exc:
|
||||
_fail(f"Auch der CPU-Modus ist fehlgeschlagen: {cpu_exc}")
|
||||
else:
|
||||
_fail(str(exc))
|
||||
|
||||
worker = TranscriptionWorker(
|
||||
repository,
|
||||
engine,
|
||||
config.processing,
|
||||
device=resolved.device,
|
||||
compute_type=resolved.compute_type,
|
||||
)
|
||||
|
||||
columns = [
|
||||
SpinnerColumn(),
|
||||
TextColumn("[bold]{task.description}"),
|
||||
BarColumn(),
|
||||
MofNCompleteColumn(),
|
||||
TimeElapsedColumn(),
|
||||
TextColumn("{task.fields[rate]}"),
|
||||
]
|
||||
errors: list[tuple[str, str]] = []
|
||||
|
||||
with Progress(*columns, console=console) as progress:
|
||||
task = progress.add_task("Starte Transkription", total=len(queue), rate="")
|
||||
|
||||
def on_event(event: WorkerEvent) -> None:
|
||||
if event.kind == "start" and event.record:
|
||||
progress.update(
|
||||
task, description=escape(event.record.filename), rate=""
|
||||
)
|
||||
elif event.kind == "done":
|
||||
stats = worker.stats
|
||||
progress.update(
|
||||
task,
|
||||
advance=1,
|
||||
rate=format_rate(stats.audio_seconds, stats.elapsed) if stats.elapsed else "",
|
||||
)
|
||||
elif event.kind == "error" and event.record:
|
||||
errors.append((event.record.path, event.message or "unbekannter Fehler"))
|
||||
progress.update(task, advance=1, rate="[red]Fehler[/red]")
|
||||
|
||||
try:
|
||||
stats = worker.run(limit=limit, force=force, event_callback=on_event)
|
||||
except KeyboardInterrupt:
|
||||
console.print()
|
||||
console.print("[yellow]Abgebrochen.[/yellow] Der aktuelle Stand ist gespeichert, "
|
||||
"die Unterbrechung wird beim naechsten Lauf fortgesetzt.")
|
||||
raise typer.Exit(130)
|
||||
except TranscriptionError as exc:
|
||||
_fail(str(exc))
|
||||
|
||||
_render_run_summary(stats, settings.model, resolved.device, resolved.compute_type, errors)
|
||||
|
||||
|
||||
def _render_run_summary(
|
||||
stats: RunStats, model: str, device: str, compute_type: str, errors: list[tuple[str, str]]
|
||||
) -> None:
|
||||
table = Table(title="Transkription abgeschlossen", show_header=False, box=None)
|
||||
table.add_column(style="bold cyan", no_wrap=True)
|
||||
table.add_column()
|
||||
table.add_row("Modell", model)
|
||||
table.add_row("Ausfuehrung", f"{device} / {compute_type}")
|
||||
table.add_row("Warteschlange", str(stats.queued))
|
||||
table.add_row("Erfolgreich", str(stats.processed))
|
||||
table.add_row("Fehlgeschlagen", str(stats.failed))
|
||||
table.add_row("Verarbeitete Audiodauer", format_duration(stats.audio_seconds))
|
||||
table.add_row("Laufzeit", format_duration(stats.elapsed))
|
||||
table.add_row("Durchschnitt", f"{format_rate(stats.audio_seconds, stats.elapsed)}")
|
||||
console.print(table)
|
||||
|
||||
if errors:
|
||||
error_table = Table(title="Fehlerhafte Dateien")
|
||||
error_table.add_column("Datei", overflow="fold")
|
||||
error_table.add_column("Grund", overflow="fold")
|
||||
for path, message in errors[:15]:
|
||||
error_table.add_row(shorten_path(path, 70), escape(message))
|
||||
console.print(error_table)
|
||||
console.print("Mit [bold]sjaudiosearch retry[/bold] koennen fehlgeschlagene Dateien erneut versucht werden.")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# search
|
||||
# ----------------------------------------------------------------------
|
||||
@app.command()
|
||||
def search(
|
||||
ctx: typer.Context,
|
||||
query: str = typer.Argument(..., help="Gesuchter Text (auch ungenau)"),
|
||||
limit: Optional[int] = typer.Option(None, help="Maximale Anzahl Treffer"),
|
||||
min_score: Optional[float] = typer.Option(None, "--min-score", help="Mindestaehnlichkeit 0-100"),
|
||||
context_words: Optional[int] = typer.Option(None, "--context-words", help="Woerter vor/nach dem Treffer"),
|
||||
language: Optional[str] = typer.Option(None, help="Nur Transkripte dieser Sprache"),
|
||||
json_output: bool = typer.Option(False, "--json", help="Ergebnisse als JSON ausgeben"),
|
||||
verbose: bool = typer.Option(False, "--verbose", help="Zusaetzliche Details anzeigen"),
|
||||
) -> None:
|
||||
"""Sucht in allen Transkripten - auch bei ungenauen Erinnerungen."""
|
||||
app_context = _get_context(ctx)
|
||||
repository = app_context.repository()
|
||||
config = app_context.config
|
||||
engine = SearchEngine(repository, config.search)
|
||||
|
||||
effective_limit = limit if limit is not None else config.search.default_limit
|
||||
effective_min = min_score if min_score is not None else config.search.default_min_score
|
||||
effective_context = context_words if context_words is not None else config.search.context_words
|
||||
|
||||
progress_state: dict[str, int] = {}
|
||||
|
||||
with console.status("[cyan]Suche ...[/cyan]", spinner="dots") as status:
|
||||
|
||||
def on_progress(current: int, total: int) -> None:
|
||||
progress_state["current"] = current
|
||||
progress_state["total"] = total
|
||||
status.update(f"[cyan]Suche ... {current}/{total} Dateien[/cyan]")
|
||||
|
||||
try:
|
||||
hits = engine.search(
|
||||
query,
|
||||
limit=effective_limit,
|
||||
min_score=effective_min,
|
||||
context_words=effective_context,
|
||||
language=language,
|
||||
progress=on_progress,
|
||||
)
|
||||
except SearchError as exc:
|
||||
_fail(str(exc))
|
||||
except sqlite3.Error as exc:
|
||||
_fail(f"Datenbankfehler bei der Suche: {exc}")
|
||||
|
||||
if json_output:
|
||||
payload = {
|
||||
"query": query,
|
||||
"limit": effective_limit,
|
||||
"min_score": effective_min,
|
||||
"context_words": effective_context,
|
||||
"count": len(hits),
|
||||
"results": [hit.to_dict(effective_context) for hit in hits],
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
return
|
||||
|
||||
console.print(Panel.fit("[bold]SjAudioSearch[/bold]", border_style="cyan"))
|
||||
console.print(f'Query: [bold]"{escape(query)}"[/bold]')
|
||||
console.print(
|
||||
f"[bold]{len(hits)}[/bold] Treffer (Mindestaehnlichkeit {effective_min:.0f}%, "
|
||||
f"Kontext {effective_context} Woerter)"
|
||||
)
|
||||
if not hits:
|
||||
console.print()
|
||||
console.print(
|
||||
"[yellow]Keine Treffer.[/yellow] Versuche eine hoehere Kontextlaenge, "
|
||||
"eine niedrigere Mindestaehnlichkeit oder ein anderes Modell."
|
||||
)
|
||||
|
||||
for index, hit in enumerate(hits, start=1):
|
||||
header = f"[bold][{index}] {escape(hit.filename)}[/bold]"
|
||||
details = [
|
||||
f"Similarity: [bold green]{format_score(hit.score)}[/bold green]",
|
||||
f"Timestamp: [bold]{format_timestamp_verbose(hit.start_time)}[/bold] "
|
||||
f"(bis {format_timestamp_verbose(hit.end_time)}, "
|
||||
f"{'Wort' if hit.context.precision == 'word' else 'Segment'}-Genauigkeit)",
|
||||
]
|
||||
if verbose:
|
||||
details.append(
|
||||
f"Audio-ID: {hit.audio_id} | Sprache: {hit.language or '?'} | Modell: {escape(hit.model or '?')} | "
|
||||
f"Quelle: {hit.source}"
|
||||
)
|
||||
body = "\n".join(details)
|
||||
console.print()
|
||||
console.print(header)
|
||||
console.print(body)
|
||||
console.print()
|
||||
console.print(Panel(hit.context.as_block(), title="Context", border_style="dim"))
|
||||
console.print(f"Path: {escape(hit.path)}")
|
||||
console.print("[dim]" + "-" * 40 + "[/dim]")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# status
|
||||
# ----------------------------------------------------------------------
|
||||
@app.command()
|
||||
def status(ctx: typer.Context) -> None:
|
||||
"""Zeigt Kennzahlen der Sammlung und den Datenbankzustand."""
|
||||
app_context = _get_context(ctx)
|
||||
repository = app_context.repository()
|
||||
stats = repository.stats(repository.databases_size())
|
||||
config = app_context.config
|
||||
|
||||
table = Table(title="SjAudioSearch Status", show_header=False, box=None)
|
||||
table.add_column(style="bold cyan", no_wrap=True)
|
||||
table.add_column()
|
||||
table.add_row("Audiodateien gesamt", str(stats.total_files))
|
||||
table.add_row("Transkribiert", str(stats.done))
|
||||
table.add_row("Ausstehend", str(stats.pending))
|
||||
table.add_row("In Bearbeitung", str(stats.processing))
|
||||
table.add_row("Fehlgeschlagen", str(stats.error))
|
||||
table.add_row("Duplikate", str(stats.duplicates))
|
||||
table.add_row("Gesamtspieldauer", format_duration(stats.total_duration))
|
||||
table.add_row("Transkripte", str(stats.transcripts))
|
||||
table.add_row("Segmente", str(stats.segment_count))
|
||||
table.add_row("Woerter", str(stats.word_count))
|
||||
table.add_row("Datenbank", f"{config.database_path} ({format_size(stats.database_size)})")
|
||||
model_text = ", ".join(stats.models) if stats.models else f"{config.transcription.model} (noch nicht verwendet)"
|
||||
table.add_row("Transkriptionsmodell", model_text)
|
||||
console.print(table)
|
||||
|
||||
if stats.error:
|
||||
console.print("[yellow]Hinweis:[/yellow] 'sjaudiosearch retry' setzt fehlgeschlagene Dateien zurueck.")
|
||||
if stats.pending or stats.processing:
|
||||
console.print("[yellow]Hinweis:[/yellow] 'sjaudiosearch transcribe' arbeitet die Warteschlange ab.")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# list
|
||||
# ----------------------------------------------------------------------
|
||||
@app.command("list")
|
||||
def list_files(
|
||||
ctx: typer.Context,
|
||||
limit: Optional[int] = typer.Option(None, help="Maximale Anzahl Eintraege"),
|
||||
status_filter: Optional[str] = typer.Option(
|
||||
None, "--status", help="pending, processing, done oder error"
|
||||
),
|
||||
search: Optional[str] = typer.Option(None, "--search", help="Dateiname/Pfad filtern"),
|
||||
) -> None:
|
||||
"""Listet bekannte Audiodateien auf."""
|
||||
app_context = _get_context(ctx)
|
||||
repository = app_context.repository()
|
||||
if status_filter and status_filter not in FileStatus.values():
|
||||
_fail(f"Unbekannter Status '{status_filter}'. Erlaubt: {', '.join(FileStatus.values())}")
|
||||
|
||||
records = repository.list_audio(status=status_filter, limit=limit, search=search)
|
||||
if not records:
|
||||
console.print("Keine Dateien gefunden. Nutze 'sjaudiosearch scan <ordner>'.")
|
||||
return
|
||||
|
||||
table = Table(title=f"Audiodateien ({len(records)})", show_lines=False)
|
||||
table.add_column("ID", justify="right", style="cyan")
|
||||
table.add_column("Status")
|
||||
table.add_column("Dauer", justify="right")
|
||||
table.add_column("Groesse", justify="right")
|
||||
table.add_column("Datei", overflow="fold")
|
||||
table.add_column("Pfad", overflow="fold")
|
||||
|
||||
colors = {
|
||||
FileStatus.DONE.value: "green",
|
||||
FileStatus.PENDING.value: "yellow",
|
||||
FileStatus.PROCESSING.value: "cyan",
|
||||
FileStatus.ERROR.value: "red",
|
||||
}
|
||||
for record in records:
|
||||
status_text = record.status
|
||||
if record.duplicate_of:
|
||||
status_text = f"duplikat von #{record.duplicate_of}"
|
||||
color = colors.get(record.status, "white")
|
||||
table.add_row(
|
||||
str(record.id),
|
||||
f"[{color}]{status_text}[/{color}]",
|
||||
format_duration(record.duration),
|
||||
format_size(record.file_size),
|
||||
escape(record.filename),
|
||||
escape(shorten_path(record.path, 70)),
|
||||
)
|
||||
console.print(table)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# info
|
||||
# ----------------------------------------------------------------------
|
||||
@app.command()
|
||||
def info(
|
||||
ctx: typer.Context,
|
||||
audio_id: int = typer.Argument(..., help="ID der Audiodatei"),
|
||||
) -> None:
|
||||
"""Zeigt Details zu einer Audiodatei und ihrem Transkript."""
|
||||
app_context = _get_context(ctx)
|
||||
repository = app_context.repository()
|
||||
record = repository.get_audio(audio_id)
|
||||
if record is None:
|
||||
_fail(f"Keine Audiodatei mit ID {audio_id} gefunden.")
|
||||
|
||||
table = Table(show_header=False, box=None)
|
||||
table.add_column(style="bold cyan", no_wrap=True)
|
||||
table.add_column(overflow="fold")
|
||||
table.add_row("ID", str(record.id))
|
||||
table.add_row("Dateiname", escape(record.filename))
|
||||
table.add_row("Pfad", escape(record.path))
|
||||
table.add_row("Status", record.status)
|
||||
table.add_row("Dauer", format_duration(record.duration))
|
||||
table.add_row("Groesse", format_size(record.file_size))
|
||||
table.add_row("Geaendert", format_datetime(record.updated_at))
|
||||
table.add_row("Index-Hash", record.content_hash or "-")
|
||||
if record.duplicate_of:
|
||||
table.add_row("Duplikat von", f"#{record.duplicate_of}")
|
||||
if record.error_message:
|
||||
table.add_row("Fehler", escape(record.error_message))
|
||||
|
||||
try:
|
||||
meta = read_metadata(record.path)
|
||||
if meta.tags:
|
||||
tags = ", ".join(f"{key}={value}" for key, value in list(meta.tags.items())[:6])
|
||||
table.add_row("Tags", escape(tags))
|
||||
except AudioMetadataError:
|
||||
pass
|
||||
|
||||
console.print(Panel(table, title=f"Audiodatei #{record.id}", border_style="cyan"))
|
||||
|
||||
transcript = repository.get_transcript(record.id)
|
||||
if transcript is None:
|
||||
console.print("[yellow]Kein Transkript vorhanden.[/yellow]")
|
||||
return
|
||||
|
||||
segments = repository.get_segments(record.id)
|
||||
console.print(
|
||||
f"Transkript: {len(segments)} Segmente, Modell [bold]{escape(transcript['model'])}[/bold], "
|
||||
f"Sprache {transcript['language'] or '?'}, erstellt {format_datetime(transcript['created_at'])}"
|
||||
)
|
||||
preview = Table(title="Segmente (Auszug)")
|
||||
preview.add_column("Start", justify="right")
|
||||
preview.add_column("Ende", justify="right")
|
||||
preview.add_column("Text", overflow="fold")
|
||||
for segment in segments[:20]:
|
||||
preview.add_row(
|
||||
format_timestamp_verbose(segment["start_time"]),
|
||||
format_timestamp_verbose(segment["end_time"]),
|
||||
escape(segment["text"]),
|
||||
)
|
||||
console.print(preview)
|
||||
if len(segments) > 20:
|
||||
console.print(f"[dim]... {len(segments) - 20} weitere Segmente[/dim]")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# play
|
||||
# ----------------------------------------------------------------------
|
||||
@app.command()
|
||||
def play(
|
||||
ctx: typer.Context,
|
||||
audio_id: int = typer.Argument(..., help="ID der Audiodatei"),
|
||||
time: float = typer.Option(0.0, "--time", help="Startzeit in Sekunden"),
|
||||
) -> None:
|
||||
"""Spielt eine Audiodatei ab einer Zeitposition ab."""
|
||||
app_context = _get_context(ctx)
|
||||
repository = app_context.repository()
|
||||
record = repository.get_audio(audio_id)
|
||||
if record is None:
|
||||
_fail(f"Keine Audiodatei mit ID {audio_id} gefunden.")
|
||||
|
||||
console.print(
|
||||
f"Spiele [bold]{escape(record.filename)}[/bold] ab {format_timestamp_verbose(time)} ..."
|
||||
)
|
||||
try:
|
||||
play_audio(record.path, time)
|
||||
except PlaybackError as exc:
|
||||
_fail(str(exc), code=127)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# retry / reindex / remove
|
||||
# ----------------------------------------------------------------------
|
||||
@app.command()
|
||||
def retry(ctx: typer.Context) -> None:
|
||||
"""Setzt fehlgeschlagene Transkriptionen zurueck auf 'pending'."""
|
||||
app_context = _get_context(ctx)
|
||||
repository = app_context.repository()
|
||||
count = repository.reset_errors()
|
||||
console.print(f"{count} Datei(en) zurueckgesetzt. Starte 'sjaudiosearch transcribe'.")
|
||||
|
||||
|
||||
@app.command()
|
||||
def reindex(ctx: typer.Context) -> None:
|
||||
"""Baut den Suchindex aus den vorhandenen Transkripten neu auf."""
|
||||
app_context = _get_context(ctx)
|
||||
repository = app_context.repository()
|
||||
before = repository.stats().transcripts
|
||||
with console.status("Baue Suchindex neu ...", spinner="dots"):
|
||||
indexed = repository.rebuild_search_index()
|
||||
after = repository.stats().transcripts
|
||||
if before != after:
|
||||
_fail(f"Transkriptanzahl hat sich geaendert ({before} -> {after}) - bitte pruefen.")
|
||||
console.print(
|
||||
f"Suchindex neu aufgebaut: {indexed} Segmente indexiert. "
|
||||
f"Transkripte unveraendert ({after})."
|
||||
)
|
||||
|
||||
|
||||
@app.command()
|
||||
def remove(
|
||||
ctx: typer.Context,
|
||||
audio_id: int = typer.Argument(..., help="ID der Audiodatei"),
|
||||
) -> None:
|
||||
"""Entfernt eine Datei aus dem Index (die Audiodatei selbst bleibt erhalten)."""
|
||||
app_context = _get_context(ctx)
|
||||
repository = app_context.repository()
|
||||
record = repository.get_audio(audio_id)
|
||||
if record is None:
|
||||
_fail(f"Keine Audiodatei mit ID {audio_id} gefunden.")
|
||||
|
||||
if not repository.delete_audio(audio_id):
|
||||
_fail(f"Datei #{audio_id} konnte nicht entfernt werden.")
|
||||
|
||||
console.print(
|
||||
f"[green]Entfernt:[/green] #{audio_id} {escape(record.filename)} "
|
||||
"(Transkript und Suchindex geloescht)"
|
||||
)
|
||||
console.print(f"[dim]Die Originaldatei bleibt erhalten: {escape(record.path)}[/dim]")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# search-index Diagnose
|
||||
# ----------------------------------------------------------------------
|
||||
@app.command("check")
|
||||
def check(ctx: typer.Context) -> None:
|
||||
"""Prueft Datenbank, FTS5 und Trigramm-Unterstuetzung."""
|
||||
app_context = _get_context(ctx)
|
||||
config = app_context.config
|
||||
conn = app_context.repository().conn
|
||||
fts5, trigram = check_fts5(conn)
|
||||
console.print(f"Datenbank: {config.database_path}")
|
||||
console.print(f"FTS5: {'OK' if fts5 else 'nicht verfuegbar'}")
|
||||
console.print(f"Trigramm-Tokenizer: {'OK' if trigram else 'nicht verfuegbar'}")
|
||||
console.print(f"Logdatei: {config.log_path}")
|
||||
console.print(f"Konfiguration: {config.source_path or 'eingebaute Defaults'}")
|
||||
|
||||
|
||||
def run() -> None:
|
||||
"""Einstiegspunkt fuer ``python -m sjaudiosearch``."""
|
||||
try:
|
||||
app()
|
||||
except KeyboardInterrupt:
|
||||
console.print("\n[yellow]Abgebrochen.[/yellow]")
|
||||
sys.exit(130)
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
run()
|
||||
@@ -0,0 +1,225 @@
|
||||
"""Konfigurationsverwaltung fuer SjAudioSearch.
|
||||
|
||||
Die Konfiguration wird aus einer TOML-Datei gelesen. Fehlende Werte werden
|
||||
durch Defaults ersetzt, unbekannte Schluessel werden gesammelt und gemeldet,
|
||||
statt stillschweigend ignoriert zu werden.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tomllib
|
||||
from dataclasses import dataclass, field, fields
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
DEFAULT_CONFIG_FILENAME = "config.toml"
|
||||
ENV_CONFIG_PATH = "SJAUDIOSEARCH_CONFIG"
|
||||
|
||||
|
||||
def _xdg_dir(env_var: str, fallback: str) -> Path:
|
||||
"""Ermittelt ein XDG-Verzeichnis (Konfiguration bzw. Daten)."""
|
||||
base = os.environ.get(env_var)
|
||||
root = Path(base).expanduser() if base else Path.home() / fallback
|
||||
return root / "sjaudiosearch"
|
||||
|
||||
|
||||
# ~/.config/sjaudiosearch/config.toml (bzw. $XDG_CONFIG_HOME/...)
|
||||
USER_CONFIG_DIR = _xdg_dir("XDG_CONFIG_HOME", ".config")
|
||||
# Ablage fuer Datenbank und Logs, wenn das Programm systemweit installiert ist.
|
||||
USER_DATA_DIR = _xdg_dir("XDG_DATA_HOME", ".local/share")
|
||||
SYSTEM_CONFIG_PATH = Path("/etc/sjaudiosearch/config.toml")
|
||||
|
||||
|
||||
@dataclass
|
||||
class TranscriptionConfig:
|
||||
model: str = "large-v3"
|
||||
device: str = "cuda"
|
||||
compute_type: str = "int8_float32"
|
||||
language: str | None = "de"
|
||||
beam_size: int = 5
|
||||
word_timestamps: bool = True
|
||||
vad_filter: bool = True
|
||||
model_dir: str = "./data/models"
|
||||
|
||||
@property
|
||||
def language_for_whisper(self) -> str | None:
|
||||
"""Gibt ``None`` zurueck, wenn die Sprache automatisch erkannt werden soll."""
|
||||
if self.language is None:
|
||||
return None
|
||||
value = self.language.strip().lower()
|
||||
if value in {"", "auto", "none"}:
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchConfig:
|
||||
default_limit: int = 50
|
||||
default_min_score: float = 70.0
|
||||
context_words: int = 50
|
||||
window_before: int = 2
|
||||
window_after: int = 4
|
||||
candidate_limit: int = 4000
|
||||
min_token_overlap: float = 0.34
|
||||
exhaustive_fallback: bool = True
|
||||
max_hits_per_file: int = 5
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProcessingConfig:
|
||||
workers: int = 1
|
||||
resume: bool = True
|
||||
compute_hash: str = "auto"
|
||||
hash_max_size_mb: int = 4096
|
||||
|
||||
|
||||
@dataclass
|
||||
class DatabaseConfig:
|
||||
path: str = "./data/sjaudiosearch.db"
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoggingConfig:
|
||||
level: str = "INFO"
|
||||
path: str = "./logs/sjaudiosearch.log"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
transcription: TranscriptionConfig = field(default_factory=TranscriptionConfig)
|
||||
search: SearchConfig = field(default_factory=SearchConfig)
|
||||
processing: ProcessingConfig = field(default_factory=ProcessingConfig)
|
||||
database: DatabaseConfig = field(default_factory=DatabaseConfig)
|
||||
logging: LoggingConfig = field(default_factory=LoggingConfig)
|
||||
|
||||
# Metadaten zur Herkunft der Konfiguration
|
||||
source_path: Path | None = None
|
||||
base_dir: Path = field(default_factory=lambda: PROJECT_ROOT)
|
||||
unknown_keys: list[str] = field(default_factory=list)
|
||||
load_warnings: list[str] = field(default_factory=list)
|
||||
|
||||
def resolve_path(self, value: str) -> Path:
|
||||
"""Loest einen (moeglicherweise relativen) Pfad gegen das Config-Verzeichnis auf."""
|
||||
path = Path(value).expanduser()
|
||||
if path.is_absolute():
|
||||
return path
|
||||
return (self.base_dir / path).resolve()
|
||||
|
||||
@property
|
||||
def database_path(self) -> Path:
|
||||
return self.resolve_path(self.database.path)
|
||||
|
||||
@property
|
||||
def log_path(self) -> Path:
|
||||
return self.resolve_path(self.logging.path)
|
||||
|
||||
@property
|
||||
def model_path(self) -> Path:
|
||||
return self.resolve_path(self.transcription.model_dir)
|
||||
|
||||
|
||||
def _coerce(current: Any, value: Any) -> Any:
|
||||
"""Konvertiert TOML-Werte tolerant auf den Typ des Defaults."""
|
||||
if current is None:
|
||||
return value
|
||||
if isinstance(current, bool):
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
return bool(value)
|
||||
if isinstance(current, int) and not isinstance(current, bool):
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return current
|
||||
if isinstance(current, float):
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return current
|
||||
if isinstance(current, str):
|
||||
return str(value)
|
||||
return value
|
||||
|
||||
|
||||
def _apply_section(target: Any, table: dict[str, Any], section: str, config: Config) -> None:
|
||||
known = {f.name: f for f in fields(target)}
|
||||
for key, value in table.items():
|
||||
if key not in known:
|
||||
config.unknown_keys.append(f"{section}.{key}")
|
||||
continue
|
||||
current = getattr(target, key)
|
||||
setattr(target, key, _coerce(current, value))
|
||||
|
||||
|
||||
def _is_writable_dir(path: Path) -> bool:
|
||||
try:
|
||||
return path.is_dir() and os.access(path, os.W_OK)
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def find_config_path(explicit: str | Path | None = None) -> Path | None:
|
||||
"""Ermittelt die zu verwendende Konfigurationsdatei."""
|
||||
candidates: list[Path] = []
|
||||
if explicit:
|
||||
candidates.append(Path(explicit).expanduser())
|
||||
env_value = os.environ.get(ENV_CONFIG_PATH)
|
||||
if env_value:
|
||||
candidates.append(Path(env_value).expanduser())
|
||||
candidates.append(Path.cwd() / DEFAULT_CONFIG_FILENAME)
|
||||
candidates.append(PROJECT_ROOT / DEFAULT_CONFIG_FILENAME)
|
||||
candidates.append(USER_CONFIG_DIR / DEFAULT_CONFIG_FILENAME)
|
||||
candidates.append(SYSTEM_CONFIG_PATH)
|
||||
|
||||
for candidate in candidates:
|
||||
if candidate.is_file():
|
||||
return candidate.resolve()
|
||||
return None
|
||||
|
||||
|
||||
def load_config(explicit: str | Path | None = None) -> Config:
|
||||
"""Laedt die Konfiguration; ohne Datei werden die Defaults verwendet."""
|
||||
path = find_config_path(explicit)
|
||||
config = Config()
|
||||
|
||||
if path is None:
|
||||
# Im Quellbaum (beschreibbar) bleiben Daten im Projekt, nach einer
|
||||
# systemweiten Installation liegen sie im Benutzerverzeichnis.
|
||||
config.base_dir = PROJECT_ROOT if _is_writable_dir(PROJECT_ROOT) else USER_DATA_DIR
|
||||
if explicit:
|
||||
config.load_warnings.append(
|
||||
f"Konfigurationsdatei nicht gefunden: {explicit} - es gelten die Defaults."
|
||||
)
|
||||
return config
|
||||
|
||||
config.source_path = path
|
||||
config.base_dir = path.parent
|
||||
|
||||
try:
|
||||
with path.open("rb") as handle:
|
||||
raw = tomllib.load(handle)
|
||||
except (OSError, tomllib.TOMLDecodeError) as exc:
|
||||
config.load_warnings.append(f"Konfiguration konnte nicht gelesen werden ({path}): {exc}")
|
||||
return config
|
||||
|
||||
section_map = {
|
||||
"transcription": config.transcription,
|
||||
"search": config.search,
|
||||
"processing": config.processing,
|
||||
"database": config.database,
|
||||
"logging": config.logging,
|
||||
}
|
||||
for section_name, section_obj in section_map.items():
|
||||
table = raw.get(section_name)
|
||||
if isinstance(table, dict):
|
||||
_apply_section(section_obj, table, section_name, config)
|
||||
|
||||
for key in raw:
|
||||
if key not in section_map:
|
||||
config.unknown_keys.append(key)
|
||||
|
||||
return config
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Audio-Wiedergabe und Dekodierpruefungen."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from .metadata import AudioMetadataError, read_metadata
|
||||
|
||||
|
||||
class PlaybackError(RuntimeError):
|
||||
"""Wiedergabe nicht moeglich."""
|
||||
|
||||
|
||||
def find_player() -> str | None:
|
||||
"""Sucht einen installierten Player (bevorzugt ffplay)."""
|
||||
for candidate in ("ffplay", "mpv", "vlc"):
|
||||
executable = shutil.which(candidate)
|
||||
if executable:
|
||||
return executable
|
||||
return None
|
||||
|
||||
|
||||
def build_play_command(path: str | Path, start_time: float = 0.0) -> list[str]:
|
||||
"""Baut den Kommandozeilenaufruf zum Abspielen ab einer Zeitposition."""
|
||||
player = find_player()
|
||||
if player is None:
|
||||
raise PlaybackError(
|
||||
"Kein Player gefunden. Installiere ffmpeg (ffplay), mpv oder vlc."
|
||||
)
|
||||
name = Path(player).name
|
||||
start = max(0.0, float(start_time))
|
||||
|
||||
if name.startswith("ffplay"):
|
||||
# -nodisp: reine Audiowiedergabe ohne zusaetzliches Fenster.
|
||||
command = [player, "-nodisp", "-autoexit", "-loglevel", "error"]
|
||||
if start > 0:
|
||||
command += ["-ss", f"{start:.3f}"]
|
||||
command.append(str(path))
|
||||
return command
|
||||
if name.startswith("mpv"):
|
||||
command = [player]
|
||||
if start > 0:
|
||||
command += [f"--start={start:.3f}"]
|
||||
command.append(str(path))
|
||||
return command
|
||||
# vlc
|
||||
command = [player]
|
||||
if start > 0:
|
||||
command += [f"--start-time={start:.3f}"]
|
||||
command.append(str(path))
|
||||
return command
|
||||
|
||||
|
||||
def play(path: str | Path, start_time: float = 0.0) -> None:
|
||||
"""Startet die Wiedergabe und wartet, bis der Player beendet wird."""
|
||||
file_path = Path(path)
|
||||
if not file_path.is_file():
|
||||
raise PlaybackError(f"Audiodatei nicht gefunden: {file_path}")
|
||||
command = build_play_command(file_path, start_time)
|
||||
try:
|
||||
subprocess.run(command, check=False)
|
||||
except OSError as exc:
|
||||
raise PlaybackError(f"Player konnte nicht gestartet werden: {exc}") from exc
|
||||
|
||||
|
||||
def can_decode(path: str | Path) -> bool:
|
||||
"""Prueft grob, ob die Datei als Audio dekodierbar ist."""
|
||||
try:
|
||||
read_metadata(path)
|
||||
return True
|
||||
except AudioMetadataError:
|
||||
return False
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Audio-Metadaten (Dauer, Tags, Format).
|
||||
|
||||
Primaer wird Mutagen verwendet; wenn Mutagen eine Datei nicht lesen kann, wird
|
||||
PyAV als Fallback genutzt. Beide Wege lesen nur Header bzw. Container, nicht
|
||||
die komplette Audiodatei.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger("core.metadata")
|
||||
|
||||
SUPPORTED_EXTENSIONS = frozenset(
|
||||
{".mp3", ".wav", ".flac", ".m4a", ".ogg", ".opus", ".wma"}
|
||||
)
|
||||
# WMA ist nur zuverlaessig dekodierbar, wenn ffmpeg/PyAV den Codec kennt.
|
||||
CONDITIONAL_EXTENSIONS = frozenset({".wma"})
|
||||
|
||||
|
||||
class AudioMetadataError(RuntimeError):
|
||||
"""Die Datei konnte nicht als Audio gelesen werden."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class AudioInfo:
|
||||
"""Ergebnis der Metadatenanalyse."""
|
||||
|
||||
path: str
|
||||
duration: float | None = None
|
||||
codec: str | None = None
|
||||
sample_rate: int | None = None
|
||||
channels: int | None = None
|
||||
bitrate: int | None = None
|
||||
tags: dict[str, str] = field(default_factory=dict)
|
||||
source: str = "mutagen"
|
||||
|
||||
|
||||
def is_supported_audio(path: str | Path) -> bool:
|
||||
return Path(path).suffix.lower() in SUPPORTED_EXTENSIONS
|
||||
|
||||
|
||||
def _clean_tags(raw: dict[str, Any]) -> dict[str, str]:
|
||||
tags: dict[str, str] = {}
|
||||
for key, value in raw.items():
|
||||
try:
|
||||
if isinstance(value, (list, tuple)):
|
||||
text = ", ".join(str(item) for item in value if item is not None)
|
||||
else:
|
||||
text = str(value)
|
||||
except Exception: # defekte Tag-Objekte duerfen den Scan nicht stoppen
|
||||
continue
|
||||
if text:
|
||||
tags[str(key).lower()] = text
|
||||
return tags
|
||||
|
||||
|
||||
def _read_with_mutagen(path: Path) -> AudioInfo:
|
||||
import mutagen
|
||||
|
||||
audio = mutagen.File(str(path))
|
||||
if audio is None:
|
||||
raise AudioMetadataError("Format wurde von Mutagen nicht erkannt")
|
||||
|
||||
info = getattr(audio, "info", None)
|
||||
duration = getattr(info, "length", None)
|
||||
if duration is None:
|
||||
raise AudioMetadataError("Mutagen konnte keine Dauer ermitteln")
|
||||
|
||||
return AudioInfo(
|
||||
path=str(path),
|
||||
duration=float(duration),
|
||||
codec=type(audio).__name__,
|
||||
sample_rate=getattr(info, "sample_rate", None),
|
||||
channels=getattr(info, "channels", None),
|
||||
bitrate=getattr(info, "bitrate", None),
|
||||
tags=_clean_tags(dict(audio.tags or {})),
|
||||
source="mutagen",
|
||||
)
|
||||
|
||||
|
||||
def _read_with_pyav(path: Path) -> AudioInfo:
|
||||
import av
|
||||
|
||||
try:
|
||||
with av.open(str(path)) as container:
|
||||
if not container.streams.audio:
|
||||
raise AudioMetadataError("Keine Audiospur gefunden")
|
||||
stream = container.streams.audio[0]
|
||||
duration = None
|
||||
if stream.duration is not None and stream.time_base is not None:
|
||||
duration = float(stream.duration * stream.time_base)
|
||||
elif container.duration is not None:
|
||||
duration = float(container.duration / 1_000_000)
|
||||
if duration is None or duration <= 0:
|
||||
raise AudioMetadataError("Keine gueltige Dauer ermittelbar")
|
||||
return AudioInfo(
|
||||
path=str(path),
|
||||
duration=duration,
|
||||
codec=stream.codec_context.name,
|
||||
sample_rate=stream.codec_context.sample_rate,
|
||||
channels=stream.codec_context.channels,
|
||||
bitrate=stream.bit_rate,
|
||||
source="pyav",
|
||||
)
|
||||
except AudioMetadataError:
|
||||
raise
|
||||
except Exception as exc: # PyAV wirft verschiedene Fehlertypen
|
||||
raise AudioMetadataError(f"PyAV konnte die Datei nicht lesen: {exc}") from exc
|
||||
|
||||
|
||||
def read_metadata(path: str | Path) -> AudioInfo:
|
||||
"""Liest Metadaten ueber Mutagen, bei Bedarf ueber PyAV."""
|
||||
file_path = Path(path)
|
||||
if not file_path.is_file():
|
||||
raise AudioMetadataError(f"Datei nicht gefunden: {file_path}")
|
||||
|
||||
errors: list[str] = []
|
||||
try:
|
||||
return _read_with_mutagen(file_path)
|
||||
except Exception as exc:
|
||||
errors.append(f"mutagen: {exc}")
|
||||
logger.debug("Mutagen fehlgeschlagen fuer %s: %s", file_path, exc)
|
||||
|
||||
try:
|
||||
return _read_with_pyav(file_path)
|
||||
except Exception as exc:
|
||||
errors.append(f"pyav: {exc}")
|
||||
logger.debug("PyAV fehlgeschlagen fuer %s: %s", file_path, exc)
|
||||
|
||||
raise AudioMetadataError("; ".join(errors))
|
||||
|
||||
|
||||
def read_duration(path: str | Path) -> float | None:
|
||||
"""Dauer einer Datei oder ``None``, wenn sie nicht ermittelbar ist."""
|
||||
try:
|
||||
return read_metadata(path).duration
|
||||
except AudioMetadataError:
|
||||
return None
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Datenmodelle fuer Audiodateien und Scan-Ergebnisse."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class FileStatus(str, Enum):
|
||||
"""Transkriptionsstatus einer Audiodatei."""
|
||||
|
||||
PENDING = "pending"
|
||||
PROCESSING = "processing"
|
||||
DONE = "done"
|
||||
ERROR = "error"
|
||||
|
||||
@classmethod
|
||||
def values(cls) -> tuple[str, ...]:
|
||||
return tuple(item.value for item in cls)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AudioFileRecord:
|
||||
"""Repraesentiert eine Zeile aus ``audio_files``."""
|
||||
|
||||
id: int
|
||||
path: str
|
||||
filename: str
|
||||
file_size: int
|
||||
modified_at: float
|
||||
status: str
|
||||
duration: float | None = None
|
||||
content_hash: str | None = None
|
||||
duplicate_of: int | None = None
|
||||
error_message: str | None = None
|
||||
created_at: str | None = None
|
||||
updated_at: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanCandidate:
|
||||
"""Beim Scan gefundene Datei (noch nicht in der Datenbank)."""
|
||||
|
||||
path: str
|
||||
filename: str
|
||||
file_size: int
|
||||
modified_at: float
|
||||
duration: float | None = None
|
||||
content_hash: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanError:
|
||||
"""Fehler, der beim Einlesen einer Datei aufgetreten ist."""
|
||||
|
||||
path: str
|
||||
message: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanResult:
|
||||
"""Zusammenfassung eines Scan-Laufs."""
|
||||
|
||||
root: str
|
||||
files_found: int = 0
|
||||
added: int = 0
|
||||
changed: int = 0
|
||||
unchanged: int = 0
|
||||
duplicates: int = 0
|
||||
errors: list[ScanError] = field(default_factory=list)
|
||||
skipped_extensions: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class RepositoryStats:
|
||||
"""Kennzahlen fuer ``sjaudiosearch status``."""
|
||||
|
||||
total_files: int = 0
|
||||
done: int = 0
|
||||
pending: int = 0
|
||||
processing: int = 0
|
||||
error: int = 0
|
||||
duplicates: int = 0
|
||||
total_duration: float = 0.0
|
||||
database_size: int = 0
|
||||
segment_count: int = 0
|
||||
word_count: int = 0
|
||||
transcripts: int = 0
|
||||
models: list[str] = field(default_factory=list)
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Rekursiver Audio-Scanner.
|
||||
|
||||
Der Scanner nutzt Dateigroesse und Aenderungszeit fuer eine schnelle
|
||||
Aenderungspruefung. SHA256-Hashes werden nur berechnet, wenn sie fuer die
|
||||
Duplikaterkennung tatsaechlich gebraucht werden (Modus ``auto``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from ..config import ProcessingConfig
|
||||
from ..database.repository import Repository
|
||||
from ..utils.hashing import sha256_file
|
||||
from ..utils.logging import get_logger
|
||||
from .metadata import AudioMetadataError, is_supported_audio, read_metadata
|
||||
from .models import FileStatus, ScanCandidate, ScanError, ScanResult
|
||||
|
||||
logger = get_logger("core.scanner")
|
||||
|
||||
ScanProgress = Callable[[int, Path], None]
|
||||
# Toleranz fuer Dateisystem-Zeitstempel (Sekunden).
|
||||
MTIME_TOLERANCE = 1.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class _DiscoveredFile:
|
||||
path: Path
|
||||
size: int
|
||||
mtime: float
|
||||
duration: float | None = None
|
||||
content_hash: str | None = None
|
||||
error: str | None = None
|
||||
existing_id: int | None = None
|
||||
changed: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ScanState:
|
||||
new_files: list[_DiscoveredFile] = field(default_factory=list)
|
||||
changed_files: list[_DiscoveredFile] = field(default_factory=list)
|
||||
|
||||
|
||||
def _iter_audio_files(root: Path) -> list[Path]:
|
||||
found: list[Path] = []
|
||||
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
|
||||
dirnames.sort()
|
||||
for filename in sorted(filenames):
|
||||
if filename.startswith("."):
|
||||
continue
|
||||
path = Path(dirpath) / filename
|
||||
if is_supported_audio(path):
|
||||
found.append(path)
|
||||
return found
|
||||
|
||||
|
||||
def _should_hash(mode: str) -> bool:
|
||||
return mode.strip().lower() == "always"
|
||||
|
||||
|
||||
def scan(
|
||||
root: str | Path,
|
||||
repository: Repository,
|
||||
processing: ProcessingConfig | None = None,
|
||||
*,
|
||||
progress: ScanProgress | None = None,
|
||||
) -> ScanResult:
|
||||
"""Scannt ``root`` rekursiv und aktualisiert die Datenbank."""
|
||||
processing = processing or ProcessingConfig()
|
||||
root_path = Path(root).expanduser().resolve()
|
||||
if not root_path.exists():
|
||||
raise FileNotFoundError(f"Pfad existiert nicht: {root_path}")
|
||||
if not root_path.is_dir():
|
||||
raise NotADirectoryError(f"Pfad ist kein Ordner: {root_path}")
|
||||
|
||||
result = ScanResult(root=str(root_path))
|
||||
state = _ScanState()
|
||||
|
||||
files = _iter_audio_files(root_path)
|
||||
result.files_found = len(files)
|
||||
|
||||
for index, path in enumerate(files, start=1):
|
||||
if progress:
|
||||
progress(index, path)
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError as exc:
|
||||
result.errors.append(ScanError(str(path), f"Dateizugriff fehlgeschlagen: {exc}"))
|
||||
continue
|
||||
|
||||
existing = repository.get_audio_by_path(str(path))
|
||||
discovered = _DiscoveredFile(
|
||||
path=path,
|
||||
size=int(stat.st_size),
|
||||
mtime=float(stat.st_mtime),
|
||||
existing_id=existing.id if existing else None,
|
||||
)
|
||||
|
||||
if existing is not None:
|
||||
modified = abs(float(existing.modified_at) - discovered.mtime) > MTIME_TOLERANCE
|
||||
if int(existing.file_size) != discovered.size or modified:
|
||||
discovered.changed = True
|
||||
state.changed_files.append(discovered)
|
||||
else:
|
||||
result.unchanged += 1
|
||||
continue
|
||||
else:
|
||||
state.new_files.append(discovered)
|
||||
|
||||
try:
|
||||
info = read_metadata(path)
|
||||
discovered.duration = info.duration
|
||||
except AudioMetadataError as exc:
|
||||
discovered.error = str(exc)
|
||||
result.errors.append(ScanError(str(path), str(exc)))
|
||||
logger.warning("Metadaten konnten nicht gelesen werden: %s (%s)", path, exc)
|
||||
|
||||
_assign_hashes(state, repository, processing)
|
||||
_insert_new_files(state.new_files, repository, processing, result)
|
||||
_update_changed_files(state.changed_files, repository, processing, result)
|
||||
|
||||
logger.info(
|
||||
"Scan abgeschlossen: %d gefunden, %d neu, %d geaendert, %d unveraendert, %d Duplikate, %d Fehler",
|
||||
result.files_found,
|
||||
result.added,
|
||||
result.changed,
|
||||
result.unchanged,
|
||||
result.duplicates,
|
||||
len(result.errors),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _assign_hashes(
|
||||
state: _ScanState, repository: Repository, processing: ProcessingConfig
|
||||
) -> None:
|
||||
"""Berechnet Hashes nur dort, wo Duplikate moeglich sind."""
|
||||
mode = (processing.compute_hash or "auto").strip().lower()
|
||||
if mode == "never":
|
||||
return
|
||||
|
||||
candidates = state.new_files + state.changed_files
|
||||
if not candidates:
|
||||
return
|
||||
|
||||
limit_bytes = max(0, int(processing.hash_max_size_mb)) * 1024 * 1024
|
||||
|
||||
if _should_hash(mode):
|
||||
targets = [item for item in candidates if item.error is None]
|
||||
else:
|
||||
size_counts: dict[int, int] = {}
|
||||
for item in candidates:
|
||||
size_counts[item.size] = size_counts.get(item.size, 0) + 1
|
||||
targets = []
|
||||
for item in candidates:
|
||||
if item.error is not None:
|
||||
continue
|
||||
needs_check = size_counts.get(item.size, 0) > 1
|
||||
if not needs_check:
|
||||
needs_check = bool(repository.paths_with_size(item.size))
|
||||
if needs_check:
|
||||
targets.append(item)
|
||||
|
||||
for item in targets:
|
||||
if limit_bytes and item.size > limit_bytes:
|
||||
logger.info("Datei zu gross fuer Hashing, wird uebersprungen: %s", item.path)
|
||||
continue
|
||||
try:
|
||||
item.content_hash = sha256_file(item.path)
|
||||
except OSError as exc:
|
||||
logger.warning("Hashing fehlgeschlagen fuer %s: %s", item.path, exc)
|
||||
|
||||
|
||||
def _insert_new_files(
|
||||
files: list[_DiscoveredFile],
|
||||
repository: Repository,
|
||||
processing: ProcessingConfig,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
seen_hashes: dict[str, int] = {}
|
||||
for item in files:
|
||||
duplicate_of: int | None = None
|
||||
status = FileStatus.PENDING.value
|
||||
if item.content_hash:
|
||||
existing = repository.find_by_hash(item.content_hash)
|
||||
if existing is not None:
|
||||
duplicate_of = existing.id
|
||||
elif item.content_hash in seen_hashes:
|
||||
duplicate_of = seen_hashes[item.content_hash]
|
||||
if item.error is not None and duplicate_of is None:
|
||||
status = FileStatus.ERROR.value
|
||||
|
||||
candidate = ScanCandidate(
|
||||
path=str(item.path),
|
||||
filename=item.path.name,
|
||||
file_size=item.size,
|
||||
modified_at=item.mtime,
|
||||
duration=item.duration,
|
||||
content_hash=item.content_hash,
|
||||
)
|
||||
audio_id = repository.insert_audio(
|
||||
candidate,
|
||||
status=status,
|
||||
duplicate_of=duplicate_of,
|
||||
error_message=item.error,
|
||||
)
|
||||
result.added += 1
|
||||
if duplicate_of is not None:
|
||||
result.duplicates += 1
|
||||
elif item.content_hash:
|
||||
seen_hashes[item.content_hash] = audio_id
|
||||
|
||||
|
||||
def _update_changed_files(
|
||||
files: list[_DiscoveredFile],
|
||||
repository: Repository,
|
||||
processing: ProcessingConfig,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
for item in files:
|
||||
assert item.existing_id is not None
|
||||
duplicate_of: int | None = None
|
||||
status = FileStatus.PENDING.value
|
||||
if item.content_hash:
|
||||
existing = repository.find_by_hash(item.content_hash, exclude_id=item.existing_id)
|
||||
if existing is not None:
|
||||
duplicate_of = existing.id
|
||||
if item.error is not None and duplicate_of is None:
|
||||
status = FileStatus.ERROR.value
|
||||
|
||||
candidate = ScanCandidate(
|
||||
path=str(item.path),
|
||||
filename=item.path.name,
|
||||
file_size=item.size,
|
||||
modified_at=item.mtime,
|
||||
duration=item.duration,
|
||||
content_hash=item.content_hash,
|
||||
)
|
||||
repository.update_scan_data(
|
||||
item.existing_id,
|
||||
candidate,
|
||||
status=status,
|
||||
duplicate_of=duplicate_of,
|
||||
error_message=item.error,
|
||||
)
|
||||
result.changed += 1
|
||||
if duplicate_of is not None:
|
||||
result.duplicates += 1
|
||||
|
||||
if files:
|
||||
logger.info("%d geaenderte Dateien zur erneuten Verarbeitung vorgemerkt", len(files))
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Verbindungsaufbau fuer die SQLite-Datenbank."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger("database")
|
||||
|
||||
|
||||
def connect(db_path: str | Path, *, create: bool = True) -> sqlite3.Connection:
|
||||
"""Oeffnet die Datenbank mit sinnvollen PRAGMAs.
|
||||
|
||||
``create=False`` verhindert, dass versehentlich eine neue Datenbankdatei
|
||||
angelegt wird (z. B. bei reiner Diagnose).
|
||||
"""
|
||||
path = Path(db_path)
|
||||
if not create and not path.exists():
|
||||
raise FileNotFoundError(f"Datenbank nicht gefunden: {path}")
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
conn = sqlite3.connect(str(path), timeout=30.0)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.isolation_level = "" # explizite Transaktionen via "with conn:"
|
||||
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute("PRAGMA journal_mode = WAL")
|
||||
conn.execute("PRAGMA synchronous = NORMAL")
|
||||
conn.execute("PRAGMA busy_timeout = 30000")
|
||||
conn.execute("PRAGMA temp_store = MEMORY")
|
||||
|
||||
return conn
|
||||
|
||||
|
||||
def check_fts5(conn: sqlite3.Connection) -> tuple[bool, bool]:
|
||||
"""Prueft, ob FTS5 und der Trigram-Tokenizer verfuegbar sind."""
|
||||
fts5 = False
|
||||
trigram = False
|
||||
try:
|
||||
conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS _fts_check USING fts5(x)")
|
||||
conn.execute("DROP TABLE IF EXISTS _fts_check")
|
||||
fts5 = True
|
||||
except sqlite3.OperationalError as exc:
|
||||
logger.warning("FTS5 nicht verfuegbar: %s", exc)
|
||||
return False, False
|
||||
|
||||
if fts5:
|
||||
try:
|
||||
conn.execute(
|
||||
"CREATE VIRTUAL TABLE IF NOT EXISTS _trigram_check USING fts5(x, tokenize='trigram')"
|
||||
)
|
||||
conn.execute("DROP TABLE IF EXISTS _trigram_check")
|
||||
trigram = True
|
||||
except sqlite3.OperationalError as exc:
|
||||
logger.warning("FTS5-Trigram-Tokenizer nicht verfuegbar: %s", exc)
|
||||
return fts5, trigram
|
||||
|
||||
|
||||
def sqlite_version() -> str:
|
||||
return sqlite3.sqlite_version
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Schema-Migrationen fuer SjAudioSearch.
|
||||
|
||||
Die Version wird in ``PRAGMA user_version`` gespeichert. Jede Migration laeuft
|
||||
in einer eigenen Transaktion, damit ein Abbruch keinen halb migrierten Zustand
|
||||
hinterlaesst.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
from .schema import SCHEMA_VERSION, TABLES_SQL
|
||||
|
||||
logger = get_logger("database.migrations")
|
||||
|
||||
|
||||
def current_version(conn: sqlite3.Connection) -> int:
|
||||
row = conn.execute("PRAGMA user_version").fetchone()
|
||||
return int(row[0]) if row else 0
|
||||
|
||||
|
||||
def _migration_1(conn: sqlite3.Connection) -> None:
|
||||
conn.executescript(TABLES_SQL)
|
||||
|
||||
|
||||
MIGRATIONS: list[tuple[int, object]] = [
|
||||
(1, _migration_1),
|
||||
]
|
||||
|
||||
|
||||
def migrate(conn: sqlite3.Connection, target: int = SCHEMA_VERSION) -> int:
|
||||
"""Fuehrt alle ausstehenden Migrationen aus und liefert die neue Version."""
|
||||
version = current_version(conn)
|
||||
if version > target:
|
||||
raise RuntimeError(
|
||||
f"Datenbankversion {version} ist neuer als die unterstuetzte Version {target}."
|
||||
)
|
||||
if version == target:
|
||||
return version
|
||||
|
||||
for migration_version, migration in MIGRATIONS:
|
||||
if migration_version <= version:
|
||||
continue
|
||||
logger.info("Migration %s wird angewendet", migration_version)
|
||||
with conn:
|
||||
migration(conn) # type: ignore[operator]
|
||||
conn.execute(f"PRAGMA user_version = {int(migration_version)}")
|
||||
version = migration_version
|
||||
|
||||
if version != target:
|
||||
raise RuntimeError(
|
||||
f"Migration unvollstaendig: Version {version}, erwartet {target}."
|
||||
)
|
||||
return version
|
||||
@@ -0,0 +1,547 @@
|
||||
"""Datenbankzugriff fuer SjAudioSearch.
|
||||
|
||||
Alle Schreibvorgaenge laufen in expliziten Transaktionen, damit ein Abbruch
|
||||
keine halben Transkripte hinterlaesst.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from collections.abc import Iterator, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..core.models import AudioFileRecord, FileStatus, RepositoryStats, ScanCandidate
|
||||
from ..search.fuzzy import normalize_text
|
||||
from ..utils.formatting import utc_now_iso
|
||||
from ..utils.logging import get_logger
|
||||
from .connection import connect
|
||||
from .migrations import migrate
|
||||
|
||||
logger = get_logger("database.repository")
|
||||
|
||||
_AUDIO_COLUMNS = """
|
||||
id, path, filename, duration, file_size, modified_at, content_hash,
|
||||
status, duplicate_of, error_message, created_at, updated_at
|
||||
"""
|
||||
|
||||
|
||||
def _row_to_audio(row: sqlite3.Row) -> AudioFileRecord:
|
||||
return AudioFileRecord(
|
||||
id=row["id"],
|
||||
path=row["path"],
|
||||
filename=row["filename"],
|
||||
duration=row["duration"],
|
||||
file_size=row["file_size"],
|
||||
modified_at=row["modified_at"],
|
||||
content_hash=row["content_hash"],
|
||||
status=row["status"],
|
||||
duplicate_of=row["duplicate_of"],
|
||||
error_message=row["error_message"],
|
||||
created_at=row["created_at"],
|
||||
updated_at=row["updated_at"],
|
||||
)
|
||||
|
||||
|
||||
class Repository:
|
||||
"""Kapselt saemtliche SQL-Zugriffe."""
|
||||
|
||||
def __init__(self, connection: sqlite3.Connection):
|
||||
self.conn = connection
|
||||
|
||||
@classmethod
|
||||
def open(cls, db_path: str | Path) -> "Repository":
|
||||
conn = connect(db_path)
|
||||
migrate(conn)
|
||||
return cls(conn)
|
||||
|
||||
def close(self) -> None:
|
||||
self.conn.close()
|
||||
|
||||
def __enter__(self) -> "Repository":
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc_info: object) -> None:
|
||||
self.close()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# audio_files
|
||||
# ------------------------------------------------------------------
|
||||
def get_audio(self, audio_id: int) -> AudioFileRecord | None:
|
||||
row = self.conn.execute(
|
||||
f"SELECT {_AUDIO_COLUMNS} FROM audio_files WHERE id = ?", (audio_id,)
|
||||
).fetchone()
|
||||
return _row_to_audio(row) if row else None
|
||||
|
||||
def get_audio_by_path(self, path: str) -> AudioFileRecord | None:
|
||||
row = self.conn.execute(
|
||||
f"SELECT {_AUDIO_COLUMNS} FROM audio_files WHERE path = ?", (str(path),)
|
||||
).fetchone()
|
||||
return _row_to_audio(row) if row else None
|
||||
|
||||
def list_audio(
|
||||
self,
|
||||
*,
|
||||
status: str | None = None,
|
||||
limit: int | None = None,
|
||||
search: str | None = None,
|
||||
include_duplicates: bool = True,
|
||||
) -> list[AudioFileRecord]:
|
||||
clauses: list[str] = []
|
||||
params: list[Any] = []
|
||||
if status:
|
||||
clauses.append("status = ?")
|
||||
params.append(status)
|
||||
if not include_duplicates:
|
||||
clauses.append("duplicate_of IS NULL")
|
||||
if search:
|
||||
clauses.append("(filename LIKE ? OR path LIKE ?)")
|
||||
like = f"%{search}%"
|
||||
params.extend([like, like])
|
||||
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
|
||||
sql = f"SELECT {_AUDIO_COLUMNS} FROM audio_files {where} ORDER BY filename COLLATE NOCASE, id"
|
||||
if limit:
|
||||
sql += " LIMIT ?"
|
||||
params.append(int(limit))
|
||||
rows = self.conn.execute(sql, params).fetchall()
|
||||
return [_row_to_audio(row) for row in rows]
|
||||
|
||||
def insert_audio(
|
||||
self,
|
||||
candidate: ScanCandidate,
|
||||
*,
|
||||
status: str = FileStatus.PENDING.value,
|
||||
duplicate_of: int | None = None,
|
||||
error_message: str | None = None,
|
||||
) -> int:
|
||||
now = utc_now_iso()
|
||||
with self.conn:
|
||||
cursor = self.conn.execute(
|
||||
"""
|
||||
INSERT INTO audio_files
|
||||
(path, filename, duration, file_size, modified_at, content_hash,
|
||||
status, duplicate_of, error_message, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
candidate.path,
|
||||
candidate.filename,
|
||||
candidate.duration,
|
||||
int(candidate.file_size),
|
||||
float(candidate.modified_at),
|
||||
candidate.content_hash,
|
||||
status,
|
||||
duplicate_of,
|
||||
error_message,
|
||||
now,
|
||||
now,
|
||||
),
|
||||
)
|
||||
return int(cursor.lastrowid)
|
||||
|
||||
def update_scan_data(
|
||||
self,
|
||||
audio_id: int,
|
||||
candidate: ScanCandidate,
|
||||
*,
|
||||
status: str = FileStatus.PENDING.value,
|
||||
duplicate_of: int | None = None,
|
||||
error_message: str | None = None,
|
||||
) -> None:
|
||||
with self.conn:
|
||||
self.conn.execute(
|
||||
"""
|
||||
UPDATE audio_files
|
||||
SET duration = ?, file_size = ?, modified_at = ?, content_hash = ?,
|
||||
status = ?, duplicate_of = ?, error_message = ?, updated_at = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(
|
||||
candidate.duration,
|
||||
int(candidate.file_size),
|
||||
float(candidate.modified_at),
|
||||
candidate.content_hash,
|
||||
status,
|
||||
duplicate_of,
|
||||
error_message,
|
||||
utc_now_iso(),
|
||||
audio_id,
|
||||
),
|
||||
)
|
||||
|
||||
def set_duration(self, audio_id: int, duration: float | None) -> None:
|
||||
if duration is None:
|
||||
return
|
||||
with self.conn:
|
||||
self.conn.execute(
|
||||
"UPDATE audio_files SET duration = ?, updated_at = ? WHERE id = ? AND duration IS NULL",
|
||||
(float(duration), utc_now_iso(), audio_id),
|
||||
)
|
||||
|
||||
def set_status(self, audio_id: int, status: str, error_message: str | None = None) -> None:
|
||||
with self.conn:
|
||||
self.conn.execute(
|
||||
"UPDATE audio_files SET status = ?, error_message = ?, updated_at = ? WHERE id = ?",
|
||||
(status, error_message, utc_now_iso(), audio_id),
|
||||
)
|
||||
|
||||
def reset_stale_processing(self) -> int:
|
||||
"""Setzt abgebrochene 'processing'-Eintraege nach einem Absturz zurueck."""
|
||||
with self.conn:
|
||||
cursor = self.conn.execute(
|
||||
"UPDATE audio_files SET status = 'pending', updated_at = ? WHERE status = 'processing'",
|
||||
(utc_now_iso(),),
|
||||
)
|
||||
return cursor.rowcount or 0
|
||||
|
||||
def reset_errors(self) -> int:
|
||||
"""Setzt fehlgeschlagene Transkriptionen zurueck auf 'pending'."""
|
||||
with self.conn:
|
||||
cursor = self.conn.execute(
|
||||
"""
|
||||
UPDATE audio_files
|
||||
SET status = 'pending', error_message = NULL, updated_at = ?
|
||||
WHERE status = 'error'
|
||||
""",
|
||||
(utc_now_iso(),),
|
||||
)
|
||||
return cursor.rowcount or 0
|
||||
|
||||
def transcription_queue(self, *, force: bool = False) -> list[AudioFileRecord]:
|
||||
if force:
|
||||
sql = f"""
|
||||
SELECT {_AUDIO_COLUMNS} FROM audio_files
|
||||
WHERE duplicate_of IS NULL
|
||||
ORDER BY id
|
||||
"""
|
||||
rows = self.conn.execute(sql).fetchall()
|
||||
else:
|
||||
sql = f"""
|
||||
SELECT {_AUDIO_COLUMNS} FROM audio_files
|
||||
WHERE duplicate_of IS NULL AND status IN ('pending', 'processing')
|
||||
ORDER BY id
|
||||
"""
|
||||
rows = self.conn.execute(sql).fetchall()
|
||||
return [_row_to_audio(row) for row in rows]
|
||||
|
||||
def find_by_hash(self, content_hash: str, *, exclude_id: int | None = None) -> AudioFileRecord | None:
|
||||
sql = f"SELECT {_AUDIO_COLUMNS} FROM audio_files WHERE content_hash = ? AND duplicate_of IS NULL"
|
||||
params: list[Any] = [content_hash]
|
||||
if exclude_id is not None:
|
||||
sql += " AND id != ?"
|
||||
params.append(exclude_id)
|
||||
row = self.conn.execute(sql + " ORDER BY id LIMIT 1", params).fetchone()
|
||||
return _row_to_audio(row) if row else None
|
||||
|
||||
def paths_with_size(self, size: int) -> list[AudioFileRecord]:
|
||||
rows = self.conn.execute(
|
||||
f"SELECT {_AUDIO_COLUMNS} FROM audio_files WHERE file_size = ? AND content_hash IS NOT NULL",
|
||||
(int(size),),
|
||||
).fetchall()
|
||||
return [_row_to_audio(row) for row in rows]
|
||||
|
||||
def count_by_status(self) -> dict[str, int]:
|
||||
rows = self.conn.execute(
|
||||
"SELECT status, COUNT(*) AS amount FROM audio_files GROUP BY status"
|
||||
).fetchall()
|
||||
return {row["status"]: int(row["amount"]) for row in rows}
|
||||
|
||||
def stats(self, db_size_bytes: int = 0) -> RepositoryStats:
|
||||
counts = self.count_by_status()
|
||||
total_row = self.conn.execute(
|
||||
"SELECT COUNT(*) AS total, COALESCE(SUM(duration), 0) AS duration FROM audio_files"
|
||||
).fetchone()
|
||||
dup_row = self.conn.execute(
|
||||
"SELECT COUNT(*) AS amount FROM audio_files WHERE duplicate_of IS NOT NULL"
|
||||
).fetchone()
|
||||
segment_row = self.conn.execute("SELECT COUNT(*) AS amount FROM segments").fetchone()
|
||||
word_row = self.conn.execute("SELECT COUNT(*) AS amount FROM words").fetchone()
|
||||
transcript_row = self.conn.execute("SELECT COUNT(*) AS amount FROM transcripts").fetchone()
|
||||
model_rows = self.conn.execute("SELECT DISTINCT model FROM transcripts").fetchall()
|
||||
return RepositoryStats(
|
||||
total_files=int(total_row["total"]),
|
||||
done=counts.get("done", 0),
|
||||
pending=counts.get("pending", 0),
|
||||
processing=counts.get("processing", 0),
|
||||
error=counts.get("error", 0),
|
||||
duplicates=int(dup_row["amount"]),
|
||||
total_duration=float(total_row["duration"] or 0.0),
|
||||
database_size=int(db_size_bytes),
|
||||
segment_count=int(segment_row["amount"]),
|
||||
word_count=int(word_row["amount"]),
|
||||
transcripts=int(transcript_row["amount"]),
|
||||
models=[row["model"] for row in model_rows],
|
||||
)
|
||||
|
||||
def databases_size(self) -> int:
|
||||
row = self.conn.execute("PRAGMA database_list").fetchone()
|
||||
if not row:
|
||||
return 0
|
||||
path = Path(row["file"])
|
||||
size = path.stat().st_size if path.exists() else 0
|
||||
wal = path.with_name(path.name + "-wal")
|
||||
if wal.exists():
|
||||
size += wal.stat().st_size
|
||||
return size
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Transkripte
|
||||
# ------------------------------------------------------------------
|
||||
def save_transcript(
|
||||
self,
|
||||
audio_id: int,
|
||||
*,
|
||||
language: str | None,
|
||||
model: str,
|
||||
full_text: str,
|
||||
segments: Sequence[Any],
|
||||
duration: float | None = None,
|
||||
) -> int:
|
||||
"""Speichert ein Transkript samt Segmenten, Woertern und Suchindex.
|
||||
|
||||
Vorhandene Daten der Datei werden ersetzt. Alles geschieht in einer
|
||||
Transaktion, damit ein Abbruch keinen Teilzustand hinterlaesst.
|
||||
"""
|
||||
now = utc_now_iso()
|
||||
with self.conn:
|
||||
self.conn.execute("DELETE FROM transcripts WHERE audio_id = ?", (audio_id,))
|
||||
self.conn.execute("DELETE FROM search_index WHERE audio_id = ?", (audio_id,))
|
||||
self.conn.execute("DELETE FROM search_trigram WHERE audio_id = ?", (audio_id,))
|
||||
|
||||
cursor = self.conn.execute(
|
||||
"""
|
||||
INSERT INTO transcripts (audio_id, language, model, full_text, created_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""",
|
||||
(audio_id, language, model, full_text, now),
|
||||
)
|
||||
transcript_id = int(cursor.lastrowid)
|
||||
|
||||
for segment in segments:
|
||||
cursor = self.conn.execute(
|
||||
"""
|
||||
INSERT INTO segments (transcript_id, start_time, end_time, text, confidence)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
transcript_id,
|
||||
float(segment.start),
|
||||
float(segment.end),
|
||||
segment.text,
|
||||
getattr(segment, "confidence", None),
|
||||
),
|
||||
)
|
||||
segment_id = int(cursor.lastrowid)
|
||||
|
||||
words = getattr(segment, "words", None) or []
|
||||
if words:
|
||||
self.conn.executemany(
|
||||
"INSERT INTO words (segment_id, word, start_time, end_time) VALUES (?, ?, ?, ?)",
|
||||
[
|
||||
(
|
||||
segment_id,
|
||||
word.word,
|
||||
getattr(word, "start", None),
|
||||
getattr(word, "end", None),
|
||||
)
|
||||
for word in words
|
||||
],
|
||||
)
|
||||
|
||||
normalized = normalize_text(segment.text)
|
||||
if normalized:
|
||||
self.conn.execute(
|
||||
"""
|
||||
INSERT INTO search_index (text, audio_id, transcript_id, segment_id)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""",
|
||||
(normalized, audio_id, transcript_id, segment_id),
|
||||
)
|
||||
self.conn.execute(
|
||||
"INSERT INTO search_trigram (text, audio_id, segment_id) VALUES (?, ?, ?)",
|
||||
(normalized, audio_id, segment_id),
|
||||
)
|
||||
|
||||
if duration is not None:
|
||||
self.conn.execute(
|
||||
"""
|
||||
UPDATE audio_files
|
||||
SET duration = COALESCE(duration, ?), status = 'done',
|
||||
error_message = NULL, updated_at = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(float(duration), now, audio_id),
|
||||
)
|
||||
else:
|
||||
self.conn.execute(
|
||||
"""
|
||||
UPDATE audio_files
|
||||
SET status = 'done', error_message = NULL, updated_at = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(now, audio_id),
|
||||
)
|
||||
return transcript_id
|
||||
|
||||
def get_transcript(self, audio_id: int) -> sqlite3.Row | None:
|
||||
return self.conn.execute(
|
||||
"SELECT * FROM transcripts WHERE audio_id = ?", (audio_id,)
|
||||
).fetchone()
|
||||
|
||||
def get_segments(self, audio_id: int) -> list[sqlite3.Row]:
|
||||
return self.conn.execute(
|
||||
"""
|
||||
SELECT s.id, s.start_time, s.end_time, s.text, s.confidence
|
||||
FROM segments s
|
||||
JOIN transcripts t ON t.id = s.transcript_id
|
||||
WHERE t.audio_id = ?
|
||||
ORDER BY s.start_time, s.id
|
||||
""",
|
||||
(audio_id,),
|
||||
).fetchall()
|
||||
|
||||
def get_words(self, segment_ids: Sequence[int]) -> dict[int, list[sqlite3.Row]]:
|
||||
result: dict[int, list[sqlite3.Row]] = {}
|
||||
ids = list(segment_ids)
|
||||
for start in range(0, len(ids), 500):
|
||||
chunk = ids[start : start + 500]
|
||||
placeholders = ",".join("?" for _ in chunk)
|
||||
rows = self.conn.execute(
|
||||
f"""
|
||||
SELECT id, segment_id, word, start_time, end_time
|
||||
FROM words WHERE segment_id IN ({placeholders})
|
||||
ORDER BY segment_id, start_time, id
|
||||
""",
|
||||
chunk,
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
result.setdefault(int(row["segment_id"]), []).append(row)
|
||||
return result
|
||||
|
||||
def segment_texts(self, audio_id: int) -> list[str]:
|
||||
return [row["text"] for row in self.get_segments(audio_id)]
|
||||
|
||||
def iter_audio_ids_with_transcripts(self, language: str | None = None) -> Iterator[int]:
|
||||
sql = "SELECT DISTINCT audio_id FROM transcripts"
|
||||
params: list[Any] = []
|
||||
if language:
|
||||
sql += " WHERE language = ?"
|
||||
params.append(language)
|
||||
sql += " ORDER BY audio_id"
|
||||
cursor = self.conn.execute(sql, params)
|
||||
for row in cursor:
|
||||
yield int(row["audio_id"])
|
||||
|
||||
def transcript_languages(self) -> list[str]:
|
||||
rows = self.conn.execute(
|
||||
"SELECT DISTINCT language FROM transcripts WHERE language IS NOT NULL ORDER BY language"
|
||||
).fetchall()
|
||||
return [row["language"] for row in rows]
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Suche
|
||||
# ------------------------------------------------------------------
|
||||
def fts_candidates(self, match_query: str, limit: int) -> list[tuple[int, int]]:
|
||||
"""Liefert (audio_id, segment_id) fuer einen FTS5-Ausdruck."""
|
||||
if not match_query:
|
||||
return []
|
||||
try:
|
||||
rows = self.conn.execute(
|
||||
"""
|
||||
SELECT audio_id, segment_id, bm25(search_index) AS rank
|
||||
FROM search_index
|
||||
WHERE search_index MATCH ?
|
||||
ORDER BY rank
|
||||
LIMIT ?
|
||||
""",
|
||||
(match_query, int(limit)),
|
||||
).fetchall()
|
||||
except sqlite3.OperationalError as exc:
|
||||
logger.warning("FTS-Anfrage fehlgeschlagen (%s): %s", match_query, exc)
|
||||
return []
|
||||
return [(int(row["audio_id"]), int(row["segment_id"])) for row in rows]
|
||||
|
||||
def trigram_candidates(self, terms: Sequence[str], limit: int) -> list[tuple[int, int]]:
|
||||
"""Teilstring-Kandidaten ueber den FTS5-Trigram-Index."""
|
||||
results: list[tuple[int, int]] = []
|
||||
seen: set[tuple[int, int]] = set()
|
||||
for term in terms:
|
||||
escaped = '"' + term.replace('"', '""') + '"'
|
||||
try:
|
||||
rows = self.conn.execute(
|
||||
"""
|
||||
SELECT audio_id, segment_id FROM search_trigram
|
||||
WHERE search_trigram MATCH ?
|
||||
LIMIT ?
|
||||
""",
|
||||
(escaped, int(limit)),
|
||||
).fetchall()
|
||||
except sqlite3.OperationalError as exc:
|
||||
logger.warning("Trigramm-Anfrage fehlgeschlagen (%s): %s", term, exc)
|
||||
continue
|
||||
for row in rows:
|
||||
key = (int(row["audio_id"]), int(row["segment_id"]))
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
results.append(key)
|
||||
if len(results) >= limit:
|
||||
return results
|
||||
return results
|
||||
|
||||
def timeline(self, audio_id: int) -> tuple[list[sqlite3.Row], dict[int, list[sqlite3.Row]]]:
|
||||
segments = self.get_segments(audio_id)
|
||||
words = self.get_words([int(segment["id"]) for segment in segments])
|
||||
return segments, words
|
||||
|
||||
def rebuild_search_index(self) -> int:
|
||||
"""Baut den FTS-Index aus den gespeicherten Segmenten neu auf."""
|
||||
with self.conn:
|
||||
self.conn.execute("DELETE FROM search_index")
|
||||
self.conn.execute("DELETE FROM search_trigram")
|
||||
rows = self.conn.execute(
|
||||
"""
|
||||
SELECT s.id AS segment_id, t.audio_id AS audio_id, t.id AS transcript_id, s.text
|
||||
FROM segments s
|
||||
JOIN transcripts t ON t.id = s.transcript_id
|
||||
ORDER BY s.id
|
||||
"""
|
||||
).fetchall()
|
||||
index_rows = []
|
||||
trigram_rows = []
|
||||
for row in rows:
|
||||
normalized = normalize_text(row["text"])
|
||||
if not normalized:
|
||||
continue
|
||||
index_rows.append(
|
||||
(normalized, int(row["audio_id"]), int(row["transcript_id"]), int(row["segment_id"]))
|
||||
)
|
||||
trigram_rows.append((normalized, int(row["audio_id"]), int(row["segment_id"])))
|
||||
self.conn.executemany(
|
||||
"INSERT INTO search_index (text, audio_id, transcript_id, segment_id) VALUES (?, ?, ?, ?)",
|
||||
index_rows,
|
||||
)
|
||||
self.conn.executemany(
|
||||
"INSERT INTO search_trigram (text, audio_id, segment_id) VALUES (?, ?, ?)",
|
||||
trigram_rows,
|
||||
)
|
||||
return len(index_rows)
|
||||
|
||||
def delete_audio(self, audio_id: int) -> bool:
|
||||
"""Entfernt eine Datei aus dem Index (die Audiodatei bleibt unberuehrt)."""
|
||||
with self.conn:
|
||||
self.conn.execute("DELETE FROM search_index WHERE audio_id = ?", (audio_id,))
|
||||
self.conn.execute("DELETE FROM search_trigram WHERE audio_id = ?", (audio_id,))
|
||||
cursor = self.conn.execute("DELETE FROM audio_files WHERE id = ?", (audio_id,))
|
||||
return (cursor.rowcount or 0) > 0
|
||||
|
||||
def upsert_meta(self, key: str, value: str) -> None:
|
||||
with self.conn:
|
||||
self.conn.execute(
|
||||
"INSERT INTO meta (key, value) VALUES (?, ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(key, value),
|
||||
)
|
||||
|
||||
def get_meta(self, key: str) -> str | None:
|
||||
row = self.conn.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone()
|
||||
return row["value"] if row else None
|
||||
@@ -0,0 +1,80 @@
|
||||
"""SQL-Schema fuer SjAudioSearch (SQLite + FTS5)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
TABLES_SQL = """
|
||||
CREATE TABLE IF NOT EXISTS audio_files (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
filename TEXT NOT NULL,
|
||||
duration REAL,
|
||||
file_size INTEGER NOT NULL,
|
||||
modified_at REAL NOT NULL,
|
||||
content_hash TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN ('pending', 'processing', 'done', 'error')),
|
||||
duplicate_of INTEGER REFERENCES audio_files(id) ON DELETE SET NULL,
|
||||
error_message TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audio_status ON audio_files(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_audio_hash ON audio_files(content_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_audio_size ON audio_files(file_size);
|
||||
CREATE INDEX IF NOT EXISTS idx_audio_duplicate ON audio_files(duplicate_of);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS transcripts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
audio_id INTEGER NOT NULL UNIQUE REFERENCES audio_files(id) ON DELETE CASCADE,
|
||||
language TEXT,
|
||||
model TEXT NOT NULL,
|
||||
full_text TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_transcripts_language ON transcripts(language);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS segments (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
transcript_id INTEGER NOT NULL REFERENCES transcripts(id) ON DELETE CASCADE,
|
||||
start_time REAL NOT NULL,
|
||||
end_time REAL NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
confidence REAL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_segments_transcript ON segments(transcript_id, start_time);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS words (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
segment_id INTEGER NOT NULL REFERENCES segments(id) ON DELETE CASCADE,
|
||||
word TEXT NOT NULL,
|
||||
start_time REAL,
|
||||
end_time REAL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_words_segment ON words(segment_id);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
|
||||
text,
|
||||
audio_id UNINDEXED,
|
||||
transcript_id UNINDEXED,
|
||||
segment_id UNINDEXED,
|
||||
tokenize = "unicode61 remove_diacritics 2"
|
||||
);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS search_trigram USING fts5(
|
||||
text,
|
||||
audio_id UNINDEXED,
|
||||
segment_id UNINDEXED,
|
||||
tokenize = "trigram"
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT
|
||||
);
|
||||
"""
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Kontextaufbau fuer Suchtreffer.
|
||||
|
||||
Aus den gespeicherten Segmenten wird eine durchgehende Wort-Timeline gebaut.
|
||||
Dadurch gehen an Segmentgrenzen keine Informationen verloren, und der Kontext
|
||||
kann mitten in einem Segment beginnen und enden.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Sequence
|
||||
|
||||
from .fuzzy import normalize_text, tokenize_original
|
||||
|
||||
|
||||
@dataclass
|
||||
class TokenRef:
|
||||
"""Ein Wort im Transkript mit Zeitbezug und Herkunft."""
|
||||
|
||||
text: str
|
||||
norm: str
|
||||
start: float
|
||||
end: float
|
||||
segment_id: int
|
||||
precision: str # "word" oder "segment"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Timeline:
|
||||
"""Durchgehende Wortfolge einer Audiodatei."""
|
||||
|
||||
audio_id: int
|
||||
tokens: list[TokenRef] = field(default_factory=list)
|
||||
language: str | None = None
|
||||
model: str | None = None
|
||||
|
||||
@property
|
||||
def normalized_tokens(self) -> list[str]:
|
||||
return [token.norm for token in self.tokens]
|
||||
|
||||
@property
|
||||
def normalized_text(self) -> str:
|
||||
return " ".join(token.norm for token in self.tokens)
|
||||
|
||||
@property
|
||||
def original_text(self) -> str:
|
||||
return " ".join(token.text for token in self.tokens)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.tokens)
|
||||
|
||||
|
||||
def _distribute_times(
|
||||
segment_start: float, segment_end: float, token_count: int
|
||||
) -> list[tuple[float, float]]:
|
||||
"""Verteilt die Segmentzeit gleichmaessig auf die Woerter (Schaetzung)."""
|
||||
if token_count <= 0:
|
||||
return []
|
||||
span = max(0.0, float(segment_end) - float(segment_start))
|
||||
step = span / token_count if span else 0.0
|
||||
times: list[tuple[float, float]] = []
|
||||
for index in range(token_count):
|
||||
start = float(segment_start) + step * index
|
||||
end = float(segment_start) + step * (index + 1)
|
||||
times.append((round(start, 3), round(end, 3)))
|
||||
return times
|
||||
|
||||
|
||||
def build_timeline(
|
||||
audio_id: int,
|
||||
segments: Sequence[Any],
|
||||
words_by_segment: dict[int, Sequence[Any]] | None = None,
|
||||
*,
|
||||
language: str | None = None,
|
||||
model: str | None = None,
|
||||
) -> Timeline:
|
||||
"""Baut die Timeline aus Segmenten (sqlite3.Row oder Objekte mit Attributen)."""
|
||||
timeline = Timeline(audio_id=audio_id, language=language, model=model)
|
||||
words_by_segment = words_by_segment or {}
|
||||
|
||||
for segment in segments:
|
||||
if isinstance(segment, dict):
|
||||
segment_id = int(segment["id"])
|
||||
start = float(segment["start_time"])
|
||||
end = float(segment["end_time"])
|
||||
text = segment["text"]
|
||||
else:
|
||||
segment_id = int(segment["id"])
|
||||
start = float(segment["start_time"])
|
||||
end = float(segment["end_time"])
|
||||
text = segment["text"]
|
||||
|
||||
original_tokens = tokenize_original(text)
|
||||
if not original_tokens:
|
||||
continue
|
||||
|
||||
words = list(words_by_segment.get(segment_id, []))
|
||||
use_words = len(words) == len(original_tokens)
|
||||
|
||||
if use_words:
|
||||
for token, word in zip(original_tokens, words):
|
||||
word_start = word["start_time"] if not isinstance(word, dict) else word.get("start_time")
|
||||
word_end = word["end_time"] if not isinstance(word, dict) else word.get("end_time")
|
||||
if word_start is None or word_end is None:
|
||||
use_words = False
|
||||
break
|
||||
|
||||
if use_words:
|
||||
for token, word in zip(original_tokens, words):
|
||||
word_start = word["start_time"] if not isinstance(word, dict) else word["start_time"]
|
||||
word_end = word["end_time"] if not isinstance(word, dict) else word["end_time"]
|
||||
timeline.tokens.append(
|
||||
TokenRef(
|
||||
text=token,
|
||||
norm=normalize_text(token),
|
||||
start=float(word_start),
|
||||
end=float(word_end),
|
||||
segment_id=segment_id,
|
||||
precision="word",
|
||||
)
|
||||
)
|
||||
else:
|
||||
times = _distribute_times(start, end, len(original_tokens))
|
||||
for token, (token_start, token_end) in zip(original_tokens, times):
|
||||
timeline.tokens.append(
|
||||
TokenRef(
|
||||
text=token,
|
||||
norm=normalize_text(token),
|
||||
start=token_start,
|
||||
end=token_end,
|
||||
segment_id=segment_id,
|
||||
precision="segment",
|
||||
)
|
||||
)
|
||||
|
||||
return timeline
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchContext:
|
||||
"""Die drei Kontextabschnitte eines Treffers."""
|
||||
|
||||
before: str
|
||||
match: str
|
||||
after: str
|
||||
start_time: float
|
||||
end_time: float
|
||||
precision: str
|
||||
|
||||
def as_block(self, quote: str = '"') -> str:
|
||||
"""Formatiert den Kontext als zusammenhaengenden Textblock."""
|
||||
before = f"\u2026{self.before} " if self.before else ""
|
||||
after = f" {self.after}\u2026" if self.after else ""
|
||||
return f"{quote}{before}{self.match}{after}{quote}"
|
||||
|
||||
|
||||
def build_context(
|
||||
timeline: Timeline, start_index: int, end_index: int, context_words: int = 50
|
||||
) -> SearchContext:
|
||||
"""Schneidet Kontext um den Treffer ``[start_index, end_index)`` aus."""
|
||||
tokens = timeline.tokens
|
||||
if not tokens:
|
||||
return SearchContext("", "", "", 0.0, 0.0, "segment")
|
||||
|
||||
start_index = max(0, min(start_index, len(tokens)))
|
||||
end_index = max(start_index + 1, min(end_index, len(tokens)))
|
||||
|
||||
before_start = max(0, start_index - max(0, context_words))
|
||||
after_end = min(len(tokens), end_index + max(0, context_words))
|
||||
|
||||
before_tokens = tokens[before_start:start_index]
|
||||
match_tokens = tokens[start_index:end_index]
|
||||
after_tokens = tokens[end_index:after_end]
|
||||
|
||||
precision = "word" if all(token.precision == "word" for token in match_tokens) else "segment"
|
||||
start_time = match_tokens[0].start
|
||||
end_time = match_tokens[-1].end
|
||||
|
||||
return SearchContext(
|
||||
before=" ".join(token.text for token in before_tokens),
|
||||
match=" ".join(token.text for token in match_tokens),
|
||||
after=" ".join(token.text for token in after_tokens),
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
precision=precision,
|
||||
)
|
||||
@@ -0,0 +1,303 @@
|
||||
"""Suchmaschine fuer SjAudioSearch.
|
||||
|
||||
Ablauf einer Suche:
|
||||
|
||||
1. Anfrage normalisieren.
|
||||
2. Kandidaten ueber FTS5 und den Trigramm-Index sammeln (schnelle Vorauswahl).
|
||||
3. Sliding-Window-Vergleich mit RapidFuzz auf der Wort-Timeline der Kandidaten.
|
||||
4. Optionaler, vollstaendiger Fallback-Lauf ueber alle verbleibenden Dateien,
|
||||
damit auch komplett falsch erkannte Woerter gefunden werden.
|
||||
5. Treffer entduplizieren, sortieren, begrenzen.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
|
||||
from rapidfuzz import fuzz
|
||||
|
||||
from ..config import SearchConfig
|
||||
from ..database.repository import Repository
|
||||
from ..utils.logging import get_logger
|
||||
from .context import SearchContext, Timeline, build_context, build_timeline
|
||||
from .fuzzy import (
|
||||
build_fts_query,
|
||||
normalize_text,
|
||||
score_window,
|
||||
significant_overlap,
|
||||
trigram_terms,
|
||||
)
|
||||
from .ranking import dedupe_overlapping, rank_hits
|
||||
|
||||
logger = get_logger("search.engine")
|
||||
|
||||
ProgressCallback = Callable[[int, int], None]
|
||||
|
||||
|
||||
class SearchError(RuntimeError):
|
||||
"""Fehler bei der Suche (z. B. leere Anfrage)."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchHit:
|
||||
"""Ein einzelner Suchtreffer."""
|
||||
|
||||
audio_id: int
|
||||
filename: str
|
||||
path: str
|
||||
score: float
|
||||
start_time: float
|
||||
end_time: float
|
||||
start_token: int
|
||||
end_token: int
|
||||
context: SearchContext
|
||||
language: str | None = None
|
||||
model: str | None = None
|
||||
source: str = "fts"
|
||||
|
||||
@property
|
||||
def matched_text(self) -> str:
|
||||
return self.context.match
|
||||
|
||||
def to_dict(self, context_words: int = 50) -> dict[str, object]:
|
||||
return {
|
||||
"audio_id": self.audio_id,
|
||||
"filename": self.filename,
|
||||
"path": self.path,
|
||||
"similarity": round(self.score, 2),
|
||||
"timestamp": self.start_time,
|
||||
"timestamp_end": self.end_time,
|
||||
"timestamp_precision": self.context.precision,
|
||||
"match": self.context.match,
|
||||
"context_before": self.context.before,
|
||||
"context_after": self.context.after,
|
||||
"context_words": context_words,
|
||||
"language": self.language,
|
||||
"model": self.model,
|
||||
"source": self.source,
|
||||
}
|
||||
|
||||
|
||||
class SearchEngine:
|
||||
"""Kombinierte Suche aus FTS5, Trigrammen und Fuzzy-Sliding-Window."""
|
||||
|
||||
def __init__(self, repository: Repository, config: SearchConfig):
|
||||
self.repo = repository
|
||||
self.config = config
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def search(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
limit: int | None = None,
|
||||
min_score: float | None = None,
|
||||
context_words: int | None = None,
|
||||
language: str | None = None,
|
||||
progress: ProgressCallback | None = None,
|
||||
) -> list[SearchHit]:
|
||||
query_normalized = normalize_text(query)
|
||||
if not query_normalized:
|
||||
raise SearchError("Die Suchanfrage enthaelt keine verwertbaren Woerter.")
|
||||
query_tokens = query_normalized.split()
|
||||
|
||||
limit = self.config.default_limit if limit is None else int(limit)
|
||||
min_score = self.config.default_min_score if min_score is None else float(min_score)
|
||||
context_words = self.config.context_words if context_words is None else int(context_words)
|
||||
if limit <= 0:
|
||||
raise SearchError("--limit muss groesser als 0 sein.")
|
||||
|
||||
candidate_audio, candidate_source = self._collect_candidates(query_tokens)
|
||||
hits: list[SearchHit] = []
|
||||
swept: set[int] = set()
|
||||
|
||||
total_candidates = len(candidate_audio)
|
||||
for index, audio_id in enumerate(candidate_audio, start=1):
|
||||
swept.add(audio_id)
|
||||
hits.extend(
|
||||
self._scan_audio(
|
||||
audio_id,
|
||||
query_normalized,
|
||||
query_tokens,
|
||||
min_score=min_score,
|
||||
context_words=context_words,
|
||||
language=language,
|
||||
source="fts",
|
||||
)
|
||||
)
|
||||
if progress:
|
||||
progress(index, total_candidates)
|
||||
|
||||
# Wichtig: FTS5 ist nur eine Vorauswahl. Wenn wenige Treffer gefunden
|
||||
# wurden, werden alle uebrigen Dateien vollstaendig durchsucht.
|
||||
fallback_used = False
|
||||
if self.config.exhaustive_fallback and len(hits) < limit:
|
||||
remaining = [
|
||||
audio_id
|
||||
for audio_id in self.repo.iter_audio_ids_with_transcripts(language)
|
||||
if audio_id not in swept
|
||||
]
|
||||
fallback_used = bool(remaining)
|
||||
for index, audio_id in enumerate(remaining, start=1):
|
||||
hits.extend(
|
||||
self._scan_audio(
|
||||
audio_id,
|
||||
query_normalized,
|
||||
query_tokens,
|
||||
min_score=min_score,
|
||||
context_words=context_words,
|
||||
language=language,
|
||||
source="fallback",
|
||||
)
|
||||
)
|
||||
if progress:
|
||||
progress(index, len(remaining))
|
||||
|
||||
if fallback_used:
|
||||
logger.info(
|
||||
"Fallback-Suche aktiv: %d Dateien ohne FTS-Treffer durchsucht", len(remaining)
|
||||
)
|
||||
|
||||
deduped = dedupe_overlapping(hits)
|
||||
return rank_hits(deduped, limit)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _collect_candidates(self, query_tokens: list[str]) -> tuple[list[int], str]:
|
||||
"""Sammelt Kandidaten-Audiodateien (FTS5 + Trigramme)."""
|
||||
limit = max(1, int(self.config.candidate_limit))
|
||||
matches: list[tuple[int, int]] = []
|
||||
|
||||
fts_query = build_fts_query(query_tokens)
|
||||
if fts_query:
|
||||
matches.extend(self.repo.fts_candidates(fts_query, limit))
|
||||
|
||||
remaining = limit - len(matches)
|
||||
if remaining > 0:
|
||||
terms = trigram_terms(query_tokens)
|
||||
if terms:
|
||||
matches.extend(self.repo.trigram_candidates(terms, remaining))
|
||||
|
||||
counts: dict[int, int] = {}
|
||||
for audio_id, _segment_id in matches:
|
||||
counts[audio_id] = counts.get(audio_id, 0) + 1
|
||||
|
||||
ordered = sorted(counts, key=lambda audio_id: (-counts[audio_id], audio_id))
|
||||
return ordered, "fts"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _scan_audio(
|
||||
self,
|
||||
audio_id: int,
|
||||
query_normalized: str,
|
||||
query_tokens: list[str],
|
||||
*,
|
||||
min_score: float,
|
||||
context_words: int,
|
||||
language: str | None,
|
||||
source: str,
|
||||
) -> list[SearchHit]:
|
||||
record = self.repo.get_audio(audio_id)
|
||||
if record is None or record.duplicate_of is not None:
|
||||
return []
|
||||
|
||||
transcript = self.repo.get_transcript(audio_id)
|
||||
if transcript is None:
|
||||
return []
|
||||
if language and (transcript["language"] or "").lower() != language.lower():
|
||||
return []
|
||||
|
||||
segments, words = self.repo.timeline(audio_id)
|
||||
if not segments:
|
||||
return []
|
||||
timeline = build_timeline(
|
||||
audio_id,
|
||||
segments,
|
||||
words,
|
||||
language=transcript["language"],
|
||||
model=transcript["model"],
|
||||
)
|
||||
if not timeline.tokens:
|
||||
return []
|
||||
|
||||
windows = self._find_windows(timeline, query_normalized, query_tokens, min_score)
|
||||
if not windows:
|
||||
return []
|
||||
|
||||
hits: list[SearchHit] = []
|
||||
for score, start_index, end_index in windows:
|
||||
context = build_context(timeline, start_index, end_index, context_words)
|
||||
hits.append(
|
||||
SearchHit(
|
||||
audio_id=audio_id,
|
||||
filename=record.filename,
|
||||
path=record.path,
|
||||
score=score,
|
||||
start_time=context.start_time,
|
||||
end_time=context.end_time,
|
||||
start_token=start_index,
|
||||
end_token=end_index,
|
||||
context=context,
|
||||
language=transcript["language"],
|
||||
model=transcript["model"],
|
||||
source=source,
|
||||
)
|
||||
)
|
||||
|
||||
hits.sort(key=lambda hit: -hit.score)
|
||||
return hits[: max(1, int(self.config.max_hits_per_file))]
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _find_windows(
|
||||
self,
|
||||
timeline: Timeline,
|
||||
query_normalized: str,
|
||||
query_tokens: list[str],
|
||||
min_score: float,
|
||||
) -> list[tuple[float, int, int]]:
|
||||
"""Sliding-Window-Suche ueber die Wort-Timeline einer Datei."""
|
||||
norm_tokens = timeline.normalized_tokens
|
||||
token_count = len(norm_tokens)
|
||||
query_len = len(query_tokens)
|
||||
if token_count == 0:
|
||||
return []
|
||||
|
||||
min_size = max(1, query_len - max(0, self.config.window_before))
|
||||
max_size = max(min_size, query_len + max(0, self.config.window_after))
|
||||
overlap_threshold = max(0.0, min(1.0, self.config.min_token_overlap))
|
||||
|
||||
# Dateiebene: gibt es ueberhaupt Beruehrungspunkte? Das ist deutlich
|
||||
# billiger als jede Datei Fenster fuer Fenster zu vergleichen.
|
||||
file_overlap = significant_overlap(query_tokens, norm_tokens)
|
||||
if file_overlap < overlap_threshold * 0.5:
|
||||
partial = fuzz.partial_ratio(query_normalized, timeline.normalized_text)
|
||||
if partial < 55.0:
|
||||
return []
|
||||
|
||||
windows: list[tuple[float, int, int]] = []
|
||||
# Wenn auf Dateiebene kaum Wortueberschneidung besteht, kann der Treffer
|
||||
# nur ueber einen falsch erkannten Ausdruck zustande kommen. Dann wird
|
||||
# ohne Wort-Vorfilter verglichen (selten und durch die Dateipruefung
|
||||
# oben bereits eingegrenzt).
|
||||
use_overlap_filter = file_overlap >= overlap_threshold
|
||||
for start in range(token_count):
|
||||
pre_end = min(token_count, start + query_len + 2)
|
||||
if use_overlap_filter and significant_overlap(query_tokens, norm_tokens[start:pre_end]) < overlap_threshold:
|
||||
continue
|
||||
|
||||
best_score = 0.0
|
||||
best_size = 0
|
||||
for size in range(min_size, max_size + 1):
|
||||
end = start + size
|
||||
if end > token_count:
|
||||
break
|
||||
window_normalized = " ".join(norm_tokens[start:end])
|
||||
candidate = score_window(query_normalized, window_normalized)
|
||||
if candidate.score > best_score:
|
||||
best_score = candidate.score
|
||||
best_size = size
|
||||
|
||||
if best_size and best_score >= min_score:
|
||||
windows.append((best_score, start, start + best_size))
|
||||
|
||||
return windows
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Textnormalisierung und Fuzzy-Vergleich (RapidFuzz).
|
||||
|
||||
Wichtig: Die Normalisierung wird ausschliesslich fuer Vergleiche und fuer den
|
||||
Suchindex verwendet. Der gespeicherte Originaltext bleibt unveraendert.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
from dataclasses import dataclass
|
||||
|
||||
from rapidfuzz import fuzz
|
||||
|
||||
# Satzzeichen und Symbole, die beim Vergleich stoeren. Wortzeichen (Buchstaben,
|
||||
# Ziffern, Unterstrich) bleiben erhalten.
|
||||
_PUNCT_RE = re.compile(r"[^\w\s]", re.UNICODE)
|
||||
_WHITESPACE_RE = re.compile(r"\s+", re.UNICODE)
|
||||
|
||||
# Sehr haeufige deutsche Fuellwoerter. Sie werden beim Vorfilter nicht als
|
||||
# belastbare Uebereinstimmung gewertet, damit "der/die/das" allein keine
|
||||
# Kandidaten erzeugt.
|
||||
STOPWORDS = frozenset(
|
||||
{
|
||||
"der", "die", "das", "den", "dem", "des", "ein", "eine", "einer", "eines",
|
||||
"einem", "einen", "und", "oder", "aber", "als", "auch", "auf", "aus", "bei",
|
||||
"bin", "bis", "da", "dann", "dass", "doch", "dort", "du", "durch", "er",
|
||||
"es", "hat", "ich", "ihm", "ihn", "ihr", "im", "in", "ist", "mit", "nach",
|
||||
"nicht", "noch", "nun", "nur", "ob", "so", "sich", "sie", "sind", "um",
|
||||
"und", "vom", "von", "vor", "war", "was", "wenn", "wie", "wir", "wo", "zu",
|
||||
"zum", "zur", "the", "and", "of", "to", "in", "a", "is",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def normalize_text(text: str) -> str:
|
||||
"""Normalisiert Text fuer Vergleiche: NFKC, Kleinschreibung, ohne Satzzeichen."""
|
||||
if not text:
|
||||
return ""
|
||||
normalized = unicodedata.normalize("NFKC", text).casefold()
|
||||
normalized = _PUNCT_RE.sub(" ", normalized)
|
||||
return _WHITESPACE_RE.sub(" ", normalized).strip()
|
||||
|
||||
|
||||
def tokenize(text: str) -> list[str]:
|
||||
"""Normalisierte Wortliste eines Textes."""
|
||||
normalized = normalize_text(text)
|
||||
return normalized.split() if normalized else []
|
||||
|
||||
|
||||
def tokenize_original(text: str) -> list[str]:
|
||||
"""Wortliste mit Originaltext inklusive anhängender Satzzeichen (fuer die Ausgabe)."""
|
||||
if not text:
|
||||
return []
|
||||
return [token for token in _WHITESPACE_RE.split(text.strip()) if token]
|
||||
|
||||
|
||||
def token_norm(token: str) -> str:
|
||||
"""Normalisiert ein einzelnes Originalwort."""
|
||||
return normalize_text(token)
|
||||
|
||||
|
||||
def is_stopword(token: str) -> bool:
|
||||
return token in STOPWORDS
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WindowScore:
|
||||
"""Ergebnis eines Fenstervergleichs."""
|
||||
|
||||
score: float
|
||||
exact: bool
|
||||
|
||||
|
||||
def score_window(query_normalized: str, window_normalized: str) -> WindowScore:
|
||||
"""Vergleicht Suchanfrage und Textfenster und liefert einen Wert 0-100.
|
||||
|
||||
Es werden mehrere RapidFuzz-Kennzahlen kombiniert, damit fehlende,
|
||||
zusaetzliche, vertauschte oder falsch erkannte Woerter nicht sofort zum
|
||||
Verlust des Treffers fuehren.
|
||||
"""
|
||||
if not query_normalized or not window_normalized:
|
||||
return WindowScore(0.0, False)
|
||||
|
||||
if query_normalized == window_normalized:
|
||||
return WindowScore(100.0, True)
|
||||
|
||||
# Hinweis: Eine reine Teilzeichenketten-Pruefung ist hier absichtlich NICHT
|
||||
# enthalten. Sie wuerde z. B. das Fenster "Leider" fuer die Anfrage
|
||||
# "Leider zu spaet" mit 100 bewerten und den Treffer auf ein Wort
|
||||
# zusammenfallen lassen. Teiluebereinstimmungen werden stattdessen ueber
|
||||
# die gewichteten RapidFuzz-Kennzahlen unten bewertet.
|
||||
|
||||
ratio = fuzz.ratio(query_normalized, window_normalized)
|
||||
token_set = fuzz.token_set_ratio(query_normalized, window_normalized)
|
||||
weighted = fuzz.WRatio(query_normalized, window_normalized)
|
||||
partial = fuzz.partial_ratio(query_normalized, window_normalized)
|
||||
|
||||
# partial_ratio neigt bei kurzen Fenstern zu hohen Werten, daher nur
|
||||
# gedämpft einbeziehen.
|
||||
# token_set_ratio/WRatio/partial_ratio bewerten eine Teilmenge der Anfrage
|
||||
# mit 100. Ohne Daempfung wuerde ein einzelnes Wort einer mehrwoertrigen
|
||||
# Anfrage als perfekter Treffer gelten. Daher werden diese Kennzahlen mit
|
||||
# dem Laengenverhaeltnis gewichtet, wenn das Fenster kuerzer ist.
|
||||
length_factor = min(1.0, len(window_normalized) / max(1, len(query_normalized)))
|
||||
|
||||
score = max(
|
||||
ratio,
|
||||
token_set * length_factor,
|
||||
weighted * length_factor,
|
||||
partial * 0.92 * length_factor,
|
||||
)
|
||||
return WindowScore(min(100.0, round(score, 2)), False)
|
||||
|
||||
|
||||
def token_overlap(query_tokens: list[str], window_tokens: list[str]) -> float:
|
||||
"""Anteil der Anfragewoerter, die im Fenster vorkommen (0..1)."""
|
||||
if not query_tokens:
|
||||
return 0.0
|
||||
window_set = set(window_tokens)
|
||||
hits = sum(1 for token in query_tokens if token in window_set)
|
||||
return hits / len(query_tokens)
|
||||
|
||||
|
||||
def significant_overlap(query_tokens: list[str], window_tokens: list[str]) -> float:
|
||||
"""Wie :func:`token_overlap`, aber Fuellwoerter zaehlen nur halb."""
|
||||
if not query_tokens:
|
||||
return 0.0
|
||||
window_set = set(window_tokens)
|
||||
total = 0.0
|
||||
hits = 0.0
|
||||
for token in query_tokens:
|
||||
weight = 0.5 if is_stopword(token) else 1.0
|
||||
total += weight
|
||||
if token in window_set:
|
||||
hits += weight
|
||||
if total == 0:
|
||||
return 0.0
|
||||
return hits / total
|
||||
|
||||
|
||||
def _escape_fts_token(token: str) -> str:
|
||||
return '"' + token.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def build_fts_query(tokens: list[str], prefix_min_length: int = 4) -> str:
|
||||
"""Baut eine FTS5-Anfrage aus ODER-verknuepften Woertern.
|
||||
|
||||
Praefixsuche fuer laengere Woerter faengt deutsche Flexionsformen und
|
||||
Zusammensetzungen ab. Die Anfrage ist bewusst weit gefasst: sie dient nur
|
||||
der Kandidatensuche, die Bewertung erfolgt spaeter per Fuzzy-Vergleich.
|
||||
"""
|
||||
parts: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for token in tokens:
|
||||
if len(token) < 2 or token in seen:
|
||||
continue
|
||||
seen.add(token)
|
||||
escaped = _escape_fts_token(token)
|
||||
if len(token) >= prefix_min_length:
|
||||
parts.append(f"{escaped}*")
|
||||
else:
|
||||
parts.append(escaped)
|
||||
return " OR ".join(parts)
|
||||
|
||||
|
||||
def trigram_terms(tokens: list[str], max_terms: int = 32) -> list[str]:
|
||||
"""Sinnvolle Trigramm-Anfragen aus laengeren Anfragewoertern."""
|
||||
terms: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for token in tokens:
|
||||
if len(token) < 4 or token in seen:
|
||||
continue
|
||||
seen.add(token)
|
||||
terms.append(token)
|
||||
if len(terms) >= max_terms:
|
||||
break
|
||||
return terms
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Bewertung und Sortierung von Suchtreffern.
|
||||
|
||||
Der Score ist eine Aehnlichkeitsbewertung (0-100) und ausdruecklich keine
|
||||
statistische Wahrscheinlichkeit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from .engine import SearchHit
|
||||
|
||||
|
||||
def clamp_score(score: float) -> float:
|
||||
return max(0.0, min(100.0, float(score)))
|
||||
|
||||
|
||||
def dedupe_overlapping(hits: list["SearchHit"], gap_tokens: int = 3) -> list["SearchHit"]:
|
||||
"""Entfernt Treffer, die denselben Textabschnitt mehrfach abdecken."""
|
||||
if not hits:
|
||||
return []
|
||||
|
||||
ordered = sorted(hits, key=lambda hit: (-hit.score, hit.start_token))
|
||||
kept: list["SearchHit"] = []
|
||||
for hit in ordered:
|
||||
overlapping = False
|
||||
for existing in kept:
|
||||
if existing.audio_id != hit.audio_id:
|
||||
continue
|
||||
same_region = (
|
||||
abs(existing.start_token - hit.start_token) <= gap_tokens
|
||||
or (hit.start_token < existing.end_token and existing.start_token < hit.end_token)
|
||||
)
|
||||
if same_region:
|
||||
overlapping = True
|
||||
break
|
||||
if not overlapping:
|
||||
kept.append(hit)
|
||||
return kept
|
||||
|
||||
|
||||
def rank_hits(hits: list["SearchHit"], limit: int) -> list["SearchHit"]:
|
||||
"""Sortiert Treffer nach Relevanz und begrenzt die Anzahl."""
|
||||
ordered = sorted(
|
||||
hits,
|
||||
key=lambda hit: (
|
||||
-hit.score,
|
||||
hit.filename.lower(),
|
||||
hit.start_time,
|
||||
),
|
||||
)
|
||||
return ordered[: max(0, int(limit))]
|
||||
@@ -0,0 +1,233 @@
|
||||
"""Systemdiagnose und verstaendlicher Systembericht."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.table import Table
|
||||
|
||||
from ..config import Config
|
||||
from ..utils.formatting import format_size
|
||||
from ..utils.logging import get_logger
|
||||
from .gpu import (
|
||||
GpuInfo,
|
||||
ResolvedExecution,
|
||||
RuntimeInfo,
|
||||
collect_runtime_info,
|
||||
detect_gpu,
|
||||
is_pascal_or_older,
|
||||
resolve_execution,
|
||||
)
|
||||
|
||||
logger = get_logger("system.diagnostics")
|
||||
|
||||
CHECK = "[green]OK[/green]"
|
||||
WARN = "[yellow]Warnung[/yellow]"
|
||||
FAIL = "[red]nicht verfuegbar[/red]"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SystemReport:
|
||||
"""Gesammelte Systeminformationen."""
|
||||
|
||||
gpu: GpuInfo | None
|
||||
runtime: RuntimeInfo
|
||||
execution: ResolvedExecution
|
||||
db_path: Path
|
||||
db_exists: bool
|
||||
config_path: str | None
|
||||
model_dir: Path
|
||||
model_dir_writable: bool
|
||||
ffmpeg: str | None
|
||||
ffplay: str | None
|
||||
notes: list[str] = field(default_factory=list)
|
||||
open_tasks: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def gpu_ready(self) -> bool:
|
||||
return self.runtime.cuda_runtime_usable and self.execution.device == "cuda"
|
||||
|
||||
@property
|
||||
def verdict(self) -> str:
|
||||
if self.gpu_ready:
|
||||
return (
|
||||
f"GPU-Transkription moeglich mit compute_type='{self.execution.compute_type}'."
|
||||
)
|
||||
return "Keine GPU-Transkription moeglich - CPU-Fallback wird verwendet."
|
||||
|
||||
|
||||
def _is_writable(path: Path) -> bool:
|
||||
target = path
|
||||
while not target.exists() and target != target.parent:
|
||||
target = target.parent
|
||||
return target.is_dir() and os.access(target, os.W_OK)
|
||||
|
||||
|
||||
def collect_report(config: Config, *, requested_device: str | None = None,
|
||||
requested_compute_type: str | None = None) -> SystemReport:
|
||||
"""Erstellt einen vollstaendigen Systembericht."""
|
||||
gpu = detect_gpu()
|
||||
runtime = collect_runtime_info(check_fts=True)
|
||||
|
||||
device = requested_device or config.transcription.device
|
||||
compute_type = requested_compute_type or config.transcription.compute_type
|
||||
execution = resolve_execution(device, compute_type, gpu, runtime)
|
||||
|
||||
db_path = config.database_path
|
||||
model_dir = config.model_path
|
||||
notes: list[str] = []
|
||||
open_tasks: list[str] = []
|
||||
|
||||
if gpu is not None and is_pascal_or_older(gpu):
|
||||
notes.append(
|
||||
f"{gpu.name} (Compute Capability {gpu.compute_capability}) ist eine Pascal-Karte: "
|
||||
"kein FP16/BF16, kein Tensor-Core-Betrieb. int8_float32 ist der passende Modus."
|
||||
)
|
||||
if runtime.cudnn_libraries:
|
||||
notes.append(
|
||||
f"cuDNN gefunden ({len(runtime.cudnn_libraries)} Bibliothek(en)); "
|
||||
"SjAudioSearch laedt die pip-nvidia-Bibliotheken beim Start automatisch vor."
|
||||
)
|
||||
else:
|
||||
notes.append(
|
||||
"Kein cuDNN gefunden. Fuer GPU-Betrieb die CUDA-12-Wheels installieren: "
|
||||
"pip install -e \".[cuda]\" (nvidia-cublas-cu12, nvidia-cudnn-cu12)."
|
||||
)
|
||||
if not runtime.fts5_available:
|
||||
open_tasks.append(
|
||||
"SQLite ohne FTS5 - bitte Python mit FTS5-Unterstuetzung installieren."
|
||||
)
|
||||
if not runtime.trigram_available:
|
||||
open_tasks.append(
|
||||
"Trigramm-Tokenizer nicht verfuegbar - die Kandidatensuche arbeitet nur ueber FTS5."
|
||||
)
|
||||
|
||||
return SystemReport(
|
||||
gpu=gpu,
|
||||
runtime=runtime,
|
||||
execution=execution,
|
||||
db_path=db_path,
|
||||
db_exists=db_path.exists(),
|
||||
config_path=str(config.source_path) if config.source_path else None,
|
||||
model_dir=model_dir,
|
||||
model_dir_writable=_is_writable(model_dir.parent if not model_dir.exists() else model_dir),
|
||||
ffmpeg=shutil.which("ffmpeg"),
|
||||
ffplay=shutil.which("ffplay"),
|
||||
notes=notes,
|
||||
open_tasks=open_tasks,
|
||||
)
|
||||
|
||||
|
||||
def render_report(report: SystemReport, console: Console) -> None:
|
||||
"""Gibt den Systembericht uebersichtlich im Terminal aus."""
|
||||
console.print(Panel.fit("[bold]SjAudioSearch - Systemdiagnose[/bold]", border_style="cyan"))
|
||||
|
||||
system = Table(show_header=False, box=None, padding=(0, 1))
|
||||
system.add_column(style="bold cyan", no_wrap=True)
|
||||
system.add_column()
|
||||
system.add_row("Python", report.runtime.python_version)
|
||||
system.add_row("System", report.runtime.platform)
|
||||
system.add_row("SQLite", report.runtime.sqlite_version)
|
||||
system.add_row("FTS5", CHECK if report.runtime.fts5_available else FAIL)
|
||||
system.add_row("FTS5-Trigramme", CHECK if report.runtime.trigram_available else WARN)
|
||||
system.add_row("CTranslate2", report.runtime.ctranslate2_version or FAIL)
|
||||
system.add_row("FFmpeg", report.ffmpeg or WARN)
|
||||
system.add_row("ffplay", report.ffplay or WARN)
|
||||
system.add_row("Konfiguration", report.config_path or "eingebaute Defaults")
|
||||
system.add_row("Datenbank", f"{report.db_path} ({'vorhanden' if report.db_exists else 'wird angelegt'})")
|
||||
system.add_row("Modellordner", f"{report.model_dir} ({'beschreibbar' if report.model_dir_writable else 'NICHT beschreibbar'})")
|
||||
console.print(system)
|
||||
|
||||
console.print()
|
||||
gpu_table = Table(title="GPU", show_header=False, box=None, padding=(0, 1))
|
||||
gpu_table.add_column(style="bold cyan", no_wrap=True)
|
||||
gpu_table.add_column()
|
||||
if report.gpu is None:
|
||||
gpu_table.add_row("Status", FAIL + " (nvidia-smi lieferte keine Daten)")
|
||||
else:
|
||||
gpu_table.add_row("Modell", report.gpu.name)
|
||||
gpu_table.add_row("Treiber", report.gpu.driver_version or "unbekannt")
|
||||
gpu_table.add_row("CUDA (Treiber)", report.gpu.cuda_driver_version or "unbekannt")
|
||||
gpu_table.add_row(
|
||||
"VRAM", f"{report.gpu.memory_total_mb} MB" if report.gpu.memory_total_mb else "unbekannt"
|
||||
)
|
||||
gpu_table.add_row("Compute Capability", report.gpu.compute_capability or "unbekannt")
|
||||
gpu_table.add_row("Architektur", "Pascal oder aelter" if is_pascal_or_older(report.gpu) else "neuere Architektur")
|
||||
gpu_table.add_row(
|
||||
"CUDA-Geraete (CTranslate2)", str(report.runtime.cuda_device_count)
|
||||
)
|
||||
gpu_table.add_row(
|
||||
"Compute Types (CUDA)",
|
||||
", ".join(report.runtime.cuda_compute_types) or "keine",
|
||||
)
|
||||
gpu_table.add_row(
|
||||
"Compute Types (CPU)",
|
||||
", ".join(report.runtime.cpu_compute_types) or "keine",
|
||||
)
|
||||
gpu_table.add_row(
|
||||
"CUDA-Bibliotheken",
|
||||
", ".join(Path(lib).name for lib in report.runtime.cuda_libraries) or "keine im System",
|
||||
)
|
||||
gpu_table.add_row(
|
||||
"cuDNN-Bibliotheken",
|
||||
", ".join(Path(lib).name for lib in report.runtime.cudnn_libraries) or "keine im System",
|
||||
)
|
||||
console.print(gpu_table)
|
||||
|
||||
console.print()
|
||||
plan = Table(title="Empfohlener Ausfuehrungsmodus", show_header=False, box=None, padding=(0, 1))
|
||||
plan.add_column(style="bold cyan", no_wrap=True)
|
||||
plan.add_column()
|
||||
plan.add_row("Angefragt", f"{report.execution.requested_device} / {report.execution.requested_compute_type}")
|
||||
plan.add_row("Verwendet", f"{report.execution.device} / {report.execution.compute_type}")
|
||||
plan.add_row("FP16/BF16", "nicht verwendet" if report.execution.device == "cuda" else "irrelevant (CPU)")
|
||||
console.print(plan)
|
||||
|
||||
for note in report.execution.notes:
|
||||
console.print(f"[yellow]-[/yellow] {note}")
|
||||
for note in report.notes:
|
||||
console.print(f"[cyan]-[/cyan] {note}")
|
||||
|
||||
console.print()
|
||||
color = "green" if report.gpu_ready else "yellow"
|
||||
console.print(Panel(report.verdict, border_style=color, title="Ergebnis"))
|
||||
|
||||
if report.open_tasks:
|
||||
console.print()
|
||||
console.print("[bold red]Offene Punkte:[/bold red]")
|
||||
for task in report.open_tasks:
|
||||
console.print(f" [red]*[/red] {task}")
|
||||
|
||||
|
||||
def database_summary(db_path: Path) -> dict[str, object] | None:
|
||||
"""Kurze Zusammenfassung einer vorhandenen Datenbank (fuer die Diagnose)."""
|
||||
if not db_path.exists():
|
||||
return None
|
||||
try:
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
try:
|
||||
tables = {
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type IN ('table','view')"
|
||||
)
|
||||
}
|
||||
audio_count = 0
|
||||
if "audio_files" in tables:
|
||||
audio_count = int(conn.execute("SELECT COUNT(*) FROM audio_files").fetchone()[0])
|
||||
return {
|
||||
"tables": sorted(tables),
|
||||
"audio_files": audio_count,
|
||||
"size": format_size(db_path.stat().st_size),
|
||||
}
|
||||
finally:
|
||||
conn.close()
|
||||
except sqlite3.Error as exc:
|
||||
logger.warning("Datenbank konnte nicht gelesen werden: %s", exc)
|
||||
return None
|
||||
@@ -0,0 +1,406 @@
|
||||
"""GPU-, CUDA- und CTranslate2-Diagnose.
|
||||
|
||||
Die GTX 1070 Ti ist eine Pascal-Karte (Compute Capability 6.1). Sie besitzt
|
||||
keine FP16-Tensor-Cores. Deshalb wird FP16/BF16 hier grundsaetzlich nicht als
|
||||
GPU-Rechenmodus vorausgesetzt; bevorzugt wird ``int8_float32``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes.util
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger("system.gpu")
|
||||
|
||||
# Pascal und aelter: kein FP16-Tensor-Betrieb auf Consumer-Karten.
|
||||
PASCAL_CONSUMER_MAX_CC = 6.2
|
||||
PREFERRED_COMPUTE_TYPE = "int8_float32"
|
||||
FORBIDDEN_GPU_COMPUTE_TYPES = ("float16", "bfloat16", "int8_bfloat16", "int8_float16")
|
||||
|
||||
|
||||
@dataclass
|
||||
class GpuInfo:
|
||||
"""Angaben aus ``nvidia-smi``."""
|
||||
|
||||
name: str
|
||||
driver_version: str | None = None
|
||||
cuda_driver_version: str | None = None
|
||||
memory_total_mb: int | None = None
|
||||
compute_capability: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class RuntimeInfo:
|
||||
"""Laufzeitinformationen zu CTranslate2, CUDA und SQLite."""
|
||||
|
||||
ctranslate2_version: str | None = None
|
||||
cuda_device_count: int = 0
|
||||
supported_compute_types: dict[str, list[str]] = field(default_factory=dict)
|
||||
cuda_libraries: list[str] = field(default_factory=list)
|
||||
cudnn_libraries: list[str] = field(default_factory=list)
|
||||
python_version: str = ""
|
||||
platform: str = ""
|
||||
sqlite_version: str = ""
|
||||
fts5_available: bool = False
|
||||
trigram_available: bool = False
|
||||
|
||||
@property
|
||||
def cuda_runtime_usable(self) -> bool:
|
||||
return self.cuda_device_count > 0 and bool(self.supported_compute_types.get("cuda"))
|
||||
|
||||
@property
|
||||
def cuda_compute_types(self) -> list[str]:
|
||||
return sorted(self.supported_compute_types.get("cuda", []))
|
||||
|
||||
@property
|
||||
def cpu_compute_types(self) -> list[str]:
|
||||
return sorted(self.supported_compute_types.get("cpu", []))
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResolvedExecution:
|
||||
"""Tatsaechlich verwendeter Ausfuehrungsmodus."""
|
||||
|
||||
device: str
|
||||
compute_type: str
|
||||
requested_device: str
|
||||
requested_compute_type: str
|
||||
notes: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def used_fallback(self) -> bool:
|
||||
return self.device != self.requested_device or self.compute_type != self.requested_compute_type
|
||||
|
||||
|
||||
def _run_nvidia_smi(args: list[str]) -> str | None:
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
["nvidia-smi", *args],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=15,
|
||||
check=False,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
logger.debug("nvidia-smi nicht ausfuehrbar: %s", exc)
|
||||
return None
|
||||
if completed.returncode != 0:
|
||||
return None
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def detect_gpu() -> GpuInfo | None:
|
||||
"""Ermittelt GPU-Modell, Treiber und Compute Capability."""
|
||||
output = _run_nvidia_smi(
|
||||
[
|
||||
"--query-gpu=name,driver_version,memory.total,compute_cap",
|
||||
"--format=csv,noheader",
|
||||
]
|
||||
)
|
||||
gpu: GpuInfo | None = None
|
||||
if output:
|
||||
first_line = output.strip().splitlines()[0] if output.strip() else ""
|
||||
parts = [part.strip() for part in first_line.split(",")]
|
||||
if len(parts) >= 4:
|
||||
memory_mb = None
|
||||
match = re.search(r"(\d+)", parts[2])
|
||||
if match:
|
||||
memory_mb = int(match.group(1))
|
||||
gpu = GpuInfo(
|
||||
name=parts[0],
|
||||
driver_version=parts[1] or None,
|
||||
memory_total_mb=memory_mb,
|
||||
compute_capability=parts[3] or None,
|
||||
)
|
||||
if gpu is None:
|
||||
return None
|
||||
|
||||
header = _run_nvidia_smi([]) or ""
|
||||
match = re.search(r"CUDA Version:\s*([0-9.]+)", header)
|
||||
if match:
|
||||
gpu.cuda_driver_version = match.group(1)
|
||||
return gpu
|
||||
|
||||
|
||||
_PRELOADED_LIBRARIES: list[str] = []
|
||||
|
||||
|
||||
def ensure_cuda_libraries() -> list[str]:
|
||||
"""Laedt CUDA-/cuDNN-Bibliotheken aus den pip-``nvidia``-Paketen.
|
||||
|
||||
CTranslate2 linkt dynamisch gegen ``libcublas.so.12`` und ``libcudnn.so.9``.
|
||||
Die PyPI-Wheels legen diese Bibliotheken unter ``site-packages/nvidia``
|
||||
ab, wo der dynamische Linker sie nicht von allein findet. Ein Preload mit
|
||||
``RTLD_GLOBAL`` macht sie verfuegbar, ohne systemweite Aenderungen an
|
||||
``/etc/ld.so.conf`` oder installierte CUDA-Versionen.
|
||||
|
||||
Muss vor dem ersten CUDA-Aufruf von CTranslate2 erfolgen.
|
||||
"""
|
||||
global _PRELOADED_LIBRARIES
|
||||
if _PRELOADED_LIBRARIES or os.name != "posix":
|
||||
return _PRELOADED_LIBRARIES
|
||||
|
||||
cuda_libs, cudnn_libs = detect_cuda_libraries()
|
||||
if not cuda_libs and not cudnn_libs:
|
||||
return _PRELOADED_LIBRARIES
|
||||
|
||||
order = ("libcudart", "libcublasLt", "libcublas", "libcudnn")
|
||||
|
||||
def sort_key(path: str) -> int:
|
||||
name = Path(path).name
|
||||
for index, prefix in enumerate(order):
|
||||
if name.startswith(prefix):
|
||||
return index
|
||||
return len(order)
|
||||
|
||||
import ctypes
|
||||
|
||||
loaded: list[str] = []
|
||||
for path in sorted(cuda_libs + cudnn_libs, key=sort_key):
|
||||
# cuDNN-Teilbibliotheken werden von libcudnn.so.9 selbst nachgeladen.
|
||||
name = Path(path).name
|
||||
if name.startswith("libcudnn") and not name.startswith("libcudnn.so"):
|
||||
continue
|
||||
try:
|
||||
ctypes.CDLL(path, mode=ctypes.RTLD_GLOBAL)
|
||||
loaded.append(path)
|
||||
except OSError as exc:
|
||||
logger.debug("Bibliothek konnte nicht vorgeladen werden (%s): %s", path, exc)
|
||||
|
||||
if loaded:
|
||||
directories = sorted({str(Path(path).parent) for path in loaded})
|
||||
existing = os.environ.get("LD_LIBRARY_PATH", "")
|
||||
os.environ["LD_LIBRARY_PATH"] = os.pathsep.join(
|
||||
directories + ([existing] if existing else [])
|
||||
)
|
||||
logger.debug("CUDA-Bibliotheken vorgeladen: %s", ", ".join(Path(p).name for p in loaded))
|
||||
|
||||
_PRELOADED_LIBRARIES = loaded
|
||||
return loaded
|
||||
|
||||
|
||||
def _find_libraries(patterns: list[str], directories: list[str]) -> list[str]:
|
||||
found: list[str] = []
|
||||
for pattern in patterns:
|
||||
for directory in directories:
|
||||
try:
|
||||
found.extend(glob.glob(os.path.join(directory, pattern)))
|
||||
except OSError:
|
||||
continue
|
||||
return sorted(set(found))
|
||||
|
||||
|
||||
def detect_cuda_libraries() -> tuple[list[str], list[str]]:
|
||||
"""Sucht CUDA- und cuDNN-Bibliotheken im System und in der venv."""
|
||||
directories = [
|
||||
"/usr/lib",
|
||||
"/usr/lib64",
|
||||
"/usr/local/cuda/lib64",
|
||||
"/usr/local/lib",
|
||||
"/opt/cuda/lib64",
|
||||
str(Path.cwd() / ".venv" / "lib"),
|
||||
]
|
||||
# nvidia-Wheels (falls installiert) liegen unter site-packages/nvidia/<paket>/lib
|
||||
import site
|
||||
import sys
|
||||
|
||||
site_roots: list[Path] = []
|
||||
for base in (os.environ.get("VIRTUAL_ENV"), os.environ.get("CONDA_PREFIX")):
|
||||
if base:
|
||||
site_roots.append(Path(base))
|
||||
site_roots.extend(Path(p) for p in site.getsitepackages())
|
||||
site_roots.append(Path(sys.prefix))
|
||||
|
||||
for base in site_roots:
|
||||
directories.extend([str(base / "lib"), str(base / "lib64")])
|
||||
nvidia_root = base / "lib" / f"python{sys.version_info.major}.{sys.version_info.minor}" / "site-packages" / "nvidia"
|
||||
directories.extend(str(path) for path in nvidia_root.glob("*/lib"))
|
||||
directories.extend(str(path) for path in (base / "Lib" / "site-packages" / "nvidia").glob("*/bin"))
|
||||
|
||||
cuda_patterns = ["libcudart.so*", "libcublas.so*", "libcublasLt.so*"]
|
||||
cudnn_patterns = ["libcudnn*.so*"]
|
||||
|
||||
cuda_libs = _find_libraries(cuda_patterns, directories)
|
||||
cudnn_libs = _find_libraries(cudnn_patterns, directories)
|
||||
|
||||
# Zusaetzlich ueber den dynamischen Linker suchen.
|
||||
for name in ("cudart", "cublas", "cudnn"):
|
||||
located = ctypes.util.find_library(name)
|
||||
if located and located not in cuda_libs and name != "cudnn":
|
||||
cuda_libs.append(located)
|
||||
if located and name == "cudnn" and located not in cudnn_libs:
|
||||
cudnn_libs.append(located)
|
||||
|
||||
return sorted(set(cuda_libs)), sorted(set(cudnn_libs))
|
||||
|
||||
|
||||
def probe_ctranslate2() -> tuple[str | None, int, dict[str, list[str]], str | None]:
|
||||
"""Fragt CTranslate2 nach Geraeten und unterstuetzten Compute Types."""
|
||||
try:
|
||||
import ctranslate2
|
||||
except ImportError as exc:
|
||||
return None, 0, {}, f"CTranslate2 nicht installiert: {exc}"
|
||||
|
||||
version = getattr(ctranslate2, "__version__", None)
|
||||
error: str | None = None
|
||||
|
||||
try:
|
||||
device_count = int(ctranslate2.get_cuda_device_count())
|
||||
except Exception as exc: # pragma: no cover - hardwareabhaengig
|
||||
device_count = 0
|
||||
error = f"CUDA-Geraete konnten nicht abgefragt werden: {exc}"
|
||||
|
||||
supported: dict[str, list[str]] = {}
|
||||
for device in ("cpu", "cuda"):
|
||||
try:
|
||||
supported[device] = sorted(ctranslate2.get_supported_compute_types(device))
|
||||
except Exception as exc:
|
||||
logger.debug("Compute Types fuer %s nicht verfuegbar: %s", device, exc)
|
||||
supported[device] = []
|
||||
|
||||
return version, device_count, supported, error
|
||||
|
||||
|
||||
def collect_runtime_info(*, check_fts: bool = True) -> RuntimeInfo:
|
||||
"""Sammelt alle Laufzeitinformationen fuer den Systembericht."""
|
||||
import platform
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
version, device_count, supported, error = probe_ctranslate2()
|
||||
cuda_libs, cudnn_libs = detect_cuda_libraries()
|
||||
|
||||
info = RuntimeInfo(
|
||||
ctranslate2_version=version,
|
||||
cuda_device_count=device_count,
|
||||
supported_compute_types=supported,
|
||||
cuda_libraries=cuda_libs,
|
||||
cudnn_libraries=cudnn_libs,
|
||||
python_version=sys.version.split()[0],
|
||||
platform=f"{platform.system()} {platform.release()}",
|
||||
sqlite_version=sqlite3.sqlite_version,
|
||||
)
|
||||
if error:
|
||||
logger.debug(error)
|
||||
|
||||
if check_fts:
|
||||
from ..database.connection import check_fts5, connect
|
||||
|
||||
conn = connect(":memory:")
|
||||
try:
|
||||
info.fts5_available, info.trigram_available = check_fts5(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def _cc_tuple(compute_capability: str | None) -> tuple[int, int] | None:
|
||||
if not compute_capability:
|
||||
return None
|
||||
match = re.match(r"^\s*(\d+)\.(\d+)", compute_capability)
|
||||
if not match:
|
||||
return None
|
||||
return int(match.group(1)), int(match.group(2))
|
||||
|
||||
|
||||
def is_pascal_or_older(gpu: GpuInfo | None) -> bool:
|
||||
"""True fuer Pascal und aeltere Architekturen (Compute Capability <= 6.x)."""
|
||||
if gpu is None:
|
||||
return False
|
||||
cc = _cc_tuple(gpu.compute_capability)
|
||||
if cc is None:
|
||||
return False
|
||||
return cc[0] <= 6
|
||||
|
||||
|
||||
def resolve_execution(
|
||||
requested_device: str,
|
||||
requested_compute_type: str,
|
||||
gpu: GpuInfo | None,
|
||||
runtime: RuntimeInfo,
|
||||
) -> ResolvedExecution:
|
||||
"""Waehlt einen tatsaechlich unterstuetzten Ausfuehrungsmodus.
|
||||
|
||||
Regeln:
|
||||
* Nie einen Compute Type verwenden, den CTranslate2 fuer das Geraet nicht meldet.
|
||||
* Auf Pascal kein FP16/BF16.
|
||||
* Wenn CUDA nicht nutzbar ist, automatisch CPU mit einer verstaendlichen Notiz.
|
||||
"""
|
||||
device = (requested_device or "auto").strip().lower()
|
||||
requested_compute = (requested_compute_type or "auto").strip().lower()
|
||||
notes: list[str] = []
|
||||
|
||||
if device in {"auto", ""}:
|
||||
device = "cuda" if runtime.cuda_runtime_usable else "cpu"
|
||||
notes.append(f"device=auto auf '{device}' aufgeloest.")
|
||||
|
||||
if device == "cuda" and not runtime.cuda_runtime_usable:
|
||||
notes.append(
|
||||
"CUDA ist nicht nutzbar (keine CUDA-Geraete oder keine CUDA-Compute-Types) - "
|
||||
"es wird die CPU verwendet."
|
||||
)
|
||||
device = "cpu"
|
||||
|
||||
if device == "cuda" and gpu is None:
|
||||
notes.append("nvidia-smi lieferte keine GPU-Informationen; CUDA wird trotzdem versucht.")
|
||||
|
||||
cc = _cc_tuple(gpu.compute_capability) if gpu else None
|
||||
if device == "cuda" and cc is not None and cc[0] <= 6:
|
||||
if requested_compute in FORBIDDEN_GPU_COMPUTE_TYPES:
|
||||
notes.append(
|
||||
f"'{requested_compute}' ist auf Pascal (CC {gpu.compute_capability}) nicht sinnvoll - "
|
||||
f"stattdessen {PREFERRED_COMPUTE_TYPE}."
|
||||
)
|
||||
requested_compute = PREFERRED_COMPUTE_TYPE
|
||||
|
||||
if device == "cpu":
|
||||
supported = runtime.cpu_compute_types or ["int8", "float32"]
|
||||
preference = [requested_compute, "int8", "int8_float32", "float32"]
|
||||
notes.append("CPU-Fallback aktiv: Transkription laeuft ohne GPU und ist deutlich langsamer.")
|
||||
else:
|
||||
supported = runtime.cuda_compute_types or []
|
||||
if gpu is not None and gpu.compute_capability:
|
||||
supported = [item for item in supported]
|
||||
# Pascal: FP16/BF16 nie automatisch waehlen.
|
||||
if cc is not None and cc[0] <= 6:
|
||||
supported = [item for item in supported if item not in FORBIDDEN_GPU_COMPUTE_TYPES]
|
||||
preference = [requested_compute, PREFERRED_COMPUTE_TYPE, "float32", "int8"]
|
||||
|
||||
chosen = ""
|
||||
for candidate in preference:
|
||||
if candidate in {"", "auto"}:
|
||||
continue
|
||||
if candidate in supported:
|
||||
chosen = candidate
|
||||
break
|
||||
|
||||
if not chosen:
|
||||
# Letzter Ausweg: erster gemeldeter Typ.
|
||||
chosen = supported[0] if supported else "float32"
|
||||
notes.append(
|
||||
f"Kein bevorzugter Compute Type verfuegbar; verwende '{chosen}'."
|
||||
)
|
||||
elif chosen != requested_compute and requested_compute not in {"", "auto"}:
|
||||
notes.append(
|
||||
f"'{requested_compute}' ist nicht verfuegbar - verwende '{chosen}'."
|
||||
)
|
||||
|
||||
if device == "cuda" and chosen in FORBIDDEN_GPU_COMPUTE_TYPES and cc is not None and cc[0] <= 6:
|
||||
chosen = PREFERRED_COMPUTE_TYPE
|
||||
notes.append("FP16/BF16 auf Pascal verhindert, int8_float32 erzwungen.")
|
||||
|
||||
return ResolvedExecution(
|
||||
device=device,
|
||||
compute_type=chosen,
|
||||
requested_device=requested_device,
|
||||
requested_compute_type=requested_compute_type,
|
||||
notes=notes,
|
||||
)
|
||||
@@ -0,0 +1,179 @@
|
||||
"""faster-whisper-Engine.
|
||||
|
||||
Das Modell wird genau einmal geladen und danach fuer alle Dateien
|
||||
wiederverwendet. Die Audiodateien werden von faster-whisper gestreamt
|
||||
dekodiert, es wird nie eine komplette Sammlung in den RAM geladen.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..config import TranscriptionConfig
|
||||
from ..utils.logging import get_logger
|
||||
from .models import TranscriptResult, TranscriptSegment, WordToken
|
||||
|
||||
logger = get_logger("transcription.engine")
|
||||
|
||||
|
||||
class TranscriptionError(RuntimeError):
|
||||
"""Fehler beim Laden des Modells oder beim Transkribieren."""
|
||||
|
||||
|
||||
class FasterWhisperEngine:
|
||||
"""Duenner Wrapper um :class:`faster_whisper.WhisperModel`."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings: TranscriptionConfig,
|
||||
*,
|
||||
device: str | None = None,
|
||||
compute_type: str | None = None,
|
||||
model_dir: str | Path | None = None,
|
||||
):
|
||||
self.settings = settings
|
||||
self.device = (device or settings.device).strip().lower()
|
||||
self.compute_type = (compute_type or settings.compute_type).strip().lower()
|
||||
self.model_dir = Path(model_dir) if model_dir else None
|
||||
self._model: Any | None = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
@property
|
||||
def model_name(self) -> str:
|
||||
return self.settings.model
|
||||
|
||||
@property
|
||||
def is_loaded(self) -> bool:
|
||||
return self._model is not None
|
||||
|
||||
@property
|
||||
def language(self) -> str | None:
|
||||
return self.settings.language_for_whisper
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def load(self) -> None:
|
||||
"""Laedt das Modell (idempotent)."""
|
||||
if self._model is not None:
|
||||
return
|
||||
|
||||
# CUDA-Bibliotheken aus den pip-nvidia-Paketen verfuegbar machen, bevor
|
||||
# CTranslate2 sie laedt (sonst: "libcublas.so.12 is not found").
|
||||
if self.device == "cuda":
|
||||
from ..system.gpu import ensure_cuda_libraries
|
||||
|
||||
ensure_cuda_libraries()
|
||||
|
||||
try:
|
||||
from faster_whisper import WhisperModel
|
||||
except ImportError as exc: # pragma: no cover - Installationsthema
|
||||
raise TranscriptionError(
|
||||
"faster-whisper ist nicht installiert. Bitte die Projektabhaengigkeiten installieren."
|
||||
) from exc
|
||||
|
||||
download_root = str(self.model_dir) if self.model_dir else None
|
||||
if download_root:
|
||||
Path(download_root).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info(
|
||||
"Lade Whisper-Modell %s (device=%s, compute_type=%s)",
|
||||
self.model_name,
|
||||
self.device,
|
||||
self.compute_type,
|
||||
)
|
||||
try:
|
||||
self._model = WhisperModel(
|
||||
self.model_name,
|
||||
device=self.device,
|
||||
compute_type=self.compute_type,
|
||||
download_root=download_root,
|
||||
)
|
||||
except Exception as exc: # CTranslate2 wirft diverse RuntimeError
|
||||
raise TranscriptionError(
|
||||
f"Modell '{self.model_name}' konnte nicht geladen werden "
|
||||
f"(device={self.device}, compute_type={self.compute_type}): {exc}"
|
||||
) from exc
|
||||
|
||||
def unload(self) -> None:
|
||||
self._model = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def transcribe(self, path: str | Path) -> TranscriptResult:
|
||||
"""Transkribiert eine Audiodatei vollstaendig."""
|
||||
if self._model is None:
|
||||
self.load()
|
||||
assert self._model is not None
|
||||
|
||||
settings = self.settings
|
||||
transcribe_kwargs: dict[str, Any] = {
|
||||
"language": settings.language_for_whisper,
|
||||
"beam_size": max(1, int(settings.beam_size)),
|
||||
"word_timestamps": bool(settings.word_timestamps),
|
||||
"vad_filter": bool(settings.vad_filter),
|
||||
# Dialogpassagen nicht durch aggressive Stille-Erkennung abschneiden.
|
||||
"condition_on_previous_text": False,
|
||||
}
|
||||
if settings.vad_filter:
|
||||
transcribe_kwargs["vad_parameters"] = {
|
||||
"min_silence_duration_ms": 700,
|
||||
"speech_pad_ms": 300,
|
||||
}
|
||||
|
||||
try:
|
||||
segments_iter, info = self._model.transcribe(str(path), **transcribe_kwargs)
|
||||
except Exception as exc:
|
||||
raise TranscriptionError(f"Transkription fehlgeschlagen: {exc}") from exc
|
||||
|
||||
segments: list[TranscriptSegment] = []
|
||||
text_parts: list[str] = []
|
||||
|
||||
try:
|
||||
for segment in segments_iter:
|
||||
words: list[WordToken] = []
|
||||
for word in getattr(segment, "words", None) or []:
|
||||
words.append(
|
||||
WordToken(
|
||||
word=word.word,
|
||||
start=float(word.start),
|
||||
end=float(word.end),
|
||||
probability=getattr(word, "probability", None),
|
||||
)
|
||||
)
|
||||
confidence = None
|
||||
avg_logprob = getattr(segment, "avg_logprob", None)
|
||||
if avg_logprob is not None:
|
||||
# Nur als interne Gueteinformation, keine Wahrscheinlichkeit
|
||||
# im Sinne der Suche.
|
||||
try:
|
||||
confidence = float(math.exp(float(avg_logprob)))
|
||||
except (OverflowError, ValueError):
|
||||
confidence = None
|
||||
|
||||
text = (segment.text or "").strip()
|
||||
if text:
|
||||
text_parts.append(text)
|
||||
segments.append(
|
||||
TranscriptSegment(
|
||||
text=text,
|
||||
start=float(segment.start),
|
||||
end=float(segment.end),
|
||||
confidence=confidence,
|
||||
words=words,
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
raise TranscriptionError(f"Fehler beim Lesen der Segmente: {exc}") from exc
|
||||
|
||||
full_text = " ".join(part for part in text_parts if part).strip()
|
||||
duration = getattr(info, "duration", None)
|
||||
language = getattr(info, "language", None) or settings.language_for_whisper
|
||||
|
||||
return TranscriptResult(
|
||||
segments=segments,
|
||||
full_text=full_text,
|
||||
language=language,
|
||||
language_probability=getattr(info, "language_probability", None),
|
||||
duration=float(duration) if duration else None,
|
||||
model=self.model_name,
|
||||
)
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Datenmodelle fuer Transkriptionsergebnisse."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class WordToken:
|
||||
"""Ein Wort mit Zeitstempel aus Whisper."""
|
||||
|
||||
word: str
|
||||
start: float
|
||||
end: float
|
||||
probability: float | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class TranscriptSegment:
|
||||
"""Ein Whisper-Segment."""
|
||||
|
||||
text: str
|
||||
start: float
|
||||
end: float
|
||||
confidence: float | None = None
|
||||
words: list[WordToken] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TranscriptResult:
|
||||
"""Vollstaendiges Transkript einer Audiodatei."""
|
||||
|
||||
segments: list[TranscriptSegment]
|
||||
full_text: str
|
||||
language: str | None = None
|
||||
language_probability: float | None = None
|
||||
duration: float | None = None
|
||||
model: str = ""
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.segments)
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Transkriptionswarteschlange.
|
||||
|
||||
Eine Datei nach der anderen, Modell nur einmal geladen, Fehler einer Datei
|
||||
brechen den Lauf nicht ab. Der Status wird dauerhaft gespeichert, sodass ein
|
||||
Neustart an der letzten Stelle fortsetzt.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable, Protocol
|
||||
|
||||
from ..config import ProcessingConfig
|
||||
from ..core.models import AudioFileRecord, FileStatus
|
||||
from ..database.repository import Repository
|
||||
from ..utils.logging import get_logger
|
||||
from .models import TranscriptResult
|
||||
|
||||
logger = get_logger("transcription.worker")
|
||||
|
||||
|
||||
class EngineProtocol(Protocol):
|
||||
"""Minimal benoetigte Engine-Schnittstelle (fuer Tests austauschbar)."""
|
||||
|
||||
model_name: str
|
||||
|
||||
def transcribe(self, path: str | Path) -> TranscriptResult: ...
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkerEvent:
|
||||
"""Ereignis fuer die Fortschrittsanzeige."""
|
||||
|
||||
kind: str # start | done | error | skipped | finished
|
||||
record: AudioFileRecord | None = None
|
||||
message: str | None = None
|
||||
elapsed: float = 0.0
|
||||
audio_duration: float | None = None
|
||||
stats: "RunStats | None" = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunStats:
|
||||
"""Zusammenfassung eines Transkriptionslaufs."""
|
||||
|
||||
queued: int = 0
|
||||
processed: int = 0
|
||||
failed: int = 0
|
||||
skipped: int = 0
|
||||
audio_seconds: float = 0.0
|
||||
elapsed: float = 0.0
|
||||
model: str = ""
|
||||
device: str = ""
|
||||
compute_type: str = ""
|
||||
errors: list[tuple[str, str]] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def rate(self) -> float:
|
||||
if self.elapsed <= 0:
|
||||
return 0.0
|
||||
return self.audio_seconds / self.elapsed
|
||||
|
||||
|
||||
class TranscriptionWorker:
|
||||
"""Verarbeitet die Transkriptionswarteschlange."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repository: Repository,
|
||||
engine: EngineProtocol,
|
||||
processing: ProcessingConfig | None = None,
|
||||
*,
|
||||
device: str = "",
|
||||
compute_type: str = "",
|
||||
):
|
||||
self.repo = repository
|
||||
self.engine = engine
|
||||
self.processing = processing or ProcessingConfig()
|
||||
self.stats = RunStats(
|
||||
model=getattr(engine, "model_name", ""),
|
||||
device=device,
|
||||
compute_type=compute_type,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def run(
|
||||
self,
|
||||
*,
|
||||
limit: int | None = None,
|
||||
force: bool = False,
|
||||
event_callback: Callable[[WorkerEvent], None] | None = None,
|
||||
) -> RunStats:
|
||||
"""Arbeitet die Warteschlange ab."""
|
||||
if self.processing.resume:
|
||||
reset = self.repo.reset_stale_processing()
|
||||
if reset:
|
||||
logger.info("%d unterbrochene Eintraege zurueckgesetzt", reset)
|
||||
|
||||
queue = self.repo.transcription_queue(force=force)
|
||||
if limit is not None:
|
||||
queue = queue[: max(0, int(limit))]
|
||||
self.stats.queued = len(queue)
|
||||
|
||||
started_at = time.monotonic()
|
||||
try:
|
||||
for record in queue:
|
||||
self._process(record, event_callback)
|
||||
except KeyboardInterrupt:
|
||||
logger.warning("Transkription durch Benutzer abgebrochen")
|
||||
raise
|
||||
finally:
|
||||
self.stats.elapsed = time.monotonic() - started_at
|
||||
|
||||
if event_callback:
|
||||
event_callback(WorkerEvent(kind="finished", stats=self.stats))
|
||||
|
||||
logger.info(
|
||||
"Transkription beendet: %d verarbeitet, %d Fehler, %d uebersprungen, "
|
||||
"%.1f s Audio in %.1f s (%.2fx Echtzeit)",
|
||||
self.stats.processed,
|
||||
self.stats.failed,
|
||||
self.stats.skipped,
|
||||
self.stats.audio_seconds,
|
||||
self.stats.elapsed,
|
||||
self.stats.rate,
|
||||
)
|
||||
return self.stats
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _process(
|
||||
self, record: AudioFileRecord, event_callback: Callable[[WorkerEvent], None] | None
|
||||
) -> None:
|
||||
path = Path(record.path)
|
||||
if not path.is_file():
|
||||
message = f"Datei nicht gefunden: {record.path}"
|
||||
self.repo.set_status(record.id, FileStatus.ERROR.value, message)
|
||||
self.stats.failed += 1
|
||||
self.stats.errors.append((record.path, message))
|
||||
if event_callback:
|
||||
event_callback(WorkerEvent(kind="error", record=record, message=message))
|
||||
return
|
||||
|
||||
self.repo.set_status(record.id, FileStatus.PROCESSING.value)
|
||||
if event_callback:
|
||||
event_callback(WorkerEvent(kind="start", record=record))
|
||||
|
||||
file_started = time.monotonic()
|
||||
try:
|
||||
result = self.engine.transcribe(path)
|
||||
if not result.segments:
|
||||
raise ValueError("Keine Sprache im Transkript erkannt")
|
||||
|
||||
self.repo.save_transcript(
|
||||
record.id,
|
||||
language=result.language,
|
||||
model=result.model or self.engine.model_name,
|
||||
full_text=result.full_text,
|
||||
segments=result.segments,
|
||||
duration=result.duration or record.duration,
|
||||
)
|
||||
except KeyboardInterrupt:
|
||||
self.repo.set_status(record.id, FileStatus.PENDING.value)
|
||||
raise
|
||||
except Exception as exc:
|
||||
elapsed = time.monotonic() - file_started
|
||||
message = f"{type(exc).__name__}: {exc}"
|
||||
logger.exception("Transkription fehlgeschlagen fuer %s", record.path)
|
||||
self.repo.set_status(record.id, FileStatus.ERROR.value, message)
|
||||
self.stats.failed += 1
|
||||
self.stats.errors.append((record.path, message))
|
||||
if event_callback:
|
||||
event_callback(
|
||||
WorkerEvent(
|
||||
kind="error",
|
||||
record=record,
|
||||
message=message,
|
||||
elapsed=elapsed,
|
||||
audio_duration=record.duration,
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
elapsed = time.monotonic() - file_started
|
||||
audio_duration = result.duration or record.duration or 0.0
|
||||
self.stats.processed += 1
|
||||
self.stats.audio_seconds += float(audio_duration or 0.0)
|
||||
logger.info(
|
||||
"Fertig: %s (%.1f s Audio, %.1f s Laufzeit, %d Segmente)",
|
||||
record.filename,
|
||||
audio_duration or 0.0,
|
||||
elapsed,
|
||||
len(result.segments),
|
||||
)
|
||||
if event_callback:
|
||||
event_callback(
|
||||
WorkerEvent(
|
||||
kind="done",
|
||||
record=record,
|
||||
elapsed=elapsed,
|
||||
audio_duration=audio_duration,
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Formatierungshelfer fuer Zeitangaben, Groessen und Pfade."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def format_timestamp(seconds: float | int | None) -> str:
|
||||
"""Formatiert Sekunden als HH:MM:SS bzw. MM:SS."""
|
||||
if seconds is None:
|
||||
return "--:--"
|
||||
seconds = max(0, int(round(float(seconds))))
|
||||
hours, remainder = divmod(seconds, 3600)
|
||||
minutes, secs = divmod(remainder, 60)
|
||||
if hours:
|
||||
return f"{hours:02d}:{minutes:02d}:{secs:02d}"
|
||||
return f"{minutes:02d}:{secs:02d}"
|
||||
|
||||
|
||||
def format_timestamp_verbose(seconds: float | int | None) -> str:
|
||||
"""Formatiert Sekunden immer als HH:MM:SS (fuer Fundstellen in Hoerspielen)."""
|
||||
if seconds is None:
|
||||
return "--:--:--"
|
||||
seconds = max(0, int(round(float(seconds))))
|
||||
hours, remainder = divmod(seconds, 3600)
|
||||
minutes, secs = divmod(remainder, 60)
|
||||
return f"{hours:02d}:{minutes:02d}:{secs:02d}"
|
||||
|
||||
|
||||
def format_duration(seconds: float | int | None) -> str:
|
||||
"""Menschlich lesbare Dauer, z. B. '2 h 14 min'."""
|
||||
if seconds is None:
|
||||
return "unbekannt"
|
||||
total = max(0, int(round(float(seconds))))
|
||||
hours, remainder = divmod(total, 3600)
|
||||
minutes, secs = divmod(remainder, 60)
|
||||
if hours:
|
||||
return f"{hours} h {minutes:02d} min"
|
||||
if minutes:
|
||||
return f"{minutes} min {secs:02d} s"
|
||||
return f"{secs} s"
|
||||
|
||||
|
||||
def format_size(num_bytes: int | float | None) -> str:
|
||||
"""Formatiert Bytes als B/KB/MB/GB."""
|
||||
if num_bytes is None:
|
||||
return "?"
|
||||
value = float(num_bytes)
|
||||
for unit in ("B", "KB", "MB", "GB", "TB"):
|
||||
if value < 1024 or unit == "TB":
|
||||
if unit == "B":
|
||||
return f"{int(value)} {unit}"
|
||||
return f"{value:.1f} {unit}"
|
||||
value /= 1024
|
||||
return f"{value:.1f} TB"
|
||||
|
||||
|
||||
def format_score(score: float) -> str:
|
||||
"""Aehnlichkeitswert als Prozentangabe (keine statistische Wahrscheinlichkeit)."""
|
||||
return f"{score:.1f}%"
|
||||
|
||||
|
||||
def format_rate(audio_seconds: float, elapsed_seconds: float) -> str:
|
||||
"""Verarbeitungsgeschwindigkeit als Vielfaches der Echtzeit."""
|
||||
if elapsed_seconds <= 0:
|
||||
return "-"
|
||||
return f"{audio_seconds / elapsed_seconds:.2f}x Echtzeit"
|
||||
|
||||
|
||||
def format_datetime(iso_timestamp: str | None) -> str:
|
||||
"""Formatiert einen ISO-String lesbar; unbekannte Formate werden durchgereicht."""
|
||||
if not iso_timestamp:
|
||||
return "-"
|
||||
try:
|
||||
dt = datetime.fromisoformat(iso_timestamp)
|
||||
except ValueError:
|
||||
return iso_timestamp
|
||||
if dt.tzinfo is not None:
|
||||
dt = dt.astimezone()
|
||||
return dt.strftime("%Y-%m-%d %H:%M")
|
||||
|
||||
|
||||
def utc_now_iso() -> str:
|
||||
"""Aktueller Zeitpunkt als ISO-String in UTC (sekundengenau)."""
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def shorten_path(path: str | Path, max_length: int = 80) -> str:
|
||||
"""Kuerzt lange Pfade in der Mitte."""
|
||||
text = str(path)
|
||||
if len(text) <= max_length:
|
||||
return text
|
||||
keep = max(8, (max_length - 3) // 2)
|
||||
return f"{text[:keep]}...{text[-keep:]}"
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Hash-Helfer fuer die Duplikaterkennung."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
CHUNK_SIZE = 1024 * 1024
|
||||
|
||||
|
||||
def sha256_file(path: str | Path, chunk_size: int = CHUNK_SIZE) -> str:
|
||||
"""Berechnet den SHA256-Hash einer Datei, ohne sie komplett in den RAM zu laden."""
|
||||
digest = hashlib.sha256()
|
||||
with Path(path).open("rb") as handle:
|
||||
while True:
|
||||
chunk = handle.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def quick_signature(path: str | Path, sample_size: int = 64 * 1024) -> str:
|
||||
"""Schnelle Signatur aus Groesse, erster und letzter Datenportion.
|
||||
|
||||
Nur fuer Vorpruefungen gedacht - fuer belastbare Duplikaterkennung muss
|
||||
:func:`sha256_file` verwendet werden.
|
||||
"""
|
||||
path = Path(path)
|
||||
size = path.stat().st_size
|
||||
digest = hashlib.sha256()
|
||||
digest.update(str(size).encode("ascii"))
|
||||
with path.open("rb") as handle:
|
||||
digest.update(handle.read(sample_size))
|
||||
if size > sample_size:
|
||||
handle.seek(max(sample_size, size - sample_size))
|
||||
digest.update(handle.read(sample_size))
|
||||
return digest.hexdigest()
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Logging-Setup fuer SjAudioSearch.
|
||||
|
||||
Protokolliert in eine rotierende Logdatei und optional zusaetzlich auf die
|
||||
Konsole (Rich). Die CLI schaltet den Konsolen-Handler nur bei Bedarf ein.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from pathlib import Path
|
||||
|
||||
from rich.logging import RichHandler
|
||||
|
||||
LOGGER_NAME = "sjaudiosearch"
|
||||
_CONFIGURED = False
|
||||
|
||||
|
||||
def setup_logging(log_path: Path, level: str = "INFO", console: bool = False) -> logging.Logger:
|
||||
"""Konfiguriert den Logger ``sjaudiosearch`` idempotent."""
|
||||
global _CONFIGURED
|
||||
|
||||
logger = logging.getLogger(LOGGER_NAME)
|
||||
numeric_level = getattr(logging, str(level).upper(), logging.INFO)
|
||||
logger.setLevel(numeric_level)
|
||||
|
||||
if _CONFIGURED:
|
||||
return logger
|
||||
|
||||
log_path = Path(log_path)
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
formatter = logging.Formatter(
|
||||
"%(asctime)s | %(levelname)-8s | %(name)s | %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
|
||||
file_handler = RotatingFileHandler(
|
||||
log_path,
|
||||
maxBytes=5 * 1024 * 1024,
|
||||
backupCount=3,
|
||||
encoding="utf-8",
|
||||
)
|
||||
file_handler.setFormatter(formatter)
|
||||
file_handler.setLevel(numeric_level)
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
if console:
|
||||
console_handler = RichHandler(show_path=False, rich_tracebacks=True, markup=False)
|
||||
console_handler.setLevel(max(numeric_level, logging.INFO))
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
logger.propagate = False
|
||||
_CONFIGURED = True
|
||||
return logger
|
||||
|
||||
|
||||
def get_logger(name: str | None = None) -> logging.Logger:
|
||||
"""Liefert einen Sub-Logger unterhalb von ``sjaudiosearch``."""
|
||||
if name:
|
||||
return logging.getLogger(f"{LOGGER_NAME}.{name}")
|
||||
return logging.getLogger(LOGGER_NAME)
|
||||
Reference in New Issue
Block a user