Initial commit

This commit is contained in:
2026-11-16 16:29:32 +01:00
commit 97e5c9ef4d
37 changed files with 5932 additions and 0 deletions
+179
View File
@@ -0,0 +1,179 @@
"""faster-whisper-Engine.
Das Modell wird genau einmal geladen und danach fuer alle Dateien
wiederverwendet. Die Audiodateien werden von faster-whisper gestreamt
dekodiert, es wird nie eine komplette Sammlung in den RAM geladen.
"""
from __future__ import annotations
import math
from pathlib import Path
from typing import Any
from ..config import TranscriptionConfig
from ..utils.logging import get_logger
from .models import TranscriptResult, TranscriptSegment, WordToken
logger = get_logger("transcription.engine")
class TranscriptionError(RuntimeError):
"""Fehler beim Laden des Modells oder beim Transkribieren."""
class FasterWhisperEngine:
"""Duenner Wrapper um :class:`faster_whisper.WhisperModel`."""
def __init__(
self,
settings: TranscriptionConfig,
*,
device: str | None = None,
compute_type: str | None = None,
model_dir: str | Path | None = None,
):
self.settings = settings
self.device = (device or settings.device).strip().lower()
self.compute_type = (compute_type or settings.compute_type).strip().lower()
self.model_dir = Path(model_dir) if model_dir else None
self._model: Any | None = None
# ------------------------------------------------------------------
@property
def model_name(self) -> str:
return self.settings.model
@property
def is_loaded(self) -> bool:
return self._model is not None
@property
def language(self) -> str | None:
return self.settings.language_for_whisper
# ------------------------------------------------------------------
def load(self) -> None:
"""Laedt das Modell (idempotent)."""
if self._model is not None:
return
# CUDA-Bibliotheken aus den pip-nvidia-Paketen verfuegbar machen, bevor
# CTranslate2 sie laedt (sonst: "libcublas.so.12 is not found").
if self.device == "cuda":
from ..system.gpu import ensure_cuda_libraries
ensure_cuda_libraries()
try:
from faster_whisper import WhisperModel
except ImportError as exc: # pragma: no cover - Installationsthema
raise TranscriptionError(
"faster-whisper ist nicht installiert. Bitte die Projektabhaengigkeiten installieren."
) from exc
download_root = str(self.model_dir) if self.model_dir else None
if download_root:
Path(download_root).mkdir(parents=True, exist_ok=True)
logger.info(
"Lade Whisper-Modell %s (device=%s, compute_type=%s)",
self.model_name,
self.device,
self.compute_type,
)
try:
self._model = WhisperModel(
self.model_name,
device=self.device,
compute_type=self.compute_type,
download_root=download_root,
)
except Exception as exc: # CTranslate2 wirft diverse RuntimeError
raise TranscriptionError(
f"Modell '{self.model_name}' konnte nicht geladen werden "
f"(device={self.device}, compute_type={self.compute_type}): {exc}"
) from exc
def unload(self) -> None:
self._model = None
# ------------------------------------------------------------------
def transcribe(self, path: str | Path) -> TranscriptResult:
"""Transkribiert eine Audiodatei vollstaendig."""
if self._model is None:
self.load()
assert self._model is not None
settings = self.settings
transcribe_kwargs: dict[str, Any] = {
"language": settings.language_for_whisper,
"beam_size": max(1, int(settings.beam_size)),
"word_timestamps": bool(settings.word_timestamps),
"vad_filter": bool(settings.vad_filter),
# Dialogpassagen nicht durch aggressive Stille-Erkennung abschneiden.
"condition_on_previous_text": False,
}
if settings.vad_filter:
transcribe_kwargs["vad_parameters"] = {
"min_silence_duration_ms": 700,
"speech_pad_ms": 300,
}
try:
segments_iter, info = self._model.transcribe(str(path), **transcribe_kwargs)
except Exception as exc:
raise TranscriptionError(f"Transkription fehlgeschlagen: {exc}") from exc
segments: list[TranscriptSegment] = []
text_parts: list[str] = []
try:
for segment in segments_iter:
words: list[WordToken] = []
for word in getattr(segment, "words", None) or []:
words.append(
WordToken(
word=word.word,
start=float(word.start),
end=float(word.end),
probability=getattr(word, "probability", None),
)
)
confidence = None
avg_logprob = getattr(segment, "avg_logprob", None)
if avg_logprob is not None:
# Nur als interne Gueteinformation, keine Wahrscheinlichkeit
# im Sinne der Suche.
try:
confidence = float(math.exp(float(avg_logprob)))
except (OverflowError, ValueError):
confidence = None
text = (segment.text or "").strip()
if text:
text_parts.append(text)
segments.append(
TranscriptSegment(
text=text,
start=float(segment.start),
end=float(segment.end),
confidence=confidence,
words=words,
)
)
except Exception as exc:
raise TranscriptionError(f"Fehler beim Lesen der Segmente: {exc}") from exc
full_text = " ".join(part for part in text_parts if part).strip()
duration = getattr(info, "duration", None)
language = getattr(info, "language", None) or settings.language_for_whisper
return TranscriptResult(
segments=segments,
full_text=full_text,
language=language,
language_probability=getattr(info, "language_probability", None),
duration=float(duration) if duration else None,
model=self.model_name,
)
+41
View File
@@ -0,0 +1,41 @@
"""Datenmodelle fuer Transkriptionsergebnisse."""
from __future__ import annotations
from dataclasses import dataclass, field
@dataclass
class WordToken:
"""Ein Wort mit Zeitstempel aus Whisper."""
word: str
start: float
end: float
probability: float | None = None
@dataclass
class TranscriptSegment:
"""Ein Whisper-Segment."""
text: str
start: float
end: float
confidence: float | None = None
words: list[WordToken] = field(default_factory=list)
@dataclass
class TranscriptResult:
"""Vollstaendiges Transkript einer Audiodatei."""
segments: list[TranscriptSegment]
full_text: str
language: str | None = None
language_probability: float | None = None
duration: float | None = None
model: str = ""
def __len__(self) -> int:
return len(self.segments)
+204
View File
@@ -0,0 +1,204 @@
"""Transkriptionswarteschlange.
Eine Datei nach der anderen, Modell nur einmal geladen, Fehler einer Datei
brechen den Lauf nicht ab. Der Status wird dauerhaft gespeichert, sodass ein
Neustart an der letzten Stelle fortsetzt.
"""
from __future__ import annotations
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Callable, Protocol
from ..config import ProcessingConfig
from ..core.models import AudioFileRecord, FileStatus
from ..database.repository import Repository
from ..utils.logging import get_logger
from .models import TranscriptResult
logger = get_logger("transcription.worker")
class EngineProtocol(Protocol):
"""Minimal benoetigte Engine-Schnittstelle (fuer Tests austauschbar)."""
model_name: str
def transcribe(self, path: str | Path) -> TranscriptResult: ...
@dataclass
class WorkerEvent:
"""Ereignis fuer die Fortschrittsanzeige."""
kind: str # start | done | error | skipped | finished
record: AudioFileRecord | None = None
message: str | None = None
elapsed: float = 0.0
audio_duration: float | None = None
stats: "RunStats | None" = None
@dataclass
class RunStats:
"""Zusammenfassung eines Transkriptionslaufs."""
queued: int = 0
processed: int = 0
failed: int = 0
skipped: int = 0
audio_seconds: float = 0.0
elapsed: float = 0.0
model: str = ""
device: str = ""
compute_type: str = ""
errors: list[tuple[str, str]] = field(default_factory=list)
@property
def rate(self) -> float:
if self.elapsed <= 0:
return 0.0
return self.audio_seconds / self.elapsed
class TranscriptionWorker:
"""Verarbeitet die Transkriptionswarteschlange."""
def __init__(
self,
repository: Repository,
engine: EngineProtocol,
processing: ProcessingConfig | None = None,
*,
device: str = "",
compute_type: str = "",
):
self.repo = repository
self.engine = engine
self.processing = processing or ProcessingConfig()
self.stats = RunStats(
model=getattr(engine, "model_name", ""),
device=device,
compute_type=compute_type,
)
# ------------------------------------------------------------------
def run(
self,
*,
limit: int | None = None,
force: bool = False,
event_callback: Callable[[WorkerEvent], None] | None = None,
) -> RunStats:
"""Arbeitet die Warteschlange ab."""
if self.processing.resume:
reset = self.repo.reset_stale_processing()
if reset:
logger.info("%d unterbrochene Eintraege zurueckgesetzt", reset)
queue = self.repo.transcription_queue(force=force)
if limit is not None:
queue = queue[: max(0, int(limit))]
self.stats.queued = len(queue)
started_at = time.monotonic()
try:
for record in queue:
self._process(record, event_callback)
except KeyboardInterrupt:
logger.warning("Transkription durch Benutzer abgebrochen")
raise
finally:
self.stats.elapsed = time.monotonic() - started_at
if event_callback:
event_callback(WorkerEvent(kind="finished", stats=self.stats))
logger.info(
"Transkription beendet: %d verarbeitet, %d Fehler, %d uebersprungen, "
"%.1f s Audio in %.1f s (%.2fx Echtzeit)",
self.stats.processed,
self.stats.failed,
self.stats.skipped,
self.stats.audio_seconds,
self.stats.elapsed,
self.stats.rate,
)
return self.stats
# ------------------------------------------------------------------
def _process(
self, record: AudioFileRecord, event_callback: Callable[[WorkerEvent], None] | None
) -> None:
path = Path(record.path)
if not path.is_file():
message = f"Datei nicht gefunden: {record.path}"
self.repo.set_status(record.id, FileStatus.ERROR.value, message)
self.stats.failed += 1
self.stats.errors.append((record.path, message))
if event_callback:
event_callback(WorkerEvent(kind="error", record=record, message=message))
return
self.repo.set_status(record.id, FileStatus.PROCESSING.value)
if event_callback:
event_callback(WorkerEvent(kind="start", record=record))
file_started = time.monotonic()
try:
result = self.engine.transcribe(path)
if not result.segments:
raise ValueError("Keine Sprache im Transkript erkannt")
self.repo.save_transcript(
record.id,
language=result.language,
model=result.model or self.engine.model_name,
full_text=result.full_text,
segments=result.segments,
duration=result.duration or record.duration,
)
except KeyboardInterrupt:
self.repo.set_status(record.id, FileStatus.PENDING.value)
raise
except Exception as exc:
elapsed = time.monotonic() - file_started
message = f"{type(exc).__name__}: {exc}"
logger.exception("Transkription fehlgeschlagen fuer %s", record.path)
self.repo.set_status(record.id, FileStatus.ERROR.value, message)
self.stats.failed += 1
self.stats.errors.append((record.path, message))
if event_callback:
event_callback(
WorkerEvent(
kind="error",
record=record,
message=message,
elapsed=elapsed,
audio_duration=record.duration,
)
)
return
elapsed = time.monotonic() - file_started
audio_duration = result.duration or record.duration or 0.0
self.stats.processed += 1
self.stats.audio_seconds += float(audio_duration or 0.0)
logger.info(
"Fertig: %s (%.1f s Audio, %.1f s Laufzeit, %d Segmente)",
record.filename,
audio_duration or 0.0,
elapsed,
len(result.segments),
)
if event_callback:
event_callback(
WorkerEvent(
kind="done",
record=record,
elapsed=elapsed,
audio_duration=audio_duration,
)
)