Initial commit

This commit is contained in:
2026-11-16 16:29:32 +01:00
commit 97e5c9ef4d
37 changed files with 5932 additions and 0 deletions
+74
View File
@@ -0,0 +1,74 @@
"""Audio-Wiedergabe und Dekodierpruefungen."""
from __future__ import annotations
import shutil
import subprocess
from pathlib import Path
from .metadata import AudioMetadataError, read_metadata
class PlaybackError(RuntimeError):
"""Wiedergabe nicht moeglich."""
def find_player() -> str | None:
"""Sucht einen installierten Player (bevorzugt ffplay)."""
for candidate in ("ffplay", "mpv", "vlc"):
executable = shutil.which(candidate)
if executable:
return executable
return None
def build_play_command(path: str | Path, start_time: float = 0.0) -> list[str]:
"""Baut den Kommandozeilenaufruf zum Abspielen ab einer Zeitposition."""
player = find_player()
if player is None:
raise PlaybackError(
"Kein Player gefunden. Installiere ffmpeg (ffplay), mpv oder vlc."
)
name = Path(player).name
start = max(0.0, float(start_time))
if name.startswith("ffplay"):
# -nodisp: reine Audiowiedergabe ohne zusaetzliches Fenster.
command = [player, "-nodisp", "-autoexit", "-loglevel", "error"]
if start > 0:
command += ["-ss", f"{start:.3f}"]
command.append(str(path))
return command
if name.startswith("mpv"):
command = [player]
if start > 0:
command += [f"--start={start:.3f}"]
command.append(str(path))
return command
# vlc
command = [player]
if start > 0:
command += [f"--start-time={start:.3f}"]
command.append(str(path))
return command
def play(path: str | Path, start_time: float = 0.0) -> None:
"""Startet die Wiedergabe und wartet, bis der Player beendet wird."""
file_path = Path(path)
if not file_path.is_file():
raise PlaybackError(f"Audiodatei nicht gefunden: {file_path}")
command = build_play_command(file_path, start_time)
try:
subprocess.run(command, check=False)
except OSError as exc:
raise PlaybackError(f"Player konnte nicht gestartet werden: {exc}") from exc
def can_decode(path: str | Path) -> bool:
"""Prueft grob, ob die Datei als Audio dekodierbar ist."""
try:
read_metadata(path)
return True
except AudioMetadataError:
return False
+143
View File
@@ -0,0 +1,143 @@
"""Audio-Metadaten (Dauer, Tags, Format).
Primaer wird Mutagen verwendet; wenn Mutagen eine Datei nicht lesen kann, wird
PyAV als Fallback genutzt. Beide Wege lesen nur Header bzw. Container, nicht
die komplette Audiodatei.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from ..utils.logging import get_logger
logger = get_logger("core.metadata")
SUPPORTED_EXTENSIONS = frozenset(
{".mp3", ".wav", ".flac", ".m4a", ".ogg", ".opus", ".wma"}
)
# WMA ist nur zuverlaessig dekodierbar, wenn ffmpeg/PyAV den Codec kennt.
CONDITIONAL_EXTENSIONS = frozenset({".wma"})
class AudioMetadataError(RuntimeError):
"""Die Datei konnte nicht als Audio gelesen werden."""
@dataclass
class AudioInfo:
"""Ergebnis der Metadatenanalyse."""
path: str
duration: float | None = None
codec: str | None = None
sample_rate: int | None = None
channels: int | None = None
bitrate: int | None = None
tags: dict[str, str] = field(default_factory=dict)
source: str = "mutagen"
def is_supported_audio(path: str | Path) -> bool:
return Path(path).suffix.lower() in SUPPORTED_EXTENSIONS
def _clean_tags(raw: dict[str, Any]) -> dict[str, str]:
tags: dict[str, str] = {}
for key, value in raw.items():
try:
if isinstance(value, (list, tuple)):
text = ", ".join(str(item) for item in value if item is not None)
else:
text = str(value)
except Exception: # defekte Tag-Objekte duerfen den Scan nicht stoppen
continue
if text:
tags[str(key).lower()] = text
return tags
def _read_with_mutagen(path: Path) -> AudioInfo:
import mutagen
audio = mutagen.File(str(path))
if audio is None:
raise AudioMetadataError("Format wurde von Mutagen nicht erkannt")
info = getattr(audio, "info", None)
duration = getattr(info, "length", None)
if duration is None:
raise AudioMetadataError("Mutagen konnte keine Dauer ermitteln")
return AudioInfo(
path=str(path),
duration=float(duration),
codec=type(audio).__name__,
sample_rate=getattr(info, "sample_rate", None),
channels=getattr(info, "channels", None),
bitrate=getattr(info, "bitrate", None),
tags=_clean_tags(dict(audio.tags or {})),
source="mutagen",
)
def _read_with_pyav(path: Path) -> AudioInfo:
import av
try:
with av.open(str(path)) as container:
if not container.streams.audio:
raise AudioMetadataError("Keine Audiospur gefunden")
stream = container.streams.audio[0]
duration = None
if stream.duration is not None and stream.time_base is not None:
duration = float(stream.duration * stream.time_base)
elif container.duration is not None:
duration = float(container.duration / 1_000_000)
if duration is None or duration <= 0:
raise AudioMetadataError("Keine gueltige Dauer ermittelbar")
return AudioInfo(
path=str(path),
duration=duration,
codec=stream.codec_context.name,
sample_rate=stream.codec_context.sample_rate,
channels=stream.codec_context.channels,
bitrate=stream.bit_rate,
source="pyav",
)
except AudioMetadataError:
raise
except Exception as exc: # PyAV wirft verschiedene Fehlertypen
raise AudioMetadataError(f"PyAV konnte die Datei nicht lesen: {exc}") from exc
def read_metadata(path: str | Path) -> AudioInfo:
"""Liest Metadaten ueber Mutagen, bei Bedarf ueber PyAV."""
file_path = Path(path)
if not file_path.is_file():
raise AudioMetadataError(f"Datei nicht gefunden: {file_path}")
errors: list[str] = []
try:
return _read_with_mutagen(file_path)
except Exception as exc:
errors.append(f"mutagen: {exc}")
logger.debug("Mutagen fehlgeschlagen fuer %s: %s", file_path, exc)
try:
return _read_with_pyav(file_path)
except Exception as exc:
errors.append(f"pyav: {exc}")
logger.debug("PyAV fehlgeschlagen fuer %s: %s", file_path, exc)
raise AudioMetadataError("; ".join(errors))
def read_duration(path: str | Path) -> float | None:
"""Dauer einer Datei oder ``None``, wenn sie nicht ermittelbar ist."""
try:
return read_metadata(path).duration
except AudioMetadataError:
return None
+89
View File
@@ -0,0 +1,89 @@
"""Datenmodelle fuer Audiodateien und Scan-Ergebnisse."""
from __future__ import annotations
from dataclasses import dataclass, field
from enum import Enum
class FileStatus(str, Enum):
"""Transkriptionsstatus einer Audiodatei."""
PENDING = "pending"
PROCESSING = "processing"
DONE = "done"
ERROR = "error"
@classmethod
def values(cls) -> tuple[str, ...]:
return tuple(item.value for item in cls)
@dataclass
class AudioFileRecord:
"""Repraesentiert eine Zeile aus ``audio_files``."""
id: int
path: str
filename: str
file_size: int
modified_at: float
status: str
duration: float | None = None
content_hash: str | None = None
duplicate_of: int | None = None
error_message: str | None = None
created_at: str | None = None
updated_at: str | None = None
@dataclass
class ScanCandidate:
"""Beim Scan gefundene Datei (noch nicht in der Datenbank)."""
path: str
filename: str
file_size: int
modified_at: float
duration: float | None = None
content_hash: str | None = None
@dataclass
class ScanError:
"""Fehler, der beim Einlesen einer Datei aufgetreten ist."""
path: str
message: str
@dataclass
class ScanResult:
"""Zusammenfassung eines Scan-Laufs."""
root: str
files_found: int = 0
added: int = 0
changed: int = 0
unchanged: int = 0
duplicates: int = 0
errors: list[ScanError] = field(default_factory=list)
skipped_extensions: int = 0
@dataclass
class RepositoryStats:
"""Kennzahlen fuer ``sjaudiosearch status``."""
total_files: int = 0
done: int = 0
pending: int = 0
processing: int = 0
error: int = 0
duplicates: int = 0
total_duration: float = 0.0
database_size: int = 0
segment_count: int = 0
word_count: int = 0
transcripts: int = 0
models: list[str] = field(default_factory=list)
+254
View File
@@ -0,0 +1,254 @@
"""Rekursiver Audio-Scanner.
Der Scanner nutzt Dateigroesse und Aenderungszeit fuer eine schnelle
Aenderungspruefung. SHA256-Hashes werden nur berechnet, wenn sie fuer die
Duplikaterkennung tatsaechlich gebraucht werden (Modus ``auto``).
"""
from __future__ import annotations
import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import Callable
from ..config import ProcessingConfig
from ..database.repository import Repository
from ..utils.hashing import sha256_file
from ..utils.logging import get_logger
from .metadata import AudioMetadataError, is_supported_audio, read_metadata
from .models import FileStatus, ScanCandidate, ScanError, ScanResult
logger = get_logger("core.scanner")
ScanProgress = Callable[[int, Path], None]
# Toleranz fuer Dateisystem-Zeitstempel (Sekunden).
MTIME_TOLERANCE = 1.0
@dataclass
class _DiscoveredFile:
path: Path
size: int
mtime: float
duration: float | None = None
content_hash: str | None = None
error: str | None = None
existing_id: int | None = None
changed: bool = False
@dataclass
class _ScanState:
new_files: list[_DiscoveredFile] = field(default_factory=list)
changed_files: list[_DiscoveredFile] = field(default_factory=list)
def _iter_audio_files(root: Path) -> list[Path]:
found: list[Path] = []
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
dirnames.sort()
for filename in sorted(filenames):
if filename.startswith("."):
continue
path = Path(dirpath) / filename
if is_supported_audio(path):
found.append(path)
return found
def _should_hash(mode: str) -> bool:
return mode.strip().lower() == "always"
def scan(
root: str | Path,
repository: Repository,
processing: ProcessingConfig | None = None,
*,
progress: ScanProgress | None = None,
) -> ScanResult:
"""Scannt ``root`` rekursiv und aktualisiert die Datenbank."""
processing = processing or ProcessingConfig()
root_path = Path(root).expanduser().resolve()
if not root_path.exists():
raise FileNotFoundError(f"Pfad existiert nicht: {root_path}")
if not root_path.is_dir():
raise NotADirectoryError(f"Pfad ist kein Ordner: {root_path}")
result = ScanResult(root=str(root_path))
state = _ScanState()
files = _iter_audio_files(root_path)
result.files_found = len(files)
for index, path in enumerate(files, start=1):
if progress:
progress(index, path)
try:
stat = path.stat()
except OSError as exc:
result.errors.append(ScanError(str(path), f"Dateizugriff fehlgeschlagen: {exc}"))
continue
existing = repository.get_audio_by_path(str(path))
discovered = _DiscoveredFile(
path=path,
size=int(stat.st_size),
mtime=float(stat.st_mtime),
existing_id=existing.id if existing else None,
)
if existing is not None:
modified = abs(float(existing.modified_at) - discovered.mtime) > MTIME_TOLERANCE
if int(existing.file_size) != discovered.size or modified:
discovered.changed = True
state.changed_files.append(discovered)
else:
result.unchanged += 1
continue
else:
state.new_files.append(discovered)
try:
info = read_metadata(path)
discovered.duration = info.duration
except AudioMetadataError as exc:
discovered.error = str(exc)
result.errors.append(ScanError(str(path), str(exc)))
logger.warning("Metadaten konnten nicht gelesen werden: %s (%s)", path, exc)
_assign_hashes(state, repository, processing)
_insert_new_files(state.new_files, repository, processing, result)
_update_changed_files(state.changed_files, repository, processing, result)
logger.info(
"Scan abgeschlossen: %d gefunden, %d neu, %d geaendert, %d unveraendert, %d Duplikate, %d Fehler",
result.files_found,
result.added,
result.changed,
result.unchanged,
result.duplicates,
len(result.errors),
)
return result
def _assign_hashes(
state: _ScanState, repository: Repository, processing: ProcessingConfig
) -> None:
"""Berechnet Hashes nur dort, wo Duplikate moeglich sind."""
mode = (processing.compute_hash or "auto").strip().lower()
if mode == "never":
return
candidates = state.new_files + state.changed_files
if not candidates:
return
limit_bytes = max(0, int(processing.hash_max_size_mb)) * 1024 * 1024
if _should_hash(mode):
targets = [item for item in candidates if item.error is None]
else:
size_counts: dict[int, int] = {}
for item in candidates:
size_counts[item.size] = size_counts.get(item.size, 0) + 1
targets = []
for item in candidates:
if item.error is not None:
continue
needs_check = size_counts.get(item.size, 0) > 1
if not needs_check:
needs_check = bool(repository.paths_with_size(item.size))
if needs_check:
targets.append(item)
for item in targets:
if limit_bytes and item.size > limit_bytes:
logger.info("Datei zu gross fuer Hashing, wird uebersprungen: %s", item.path)
continue
try:
item.content_hash = sha256_file(item.path)
except OSError as exc:
logger.warning("Hashing fehlgeschlagen fuer %s: %s", item.path, exc)
def _insert_new_files(
files: list[_DiscoveredFile],
repository: Repository,
processing: ProcessingConfig,
result: ScanResult,
) -> None:
seen_hashes: dict[str, int] = {}
for item in files:
duplicate_of: int | None = None
status = FileStatus.PENDING.value
if item.content_hash:
existing = repository.find_by_hash(item.content_hash)
if existing is not None:
duplicate_of = existing.id
elif item.content_hash in seen_hashes:
duplicate_of = seen_hashes[item.content_hash]
if item.error is not None and duplicate_of is None:
status = FileStatus.ERROR.value
candidate = ScanCandidate(
path=str(item.path),
filename=item.path.name,
file_size=item.size,
modified_at=item.mtime,
duration=item.duration,
content_hash=item.content_hash,
)
audio_id = repository.insert_audio(
candidate,
status=status,
duplicate_of=duplicate_of,
error_message=item.error,
)
result.added += 1
if duplicate_of is not None:
result.duplicates += 1
elif item.content_hash:
seen_hashes[item.content_hash] = audio_id
def _update_changed_files(
files: list[_DiscoveredFile],
repository: Repository,
processing: ProcessingConfig,
result: ScanResult,
) -> None:
for item in files:
assert item.existing_id is not None
duplicate_of: int | None = None
status = FileStatus.PENDING.value
if item.content_hash:
existing = repository.find_by_hash(item.content_hash, exclude_id=item.existing_id)
if existing is not None:
duplicate_of = existing.id
if item.error is not None and duplicate_of is None:
status = FileStatus.ERROR.value
candidate = ScanCandidate(
path=str(item.path),
filename=item.path.name,
file_size=item.size,
modified_at=item.mtime,
duration=item.duration,
content_hash=item.content_hash,
)
repository.update_scan_data(
item.existing_id,
candidate,
status=status,
duplicate_of=duplicate_of,
error_message=item.error,
)
result.changed += 1
if duplicate_of is not None:
result.duplicates += 1
if files:
logger.info("%d geaenderte Dateien zur erneuten Verarbeitung vorgemerkt", len(files))