Initial commit
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
"""Audio-Wiedergabe und Dekodierpruefungen."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from .metadata import AudioMetadataError, read_metadata
|
||||
|
||||
|
||||
class PlaybackError(RuntimeError):
|
||||
"""Wiedergabe nicht moeglich."""
|
||||
|
||||
|
||||
def find_player() -> str | None:
|
||||
"""Sucht einen installierten Player (bevorzugt ffplay)."""
|
||||
for candidate in ("ffplay", "mpv", "vlc"):
|
||||
executable = shutil.which(candidate)
|
||||
if executable:
|
||||
return executable
|
||||
return None
|
||||
|
||||
|
||||
def build_play_command(path: str | Path, start_time: float = 0.0) -> list[str]:
|
||||
"""Baut den Kommandozeilenaufruf zum Abspielen ab einer Zeitposition."""
|
||||
player = find_player()
|
||||
if player is None:
|
||||
raise PlaybackError(
|
||||
"Kein Player gefunden. Installiere ffmpeg (ffplay), mpv oder vlc."
|
||||
)
|
||||
name = Path(player).name
|
||||
start = max(0.0, float(start_time))
|
||||
|
||||
if name.startswith("ffplay"):
|
||||
# -nodisp: reine Audiowiedergabe ohne zusaetzliches Fenster.
|
||||
command = [player, "-nodisp", "-autoexit", "-loglevel", "error"]
|
||||
if start > 0:
|
||||
command += ["-ss", f"{start:.3f}"]
|
||||
command.append(str(path))
|
||||
return command
|
||||
if name.startswith("mpv"):
|
||||
command = [player]
|
||||
if start > 0:
|
||||
command += [f"--start={start:.3f}"]
|
||||
command.append(str(path))
|
||||
return command
|
||||
# vlc
|
||||
command = [player]
|
||||
if start > 0:
|
||||
command += [f"--start-time={start:.3f}"]
|
||||
command.append(str(path))
|
||||
return command
|
||||
|
||||
|
||||
def play(path: str | Path, start_time: float = 0.0) -> None:
|
||||
"""Startet die Wiedergabe und wartet, bis der Player beendet wird."""
|
||||
file_path = Path(path)
|
||||
if not file_path.is_file():
|
||||
raise PlaybackError(f"Audiodatei nicht gefunden: {file_path}")
|
||||
command = build_play_command(file_path, start_time)
|
||||
try:
|
||||
subprocess.run(command, check=False)
|
||||
except OSError as exc:
|
||||
raise PlaybackError(f"Player konnte nicht gestartet werden: {exc}") from exc
|
||||
|
||||
|
||||
def can_decode(path: str | Path) -> bool:
|
||||
"""Prueft grob, ob die Datei als Audio dekodierbar ist."""
|
||||
try:
|
||||
read_metadata(path)
|
||||
return True
|
||||
except AudioMetadataError:
|
||||
return False
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Audio-Metadaten (Dauer, Tags, Format).
|
||||
|
||||
Primaer wird Mutagen verwendet; wenn Mutagen eine Datei nicht lesen kann, wird
|
||||
PyAV als Fallback genutzt. Beide Wege lesen nur Header bzw. Container, nicht
|
||||
die komplette Audiodatei.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..utils.logging import get_logger
|
||||
|
||||
logger = get_logger("core.metadata")
|
||||
|
||||
SUPPORTED_EXTENSIONS = frozenset(
|
||||
{".mp3", ".wav", ".flac", ".m4a", ".ogg", ".opus", ".wma"}
|
||||
)
|
||||
# WMA ist nur zuverlaessig dekodierbar, wenn ffmpeg/PyAV den Codec kennt.
|
||||
CONDITIONAL_EXTENSIONS = frozenset({".wma"})
|
||||
|
||||
|
||||
class AudioMetadataError(RuntimeError):
|
||||
"""Die Datei konnte nicht als Audio gelesen werden."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class AudioInfo:
|
||||
"""Ergebnis der Metadatenanalyse."""
|
||||
|
||||
path: str
|
||||
duration: float | None = None
|
||||
codec: str | None = None
|
||||
sample_rate: int | None = None
|
||||
channels: int | None = None
|
||||
bitrate: int | None = None
|
||||
tags: dict[str, str] = field(default_factory=dict)
|
||||
source: str = "mutagen"
|
||||
|
||||
|
||||
def is_supported_audio(path: str | Path) -> bool:
|
||||
return Path(path).suffix.lower() in SUPPORTED_EXTENSIONS
|
||||
|
||||
|
||||
def _clean_tags(raw: dict[str, Any]) -> dict[str, str]:
|
||||
tags: dict[str, str] = {}
|
||||
for key, value in raw.items():
|
||||
try:
|
||||
if isinstance(value, (list, tuple)):
|
||||
text = ", ".join(str(item) for item in value if item is not None)
|
||||
else:
|
||||
text = str(value)
|
||||
except Exception: # defekte Tag-Objekte duerfen den Scan nicht stoppen
|
||||
continue
|
||||
if text:
|
||||
tags[str(key).lower()] = text
|
||||
return tags
|
||||
|
||||
|
||||
def _read_with_mutagen(path: Path) -> AudioInfo:
|
||||
import mutagen
|
||||
|
||||
audio = mutagen.File(str(path))
|
||||
if audio is None:
|
||||
raise AudioMetadataError("Format wurde von Mutagen nicht erkannt")
|
||||
|
||||
info = getattr(audio, "info", None)
|
||||
duration = getattr(info, "length", None)
|
||||
if duration is None:
|
||||
raise AudioMetadataError("Mutagen konnte keine Dauer ermitteln")
|
||||
|
||||
return AudioInfo(
|
||||
path=str(path),
|
||||
duration=float(duration),
|
||||
codec=type(audio).__name__,
|
||||
sample_rate=getattr(info, "sample_rate", None),
|
||||
channels=getattr(info, "channels", None),
|
||||
bitrate=getattr(info, "bitrate", None),
|
||||
tags=_clean_tags(dict(audio.tags or {})),
|
||||
source="mutagen",
|
||||
)
|
||||
|
||||
|
||||
def _read_with_pyav(path: Path) -> AudioInfo:
|
||||
import av
|
||||
|
||||
try:
|
||||
with av.open(str(path)) as container:
|
||||
if not container.streams.audio:
|
||||
raise AudioMetadataError("Keine Audiospur gefunden")
|
||||
stream = container.streams.audio[0]
|
||||
duration = None
|
||||
if stream.duration is not None and stream.time_base is not None:
|
||||
duration = float(stream.duration * stream.time_base)
|
||||
elif container.duration is not None:
|
||||
duration = float(container.duration / 1_000_000)
|
||||
if duration is None or duration <= 0:
|
||||
raise AudioMetadataError("Keine gueltige Dauer ermittelbar")
|
||||
return AudioInfo(
|
||||
path=str(path),
|
||||
duration=duration,
|
||||
codec=stream.codec_context.name,
|
||||
sample_rate=stream.codec_context.sample_rate,
|
||||
channels=stream.codec_context.channels,
|
||||
bitrate=stream.bit_rate,
|
||||
source="pyav",
|
||||
)
|
||||
except AudioMetadataError:
|
||||
raise
|
||||
except Exception as exc: # PyAV wirft verschiedene Fehlertypen
|
||||
raise AudioMetadataError(f"PyAV konnte die Datei nicht lesen: {exc}") from exc
|
||||
|
||||
|
||||
def read_metadata(path: str | Path) -> AudioInfo:
|
||||
"""Liest Metadaten ueber Mutagen, bei Bedarf ueber PyAV."""
|
||||
file_path = Path(path)
|
||||
if not file_path.is_file():
|
||||
raise AudioMetadataError(f"Datei nicht gefunden: {file_path}")
|
||||
|
||||
errors: list[str] = []
|
||||
try:
|
||||
return _read_with_mutagen(file_path)
|
||||
except Exception as exc:
|
||||
errors.append(f"mutagen: {exc}")
|
||||
logger.debug("Mutagen fehlgeschlagen fuer %s: %s", file_path, exc)
|
||||
|
||||
try:
|
||||
return _read_with_pyav(file_path)
|
||||
except Exception as exc:
|
||||
errors.append(f"pyav: {exc}")
|
||||
logger.debug("PyAV fehlgeschlagen fuer %s: %s", file_path, exc)
|
||||
|
||||
raise AudioMetadataError("; ".join(errors))
|
||||
|
||||
|
||||
def read_duration(path: str | Path) -> float | None:
|
||||
"""Dauer einer Datei oder ``None``, wenn sie nicht ermittelbar ist."""
|
||||
try:
|
||||
return read_metadata(path).duration
|
||||
except AudioMetadataError:
|
||||
return None
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Datenmodelle fuer Audiodateien und Scan-Ergebnisse."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class FileStatus(str, Enum):
|
||||
"""Transkriptionsstatus einer Audiodatei."""
|
||||
|
||||
PENDING = "pending"
|
||||
PROCESSING = "processing"
|
||||
DONE = "done"
|
||||
ERROR = "error"
|
||||
|
||||
@classmethod
|
||||
def values(cls) -> tuple[str, ...]:
|
||||
return tuple(item.value for item in cls)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AudioFileRecord:
|
||||
"""Repraesentiert eine Zeile aus ``audio_files``."""
|
||||
|
||||
id: int
|
||||
path: str
|
||||
filename: str
|
||||
file_size: int
|
||||
modified_at: float
|
||||
status: str
|
||||
duration: float | None = None
|
||||
content_hash: str | None = None
|
||||
duplicate_of: int | None = None
|
||||
error_message: str | None = None
|
||||
created_at: str | None = None
|
||||
updated_at: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanCandidate:
|
||||
"""Beim Scan gefundene Datei (noch nicht in der Datenbank)."""
|
||||
|
||||
path: str
|
||||
filename: str
|
||||
file_size: int
|
||||
modified_at: float
|
||||
duration: float | None = None
|
||||
content_hash: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanError:
|
||||
"""Fehler, der beim Einlesen einer Datei aufgetreten ist."""
|
||||
|
||||
path: str
|
||||
message: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanResult:
|
||||
"""Zusammenfassung eines Scan-Laufs."""
|
||||
|
||||
root: str
|
||||
files_found: int = 0
|
||||
added: int = 0
|
||||
changed: int = 0
|
||||
unchanged: int = 0
|
||||
duplicates: int = 0
|
||||
errors: list[ScanError] = field(default_factory=list)
|
||||
skipped_extensions: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class RepositoryStats:
|
||||
"""Kennzahlen fuer ``sjaudiosearch status``."""
|
||||
|
||||
total_files: int = 0
|
||||
done: int = 0
|
||||
pending: int = 0
|
||||
processing: int = 0
|
||||
error: int = 0
|
||||
duplicates: int = 0
|
||||
total_duration: float = 0.0
|
||||
database_size: int = 0
|
||||
segment_count: int = 0
|
||||
word_count: int = 0
|
||||
transcripts: int = 0
|
||||
models: list[str] = field(default_factory=list)
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Rekursiver Audio-Scanner.
|
||||
|
||||
Der Scanner nutzt Dateigroesse und Aenderungszeit fuer eine schnelle
|
||||
Aenderungspruefung. SHA256-Hashes werden nur berechnet, wenn sie fuer die
|
||||
Duplikaterkennung tatsaechlich gebraucht werden (Modus ``auto``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from ..config import ProcessingConfig
|
||||
from ..database.repository import Repository
|
||||
from ..utils.hashing import sha256_file
|
||||
from ..utils.logging import get_logger
|
||||
from .metadata import AudioMetadataError, is_supported_audio, read_metadata
|
||||
from .models import FileStatus, ScanCandidate, ScanError, ScanResult
|
||||
|
||||
logger = get_logger("core.scanner")
|
||||
|
||||
ScanProgress = Callable[[int, Path], None]
|
||||
# Toleranz fuer Dateisystem-Zeitstempel (Sekunden).
|
||||
MTIME_TOLERANCE = 1.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class _DiscoveredFile:
|
||||
path: Path
|
||||
size: int
|
||||
mtime: float
|
||||
duration: float | None = None
|
||||
content_hash: str | None = None
|
||||
error: str | None = None
|
||||
existing_id: int | None = None
|
||||
changed: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ScanState:
|
||||
new_files: list[_DiscoveredFile] = field(default_factory=list)
|
||||
changed_files: list[_DiscoveredFile] = field(default_factory=list)
|
||||
|
||||
|
||||
def _iter_audio_files(root: Path) -> list[Path]:
|
||||
found: list[Path] = []
|
||||
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
|
||||
dirnames.sort()
|
||||
for filename in sorted(filenames):
|
||||
if filename.startswith("."):
|
||||
continue
|
||||
path = Path(dirpath) / filename
|
||||
if is_supported_audio(path):
|
||||
found.append(path)
|
||||
return found
|
||||
|
||||
|
||||
def _should_hash(mode: str) -> bool:
|
||||
return mode.strip().lower() == "always"
|
||||
|
||||
|
||||
def scan(
|
||||
root: str | Path,
|
||||
repository: Repository,
|
||||
processing: ProcessingConfig | None = None,
|
||||
*,
|
||||
progress: ScanProgress | None = None,
|
||||
) -> ScanResult:
|
||||
"""Scannt ``root`` rekursiv und aktualisiert die Datenbank."""
|
||||
processing = processing or ProcessingConfig()
|
||||
root_path = Path(root).expanduser().resolve()
|
||||
if not root_path.exists():
|
||||
raise FileNotFoundError(f"Pfad existiert nicht: {root_path}")
|
||||
if not root_path.is_dir():
|
||||
raise NotADirectoryError(f"Pfad ist kein Ordner: {root_path}")
|
||||
|
||||
result = ScanResult(root=str(root_path))
|
||||
state = _ScanState()
|
||||
|
||||
files = _iter_audio_files(root_path)
|
||||
result.files_found = len(files)
|
||||
|
||||
for index, path in enumerate(files, start=1):
|
||||
if progress:
|
||||
progress(index, path)
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError as exc:
|
||||
result.errors.append(ScanError(str(path), f"Dateizugriff fehlgeschlagen: {exc}"))
|
||||
continue
|
||||
|
||||
existing = repository.get_audio_by_path(str(path))
|
||||
discovered = _DiscoveredFile(
|
||||
path=path,
|
||||
size=int(stat.st_size),
|
||||
mtime=float(stat.st_mtime),
|
||||
existing_id=existing.id if existing else None,
|
||||
)
|
||||
|
||||
if existing is not None:
|
||||
modified = abs(float(existing.modified_at) - discovered.mtime) > MTIME_TOLERANCE
|
||||
if int(existing.file_size) != discovered.size or modified:
|
||||
discovered.changed = True
|
||||
state.changed_files.append(discovered)
|
||||
else:
|
||||
result.unchanged += 1
|
||||
continue
|
||||
else:
|
||||
state.new_files.append(discovered)
|
||||
|
||||
try:
|
||||
info = read_metadata(path)
|
||||
discovered.duration = info.duration
|
||||
except AudioMetadataError as exc:
|
||||
discovered.error = str(exc)
|
||||
result.errors.append(ScanError(str(path), str(exc)))
|
||||
logger.warning("Metadaten konnten nicht gelesen werden: %s (%s)", path, exc)
|
||||
|
||||
_assign_hashes(state, repository, processing)
|
||||
_insert_new_files(state.new_files, repository, processing, result)
|
||||
_update_changed_files(state.changed_files, repository, processing, result)
|
||||
|
||||
logger.info(
|
||||
"Scan abgeschlossen: %d gefunden, %d neu, %d geaendert, %d unveraendert, %d Duplikate, %d Fehler",
|
||||
result.files_found,
|
||||
result.added,
|
||||
result.changed,
|
||||
result.unchanged,
|
||||
result.duplicates,
|
||||
len(result.errors),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _assign_hashes(
|
||||
state: _ScanState, repository: Repository, processing: ProcessingConfig
|
||||
) -> None:
|
||||
"""Berechnet Hashes nur dort, wo Duplikate moeglich sind."""
|
||||
mode = (processing.compute_hash or "auto").strip().lower()
|
||||
if mode == "never":
|
||||
return
|
||||
|
||||
candidates = state.new_files + state.changed_files
|
||||
if not candidates:
|
||||
return
|
||||
|
||||
limit_bytes = max(0, int(processing.hash_max_size_mb)) * 1024 * 1024
|
||||
|
||||
if _should_hash(mode):
|
||||
targets = [item for item in candidates if item.error is None]
|
||||
else:
|
||||
size_counts: dict[int, int] = {}
|
||||
for item in candidates:
|
||||
size_counts[item.size] = size_counts.get(item.size, 0) + 1
|
||||
targets = []
|
||||
for item in candidates:
|
||||
if item.error is not None:
|
||||
continue
|
||||
needs_check = size_counts.get(item.size, 0) > 1
|
||||
if not needs_check:
|
||||
needs_check = bool(repository.paths_with_size(item.size))
|
||||
if needs_check:
|
||||
targets.append(item)
|
||||
|
||||
for item in targets:
|
||||
if limit_bytes and item.size > limit_bytes:
|
||||
logger.info("Datei zu gross fuer Hashing, wird uebersprungen: %s", item.path)
|
||||
continue
|
||||
try:
|
||||
item.content_hash = sha256_file(item.path)
|
||||
except OSError as exc:
|
||||
logger.warning("Hashing fehlgeschlagen fuer %s: %s", item.path, exc)
|
||||
|
||||
|
||||
def _insert_new_files(
|
||||
files: list[_DiscoveredFile],
|
||||
repository: Repository,
|
||||
processing: ProcessingConfig,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
seen_hashes: dict[str, int] = {}
|
||||
for item in files:
|
||||
duplicate_of: int | None = None
|
||||
status = FileStatus.PENDING.value
|
||||
if item.content_hash:
|
||||
existing = repository.find_by_hash(item.content_hash)
|
||||
if existing is not None:
|
||||
duplicate_of = existing.id
|
||||
elif item.content_hash in seen_hashes:
|
||||
duplicate_of = seen_hashes[item.content_hash]
|
||||
if item.error is not None and duplicate_of is None:
|
||||
status = FileStatus.ERROR.value
|
||||
|
||||
candidate = ScanCandidate(
|
||||
path=str(item.path),
|
||||
filename=item.path.name,
|
||||
file_size=item.size,
|
||||
modified_at=item.mtime,
|
||||
duration=item.duration,
|
||||
content_hash=item.content_hash,
|
||||
)
|
||||
audio_id = repository.insert_audio(
|
||||
candidate,
|
||||
status=status,
|
||||
duplicate_of=duplicate_of,
|
||||
error_message=item.error,
|
||||
)
|
||||
result.added += 1
|
||||
if duplicate_of is not None:
|
||||
result.duplicates += 1
|
||||
elif item.content_hash:
|
||||
seen_hashes[item.content_hash] = audio_id
|
||||
|
||||
|
||||
def _update_changed_files(
|
||||
files: list[_DiscoveredFile],
|
||||
repository: Repository,
|
||||
processing: ProcessingConfig,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
for item in files:
|
||||
assert item.existing_id is not None
|
||||
duplicate_of: int | None = None
|
||||
status = FileStatus.PENDING.value
|
||||
if item.content_hash:
|
||||
existing = repository.find_by_hash(item.content_hash, exclude_id=item.existing_id)
|
||||
if existing is not None:
|
||||
duplicate_of = existing.id
|
||||
if item.error is not None and duplicate_of is None:
|
||||
status = FileStatus.ERROR.value
|
||||
|
||||
candidate = ScanCandidate(
|
||||
path=str(item.path),
|
||||
filename=item.path.name,
|
||||
file_size=item.size,
|
||||
modified_at=item.mtime,
|
||||
duration=item.duration,
|
||||
content_hash=item.content_hash,
|
||||
)
|
||||
repository.update_scan_data(
|
||||
item.existing_id,
|
||||
candidate,
|
||||
status=status,
|
||||
duplicate_of=duplicate_of,
|
||||
error_message=item.error,
|
||||
)
|
||||
result.changed += 1
|
||||
if duplicate_of is not None:
|
||||
result.duplicates += 1
|
||||
|
||||
if files:
|
||||
logger.info("%d geaenderte Dateien zur erneuten Verarbeitung vorgemerkt", len(files))
|
||||
Reference in New Issue
Block a user