255 lines
8.1 KiB
Python
255 lines
8.1 KiB
Python
"""Rekursiver Audio-Scanner.
|
|
|
|
Der Scanner nutzt Dateigroesse und Aenderungszeit fuer eine schnelle
|
|
Aenderungspruefung. SHA256-Hashes werden nur berechnet, wenn sie fuer die
|
|
Duplikaterkennung tatsaechlich gebraucht werden (Modus ``auto``).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Callable
|
|
|
|
from ..config import ProcessingConfig
|
|
from ..database.repository import Repository
|
|
from ..utils.hashing import sha256_file
|
|
from ..utils.logging import get_logger
|
|
from .metadata import AudioMetadataError, is_supported_audio, read_metadata
|
|
from .models import FileStatus, ScanCandidate, ScanError, ScanResult
|
|
|
|
logger = get_logger("core.scanner")
|
|
|
|
ScanProgress = Callable[[int, Path], None]
|
|
# Toleranz fuer Dateisystem-Zeitstempel (Sekunden).
|
|
MTIME_TOLERANCE = 1.0
|
|
|
|
|
|
@dataclass
|
|
class _DiscoveredFile:
|
|
path: Path
|
|
size: int
|
|
mtime: float
|
|
duration: float | None = None
|
|
content_hash: str | None = None
|
|
error: str | None = None
|
|
existing_id: int | None = None
|
|
changed: bool = False
|
|
|
|
|
|
@dataclass
|
|
class _ScanState:
|
|
new_files: list[_DiscoveredFile] = field(default_factory=list)
|
|
changed_files: list[_DiscoveredFile] = field(default_factory=list)
|
|
|
|
|
|
def _iter_audio_files(root: Path) -> list[Path]:
|
|
found: list[Path] = []
|
|
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
|
|
dirnames.sort()
|
|
for filename in sorted(filenames):
|
|
if filename.startswith("."):
|
|
continue
|
|
path = Path(dirpath) / filename
|
|
if is_supported_audio(path):
|
|
found.append(path)
|
|
return found
|
|
|
|
|
|
def _should_hash(mode: str) -> bool:
|
|
return mode.strip().lower() == "always"
|
|
|
|
|
|
def scan(
|
|
root: str | Path,
|
|
repository: Repository,
|
|
processing: ProcessingConfig | None = None,
|
|
*,
|
|
progress: ScanProgress | None = None,
|
|
) -> ScanResult:
|
|
"""Scannt ``root`` rekursiv und aktualisiert die Datenbank."""
|
|
processing = processing or ProcessingConfig()
|
|
root_path = Path(root).expanduser().resolve()
|
|
if not root_path.exists():
|
|
raise FileNotFoundError(f"Pfad existiert nicht: {root_path}")
|
|
if not root_path.is_dir():
|
|
raise NotADirectoryError(f"Pfad ist kein Ordner: {root_path}")
|
|
|
|
result = ScanResult(root=str(root_path))
|
|
state = _ScanState()
|
|
|
|
files = _iter_audio_files(root_path)
|
|
result.files_found = len(files)
|
|
|
|
for index, path in enumerate(files, start=1):
|
|
if progress:
|
|
progress(index, path)
|
|
try:
|
|
stat = path.stat()
|
|
except OSError as exc:
|
|
result.errors.append(ScanError(str(path), f"Dateizugriff fehlgeschlagen: {exc}"))
|
|
continue
|
|
|
|
existing = repository.get_audio_by_path(str(path))
|
|
discovered = _DiscoveredFile(
|
|
path=path,
|
|
size=int(stat.st_size),
|
|
mtime=float(stat.st_mtime),
|
|
existing_id=existing.id if existing else None,
|
|
)
|
|
|
|
if existing is not None:
|
|
modified = abs(float(existing.modified_at) - discovered.mtime) > MTIME_TOLERANCE
|
|
if int(existing.file_size) != discovered.size or modified:
|
|
discovered.changed = True
|
|
state.changed_files.append(discovered)
|
|
else:
|
|
result.unchanged += 1
|
|
continue
|
|
else:
|
|
state.new_files.append(discovered)
|
|
|
|
try:
|
|
info = read_metadata(path)
|
|
discovered.duration = info.duration
|
|
except AudioMetadataError as exc:
|
|
discovered.error = str(exc)
|
|
result.errors.append(ScanError(str(path), str(exc)))
|
|
logger.warning("Metadaten konnten nicht gelesen werden: %s (%s)", path, exc)
|
|
|
|
_assign_hashes(state, repository, processing)
|
|
_insert_new_files(state.new_files, repository, processing, result)
|
|
_update_changed_files(state.changed_files, repository, processing, result)
|
|
|
|
logger.info(
|
|
"Scan abgeschlossen: %d gefunden, %d neu, %d geaendert, %d unveraendert, %d Duplikate, %d Fehler",
|
|
result.files_found,
|
|
result.added,
|
|
result.changed,
|
|
result.unchanged,
|
|
result.duplicates,
|
|
len(result.errors),
|
|
)
|
|
return result
|
|
|
|
|
|
def _assign_hashes(
|
|
state: _ScanState, repository: Repository, processing: ProcessingConfig
|
|
) -> None:
|
|
"""Berechnet Hashes nur dort, wo Duplikate moeglich sind."""
|
|
mode = (processing.compute_hash or "auto").strip().lower()
|
|
if mode == "never":
|
|
return
|
|
|
|
candidates = state.new_files + state.changed_files
|
|
if not candidates:
|
|
return
|
|
|
|
limit_bytes = max(0, int(processing.hash_max_size_mb)) * 1024 * 1024
|
|
|
|
if _should_hash(mode):
|
|
targets = [item for item in candidates if item.error is None]
|
|
else:
|
|
size_counts: dict[int, int] = {}
|
|
for item in candidates:
|
|
size_counts[item.size] = size_counts.get(item.size, 0) + 1
|
|
targets = []
|
|
for item in candidates:
|
|
if item.error is not None:
|
|
continue
|
|
needs_check = size_counts.get(item.size, 0) > 1
|
|
if not needs_check:
|
|
needs_check = bool(repository.paths_with_size(item.size))
|
|
if needs_check:
|
|
targets.append(item)
|
|
|
|
for item in targets:
|
|
if limit_bytes and item.size > limit_bytes:
|
|
logger.info("Datei zu gross fuer Hashing, wird uebersprungen: %s", item.path)
|
|
continue
|
|
try:
|
|
item.content_hash = sha256_file(item.path)
|
|
except OSError as exc:
|
|
logger.warning("Hashing fehlgeschlagen fuer %s: %s", item.path, exc)
|
|
|
|
|
|
def _insert_new_files(
|
|
files: list[_DiscoveredFile],
|
|
repository: Repository,
|
|
processing: ProcessingConfig,
|
|
result: ScanResult,
|
|
) -> None:
|
|
seen_hashes: dict[str, int] = {}
|
|
for item in files:
|
|
duplicate_of: int | None = None
|
|
status = FileStatus.PENDING.value
|
|
if item.content_hash:
|
|
existing = repository.find_by_hash(item.content_hash)
|
|
if existing is not None:
|
|
duplicate_of = existing.id
|
|
elif item.content_hash in seen_hashes:
|
|
duplicate_of = seen_hashes[item.content_hash]
|
|
if item.error is not None and duplicate_of is None:
|
|
status = FileStatus.ERROR.value
|
|
|
|
candidate = ScanCandidate(
|
|
path=str(item.path),
|
|
filename=item.path.name,
|
|
file_size=item.size,
|
|
modified_at=item.mtime,
|
|
duration=item.duration,
|
|
content_hash=item.content_hash,
|
|
)
|
|
audio_id = repository.insert_audio(
|
|
candidate,
|
|
status=status,
|
|
duplicate_of=duplicate_of,
|
|
error_message=item.error,
|
|
)
|
|
result.added += 1
|
|
if duplicate_of is not None:
|
|
result.duplicates += 1
|
|
elif item.content_hash:
|
|
seen_hashes[item.content_hash] = audio_id
|
|
|
|
|
|
def _update_changed_files(
|
|
files: list[_DiscoveredFile],
|
|
repository: Repository,
|
|
processing: ProcessingConfig,
|
|
result: ScanResult,
|
|
) -> None:
|
|
for item in files:
|
|
assert item.existing_id is not None
|
|
duplicate_of: int | None = None
|
|
status = FileStatus.PENDING.value
|
|
if item.content_hash:
|
|
existing = repository.find_by_hash(item.content_hash, exclude_id=item.existing_id)
|
|
if existing is not None:
|
|
duplicate_of = existing.id
|
|
if item.error is not None and duplicate_of is None:
|
|
status = FileStatus.ERROR.value
|
|
|
|
candidate = ScanCandidate(
|
|
path=str(item.path),
|
|
filename=item.path.name,
|
|
file_size=item.size,
|
|
modified_at=item.mtime,
|
|
duration=item.duration,
|
|
content_hash=item.content_hash,
|
|
)
|
|
repository.update_scan_data(
|
|
item.existing_id,
|
|
candidate,
|
|
status=status,
|
|
duplicate_of=duplicate_of,
|
|
error_message=item.error,
|
|
)
|
|
result.changed += 1
|
|
if duplicate_of is not None:
|
|
result.duplicates += 1
|
|
|
|
if files:
|
|
logger.info("%d geaenderte Dateien zur erneuten Verarbeitung vorgemerkt", len(files))
|