Initial commit
This commit is contained in:
@@ -0,0 +1,254 @@
|
||||
"""Rekursiver Audio-Scanner.
|
||||
|
||||
Der Scanner nutzt Dateigroesse und Aenderungszeit fuer eine schnelle
|
||||
Aenderungspruefung. SHA256-Hashes werden nur berechnet, wenn sie fuer die
|
||||
Duplikaterkennung tatsaechlich gebraucht werden (Modus ``auto``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from ..config import ProcessingConfig
|
||||
from ..database.repository import Repository
|
||||
from ..utils.hashing import sha256_file
|
||||
from ..utils.logging import get_logger
|
||||
from .metadata import AudioMetadataError, is_supported_audio, read_metadata
|
||||
from .models import FileStatus, ScanCandidate, ScanError, ScanResult
|
||||
|
||||
logger = get_logger("core.scanner")
|
||||
|
||||
ScanProgress = Callable[[int, Path], None]
|
||||
# Toleranz fuer Dateisystem-Zeitstempel (Sekunden).
|
||||
MTIME_TOLERANCE = 1.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class _DiscoveredFile:
|
||||
path: Path
|
||||
size: int
|
||||
mtime: float
|
||||
duration: float | None = None
|
||||
content_hash: str | None = None
|
||||
error: str | None = None
|
||||
existing_id: int | None = None
|
||||
changed: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ScanState:
|
||||
new_files: list[_DiscoveredFile] = field(default_factory=list)
|
||||
changed_files: list[_DiscoveredFile] = field(default_factory=list)
|
||||
|
||||
|
||||
def _iter_audio_files(root: Path) -> list[Path]:
|
||||
found: list[Path] = []
|
||||
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
|
||||
dirnames.sort()
|
||||
for filename in sorted(filenames):
|
||||
if filename.startswith("."):
|
||||
continue
|
||||
path = Path(dirpath) / filename
|
||||
if is_supported_audio(path):
|
||||
found.append(path)
|
||||
return found
|
||||
|
||||
|
||||
def _should_hash(mode: str) -> bool:
|
||||
return mode.strip().lower() == "always"
|
||||
|
||||
|
||||
def scan(
|
||||
root: str | Path,
|
||||
repository: Repository,
|
||||
processing: ProcessingConfig | None = None,
|
||||
*,
|
||||
progress: ScanProgress | None = None,
|
||||
) -> ScanResult:
|
||||
"""Scannt ``root`` rekursiv und aktualisiert die Datenbank."""
|
||||
processing = processing or ProcessingConfig()
|
||||
root_path = Path(root).expanduser().resolve()
|
||||
if not root_path.exists():
|
||||
raise FileNotFoundError(f"Pfad existiert nicht: {root_path}")
|
||||
if not root_path.is_dir():
|
||||
raise NotADirectoryError(f"Pfad ist kein Ordner: {root_path}")
|
||||
|
||||
result = ScanResult(root=str(root_path))
|
||||
state = _ScanState()
|
||||
|
||||
files = _iter_audio_files(root_path)
|
||||
result.files_found = len(files)
|
||||
|
||||
for index, path in enumerate(files, start=1):
|
||||
if progress:
|
||||
progress(index, path)
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError as exc:
|
||||
result.errors.append(ScanError(str(path), f"Dateizugriff fehlgeschlagen: {exc}"))
|
||||
continue
|
||||
|
||||
existing = repository.get_audio_by_path(str(path))
|
||||
discovered = _DiscoveredFile(
|
||||
path=path,
|
||||
size=int(stat.st_size),
|
||||
mtime=float(stat.st_mtime),
|
||||
existing_id=existing.id if existing else None,
|
||||
)
|
||||
|
||||
if existing is not None:
|
||||
modified = abs(float(existing.modified_at) - discovered.mtime) > MTIME_TOLERANCE
|
||||
if int(existing.file_size) != discovered.size or modified:
|
||||
discovered.changed = True
|
||||
state.changed_files.append(discovered)
|
||||
else:
|
||||
result.unchanged += 1
|
||||
continue
|
||||
else:
|
||||
state.new_files.append(discovered)
|
||||
|
||||
try:
|
||||
info = read_metadata(path)
|
||||
discovered.duration = info.duration
|
||||
except AudioMetadataError as exc:
|
||||
discovered.error = str(exc)
|
||||
result.errors.append(ScanError(str(path), str(exc)))
|
||||
logger.warning("Metadaten konnten nicht gelesen werden: %s (%s)", path, exc)
|
||||
|
||||
_assign_hashes(state, repository, processing)
|
||||
_insert_new_files(state.new_files, repository, processing, result)
|
||||
_update_changed_files(state.changed_files, repository, processing, result)
|
||||
|
||||
logger.info(
|
||||
"Scan abgeschlossen: %d gefunden, %d neu, %d geaendert, %d unveraendert, %d Duplikate, %d Fehler",
|
||||
result.files_found,
|
||||
result.added,
|
||||
result.changed,
|
||||
result.unchanged,
|
||||
result.duplicates,
|
||||
len(result.errors),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _assign_hashes(
|
||||
state: _ScanState, repository: Repository, processing: ProcessingConfig
|
||||
) -> None:
|
||||
"""Berechnet Hashes nur dort, wo Duplikate moeglich sind."""
|
||||
mode = (processing.compute_hash or "auto").strip().lower()
|
||||
if mode == "never":
|
||||
return
|
||||
|
||||
candidates = state.new_files + state.changed_files
|
||||
if not candidates:
|
||||
return
|
||||
|
||||
limit_bytes = max(0, int(processing.hash_max_size_mb)) * 1024 * 1024
|
||||
|
||||
if _should_hash(mode):
|
||||
targets = [item for item in candidates if item.error is None]
|
||||
else:
|
||||
size_counts: dict[int, int] = {}
|
||||
for item in candidates:
|
||||
size_counts[item.size] = size_counts.get(item.size, 0) + 1
|
||||
targets = []
|
||||
for item in candidates:
|
||||
if item.error is not None:
|
||||
continue
|
||||
needs_check = size_counts.get(item.size, 0) > 1
|
||||
if not needs_check:
|
||||
needs_check = bool(repository.paths_with_size(item.size))
|
||||
if needs_check:
|
||||
targets.append(item)
|
||||
|
||||
for item in targets:
|
||||
if limit_bytes and item.size > limit_bytes:
|
||||
logger.info("Datei zu gross fuer Hashing, wird uebersprungen: %s", item.path)
|
||||
continue
|
||||
try:
|
||||
item.content_hash = sha256_file(item.path)
|
||||
except OSError as exc:
|
||||
logger.warning("Hashing fehlgeschlagen fuer %s: %s", item.path, exc)
|
||||
|
||||
|
||||
def _insert_new_files(
|
||||
files: list[_DiscoveredFile],
|
||||
repository: Repository,
|
||||
processing: ProcessingConfig,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
seen_hashes: dict[str, int] = {}
|
||||
for item in files:
|
||||
duplicate_of: int | None = None
|
||||
status = FileStatus.PENDING.value
|
||||
if item.content_hash:
|
||||
existing = repository.find_by_hash(item.content_hash)
|
||||
if existing is not None:
|
||||
duplicate_of = existing.id
|
||||
elif item.content_hash in seen_hashes:
|
||||
duplicate_of = seen_hashes[item.content_hash]
|
||||
if item.error is not None and duplicate_of is None:
|
||||
status = FileStatus.ERROR.value
|
||||
|
||||
candidate = ScanCandidate(
|
||||
path=str(item.path),
|
||||
filename=item.path.name,
|
||||
file_size=item.size,
|
||||
modified_at=item.mtime,
|
||||
duration=item.duration,
|
||||
content_hash=item.content_hash,
|
||||
)
|
||||
audio_id = repository.insert_audio(
|
||||
candidate,
|
||||
status=status,
|
||||
duplicate_of=duplicate_of,
|
||||
error_message=item.error,
|
||||
)
|
||||
result.added += 1
|
||||
if duplicate_of is not None:
|
||||
result.duplicates += 1
|
||||
elif item.content_hash:
|
||||
seen_hashes[item.content_hash] = audio_id
|
||||
|
||||
|
||||
def _update_changed_files(
|
||||
files: list[_DiscoveredFile],
|
||||
repository: Repository,
|
||||
processing: ProcessingConfig,
|
||||
result: ScanResult,
|
||||
) -> None:
|
||||
for item in files:
|
||||
assert item.existing_id is not None
|
||||
duplicate_of: int | None = None
|
||||
status = FileStatus.PENDING.value
|
||||
if item.content_hash:
|
||||
existing = repository.find_by_hash(item.content_hash, exclude_id=item.existing_id)
|
||||
if existing is not None:
|
||||
duplicate_of = existing.id
|
||||
if item.error is not None and duplicate_of is None:
|
||||
status = FileStatus.ERROR.value
|
||||
|
||||
candidate = ScanCandidate(
|
||||
path=str(item.path),
|
||||
filename=item.path.name,
|
||||
file_size=item.size,
|
||||
modified_at=item.mtime,
|
||||
duration=item.duration,
|
||||
content_hash=item.content_hash,
|
||||
)
|
||||
repository.update_scan_data(
|
||||
item.existing_id,
|
||||
candidate,
|
||||
status=status,
|
||||
duplicate_of=duplicate_of,
|
||||
error_message=item.error,
|
||||
)
|
||||
result.changed += 1
|
||||
if duplicate_of is not None:
|
||||
result.duplicates += 1
|
||||
|
||||
if files:
|
||||
logger.info("%d geaenderte Dateien zur erneuten Verarbeitung vorgemerkt", len(files))
|
||||
Reference in New Issue
Block a user