39 lines
1.2 KiB
Python
39 lines
1.2 KiB
Python
"""Hash-Helfer fuer die Duplikaterkennung."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from pathlib import Path
|
|
|
|
CHUNK_SIZE = 1024 * 1024
|
|
|
|
|
|
def sha256_file(path: str | Path, chunk_size: int = CHUNK_SIZE) -> str:
|
|
"""Berechnet den SHA256-Hash einer Datei, ohne sie komplett in den RAM zu laden."""
|
|
digest = hashlib.sha256()
|
|
with Path(path).open("rb") as handle:
|
|
while True:
|
|
chunk = handle.read(chunk_size)
|
|
if not chunk:
|
|
break
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def quick_signature(path: str | Path, sample_size: int = 64 * 1024) -> str:
|
|
"""Schnelle Signatur aus Groesse, erster und letzter Datenportion.
|
|
|
|
Nur fuer Vorpruefungen gedacht - fuer belastbare Duplikaterkennung muss
|
|
:func:`sha256_file` verwendet werden.
|
|
"""
|
|
path = Path(path)
|
|
size = path.stat().st_size
|
|
digest = hashlib.sha256()
|
|
digest.update(str(size).encode("ascii"))
|
|
with path.open("rb") as handle:
|
|
digest.update(handle.read(sample_size))
|
|
if size > sample_size:
|
|
handle.seek(max(sample_size, size - sample_size))
|
|
digest.update(handle.read(sample_size))
|
|
return digest.hexdigest()
|