Initial commit
This commit is contained in:
@@ -0,0 +1,38 @@
|
||||
"""Hash-Helfer fuer die Duplikaterkennung."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
CHUNK_SIZE = 1024 * 1024
|
||||
|
||||
|
||||
def sha256_file(path: str | Path, chunk_size: int = CHUNK_SIZE) -> str:
|
||||
"""Berechnet den SHA256-Hash einer Datei, ohne sie komplett in den RAM zu laden."""
|
||||
digest = hashlib.sha256()
|
||||
with Path(path).open("rb") as handle:
|
||||
while True:
|
||||
chunk = handle.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def quick_signature(path: str | Path, sample_size: int = 64 * 1024) -> str:
|
||||
"""Schnelle Signatur aus Groesse, erster und letzter Datenportion.
|
||||
|
||||
Nur fuer Vorpruefungen gedacht - fuer belastbare Duplikaterkennung muss
|
||||
:func:`sha256_file` verwendet werden.
|
||||
"""
|
||||
path = Path(path)
|
||||
size = path.stat().st_size
|
||||
digest = hashlib.sha256()
|
||||
digest.update(str(size).encode("ascii"))
|
||||
with path.open("rb") as handle:
|
||||
digest.update(handle.read(sample_size))
|
||||
if size > sample_size:
|
||||
handle.seek(max(sample_size, size - sample_size))
|
||||
digest.update(handle.read(sample_size))
|
||||
return digest.hexdigest()
|
||||
Reference in New Issue
Block a user