Initial commit

This commit is contained in:
2026-11-16 16:29:32 +01:00
commit 97e5c9ef4d
37 changed files with 5932 additions and 0 deletions
+63
View File
@@ -0,0 +1,63 @@
"""Verbindungsaufbau fuer die SQLite-Datenbank."""
from __future__ import annotations
import sqlite3
from pathlib import Path
from ..utils.logging import get_logger
logger = get_logger("database")
def connect(db_path: str | Path, *, create: bool = True) -> sqlite3.Connection:
"""Oeffnet die Datenbank mit sinnvollen PRAGMAs.
``create=False`` verhindert, dass versehentlich eine neue Datenbankdatei
angelegt wird (z. B. bei reiner Diagnose).
"""
path = Path(db_path)
if not create and not path.exists():
raise FileNotFoundError(f"Datenbank nicht gefunden: {path}")
path.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(str(path), timeout=30.0)
conn.row_factory = sqlite3.Row
conn.isolation_level = "" # explizite Transaktionen via "with conn:"
conn.execute("PRAGMA foreign_keys = ON")
conn.execute("PRAGMA journal_mode = WAL")
conn.execute("PRAGMA synchronous = NORMAL")
conn.execute("PRAGMA busy_timeout = 30000")
conn.execute("PRAGMA temp_store = MEMORY")
return conn
def check_fts5(conn: sqlite3.Connection) -> tuple[bool, bool]:
"""Prueft, ob FTS5 und der Trigram-Tokenizer verfuegbar sind."""
fts5 = False
trigram = False
try:
conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS _fts_check USING fts5(x)")
conn.execute("DROP TABLE IF EXISTS _fts_check")
fts5 = True
except sqlite3.OperationalError as exc:
logger.warning("FTS5 nicht verfuegbar: %s", exc)
return False, False
if fts5:
try:
conn.execute(
"CREATE VIRTUAL TABLE IF NOT EXISTS _trigram_check USING fts5(x, tokenize='trigram')"
)
conn.execute("DROP TABLE IF EXISTS _trigram_check")
trigram = True
except sqlite3.OperationalError as exc:
logger.warning("FTS5-Trigram-Tokenizer nicht verfuegbar: %s", exc)
return fts5, trigram
def sqlite_version() -> str:
return sqlite3.sqlite_version
+55
View File
@@ -0,0 +1,55 @@
"""Schema-Migrationen fuer SjAudioSearch.
Die Version wird in ``PRAGMA user_version`` gespeichert. Jede Migration laeuft
in einer eigenen Transaktion, damit ein Abbruch keinen halb migrierten Zustand
hinterlaesst.
"""
from __future__ import annotations
import sqlite3
from ..utils.logging import get_logger
from .schema import SCHEMA_VERSION, TABLES_SQL
logger = get_logger("database.migrations")
def current_version(conn: sqlite3.Connection) -> int:
row = conn.execute("PRAGMA user_version").fetchone()
return int(row[0]) if row else 0
def _migration_1(conn: sqlite3.Connection) -> None:
conn.executescript(TABLES_SQL)
MIGRATIONS: list[tuple[int, object]] = [
(1, _migration_1),
]
def migrate(conn: sqlite3.Connection, target: int = SCHEMA_VERSION) -> int:
"""Fuehrt alle ausstehenden Migrationen aus und liefert die neue Version."""
version = current_version(conn)
if version > target:
raise RuntimeError(
f"Datenbankversion {version} ist neuer als die unterstuetzte Version {target}."
)
if version == target:
return version
for migration_version, migration in MIGRATIONS:
if migration_version <= version:
continue
logger.info("Migration %s wird angewendet", migration_version)
with conn:
migration(conn) # type: ignore[operator]
conn.execute(f"PRAGMA user_version = {int(migration_version)}")
version = migration_version
if version != target:
raise RuntimeError(
f"Migration unvollstaendig: Version {version}, erwartet {target}."
)
return version
+547
View File
@@ -0,0 +1,547 @@
"""Datenbankzugriff fuer SjAudioSearch.
Alle Schreibvorgaenge laufen in expliziten Transaktionen, damit ein Abbruch
keine halben Transkripte hinterlaesst.
"""
from __future__ import annotations
import sqlite3
from collections.abc import Iterator, Sequence
from pathlib import Path
from typing import Any
from ..core.models import AudioFileRecord, FileStatus, RepositoryStats, ScanCandidate
from ..search.fuzzy import normalize_text
from ..utils.formatting import utc_now_iso
from ..utils.logging import get_logger
from .connection import connect
from .migrations import migrate
logger = get_logger("database.repository")
_AUDIO_COLUMNS = """
id, path, filename, duration, file_size, modified_at, content_hash,
status, duplicate_of, error_message, created_at, updated_at
"""
def _row_to_audio(row: sqlite3.Row) -> AudioFileRecord:
return AudioFileRecord(
id=row["id"],
path=row["path"],
filename=row["filename"],
duration=row["duration"],
file_size=row["file_size"],
modified_at=row["modified_at"],
content_hash=row["content_hash"],
status=row["status"],
duplicate_of=row["duplicate_of"],
error_message=row["error_message"],
created_at=row["created_at"],
updated_at=row["updated_at"],
)
class Repository:
"""Kapselt saemtliche SQL-Zugriffe."""
def __init__(self, connection: sqlite3.Connection):
self.conn = connection
@classmethod
def open(cls, db_path: str | Path) -> "Repository":
conn = connect(db_path)
migrate(conn)
return cls(conn)
def close(self) -> None:
self.conn.close()
def __enter__(self) -> "Repository":
return self
def __exit__(self, *exc_info: object) -> None:
self.close()
# ------------------------------------------------------------------
# audio_files
# ------------------------------------------------------------------
def get_audio(self, audio_id: int) -> AudioFileRecord | None:
row = self.conn.execute(
f"SELECT {_AUDIO_COLUMNS} FROM audio_files WHERE id = ?", (audio_id,)
).fetchone()
return _row_to_audio(row) if row else None
def get_audio_by_path(self, path: str) -> AudioFileRecord | None:
row = self.conn.execute(
f"SELECT {_AUDIO_COLUMNS} FROM audio_files WHERE path = ?", (str(path),)
).fetchone()
return _row_to_audio(row) if row else None
def list_audio(
self,
*,
status: str | None = None,
limit: int | None = None,
search: str | None = None,
include_duplicates: bool = True,
) -> list[AudioFileRecord]:
clauses: list[str] = []
params: list[Any] = []
if status:
clauses.append("status = ?")
params.append(status)
if not include_duplicates:
clauses.append("duplicate_of IS NULL")
if search:
clauses.append("(filename LIKE ? OR path LIKE ?)")
like = f"%{search}%"
params.extend([like, like])
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
sql = f"SELECT {_AUDIO_COLUMNS} FROM audio_files {where} ORDER BY filename COLLATE NOCASE, id"
if limit:
sql += " LIMIT ?"
params.append(int(limit))
rows = self.conn.execute(sql, params).fetchall()
return [_row_to_audio(row) for row in rows]
def insert_audio(
self,
candidate: ScanCandidate,
*,
status: str = FileStatus.PENDING.value,
duplicate_of: int | None = None,
error_message: str | None = None,
) -> int:
now = utc_now_iso()
with self.conn:
cursor = self.conn.execute(
"""
INSERT INTO audio_files
(path, filename, duration, file_size, modified_at, content_hash,
status, duplicate_of, error_message, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
candidate.path,
candidate.filename,
candidate.duration,
int(candidate.file_size),
float(candidate.modified_at),
candidate.content_hash,
status,
duplicate_of,
error_message,
now,
now,
),
)
return int(cursor.lastrowid)
def update_scan_data(
self,
audio_id: int,
candidate: ScanCandidate,
*,
status: str = FileStatus.PENDING.value,
duplicate_of: int | None = None,
error_message: str | None = None,
) -> None:
with self.conn:
self.conn.execute(
"""
UPDATE audio_files
SET duration = ?, file_size = ?, modified_at = ?, content_hash = ?,
status = ?, duplicate_of = ?, error_message = ?, updated_at = ?
WHERE id = ?
""",
(
candidate.duration,
int(candidate.file_size),
float(candidate.modified_at),
candidate.content_hash,
status,
duplicate_of,
error_message,
utc_now_iso(),
audio_id,
),
)
def set_duration(self, audio_id: int, duration: float | None) -> None:
if duration is None:
return
with self.conn:
self.conn.execute(
"UPDATE audio_files SET duration = ?, updated_at = ? WHERE id = ? AND duration IS NULL",
(float(duration), utc_now_iso(), audio_id),
)
def set_status(self, audio_id: int, status: str, error_message: str | None = None) -> None:
with self.conn:
self.conn.execute(
"UPDATE audio_files SET status = ?, error_message = ?, updated_at = ? WHERE id = ?",
(status, error_message, utc_now_iso(), audio_id),
)
def reset_stale_processing(self) -> int:
"""Setzt abgebrochene 'processing'-Eintraege nach einem Absturz zurueck."""
with self.conn:
cursor = self.conn.execute(
"UPDATE audio_files SET status = 'pending', updated_at = ? WHERE status = 'processing'",
(utc_now_iso(),),
)
return cursor.rowcount or 0
def reset_errors(self) -> int:
"""Setzt fehlgeschlagene Transkriptionen zurueck auf 'pending'."""
with self.conn:
cursor = self.conn.execute(
"""
UPDATE audio_files
SET status = 'pending', error_message = NULL, updated_at = ?
WHERE status = 'error'
""",
(utc_now_iso(),),
)
return cursor.rowcount or 0
def transcription_queue(self, *, force: bool = False) -> list[AudioFileRecord]:
if force:
sql = f"""
SELECT {_AUDIO_COLUMNS} FROM audio_files
WHERE duplicate_of IS NULL
ORDER BY id
"""
rows = self.conn.execute(sql).fetchall()
else:
sql = f"""
SELECT {_AUDIO_COLUMNS} FROM audio_files
WHERE duplicate_of IS NULL AND status IN ('pending', 'processing')
ORDER BY id
"""
rows = self.conn.execute(sql).fetchall()
return [_row_to_audio(row) for row in rows]
def find_by_hash(self, content_hash: str, *, exclude_id: int | None = None) -> AudioFileRecord | None:
sql = f"SELECT {_AUDIO_COLUMNS} FROM audio_files WHERE content_hash = ? AND duplicate_of IS NULL"
params: list[Any] = [content_hash]
if exclude_id is not None:
sql += " AND id != ?"
params.append(exclude_id)
row = self.conn.execute(sql + " ORDER BY id LIMIT 1", params).fetchone()
return _row_to_audio(row) if row else None
def paths_with_size(self, size: int) -> list[AudioFileRecord]:
rows = self.conn.execute(
f"SELECT {_AUDIO_COLUMNS} FROM audio_files WHERE file_size = ? AND content_hash IS NOT NULL",
(int(size),),
).fetchall()
return [_row_to_audio(row) for row in rows]
def count_by_status(self) -> dict[str, int]:
rows = self.conn.execute(
"SELECT status, COUNT(*) AS amount FROM audio_files GROUP BY status"
).fetchall()
return {row["status"]: int(row["amount"]) for row in rows}
def stats(self, db_size_bytes: int = 0) -> RepositoryStats:
counts = self.count_by_status()
total_row = self.conn.execute(
"SELECT COUNT(*) AS total, COALESCE(SUM(duration), 0) AS duration FROM audio_files"
).fetchone()
dup_row = self.conn.execute(
"SELECT COUNT(*) AS amount FROM audio_files WHERE duplicate_of IS NOT NULL"
).fetchone()
segment_row = self.conn.execute("SELECT COUNT(*) AS amount FROM segments").fetchone()
word_row = self.conn.execute("SELECT COUNT(*) AS amount FROM words").fetchone()
transcript_row = self.conn.execute("SELECT COUNT(*) AS amount FROM transcripts").fetchone()
model_rows = self.conn.execute("SELECT DISTINCT model FROM transcripts").fetchall()
return RepositoryStats(
total_files=int(total_row["total"]),
done=counts.get("done", 0),
pending=counts.get("pending", 0),
processing=counts.get("processing", 0),
error=counts.get("error", 0),
duplicates=int(dup_row["amount"]),
total_duration=float(total_row["duration"] or 0.0),
database_size=int(db_size_bytes),
segment_count=int(segment_row["amount"]),
word_count=int(word_row["amount"]),
transcripts=int(transcript_row["amount"]),
models=[row["model"] for row in model_rows],
)
def databases_size(self) -> int:
row = self.conn.execute("PRAGMA database_list").fetchone()
if not row:
return 0
path = Path(row["file"])
size = path.stat().st_size if path.exists() else 0
wal = path.with_name(path.name + "-wal")
if wal.exists():
size += wal.stat().st_size
return size
# ------------------------------------------------------------------
# Transkripte
# ------------------------------------------------------------------
def save_transcript(
self,
audio_id: int,
*,
language: str | None,
model: str,
full_text: str,
segments: Sequence[Any],
duration: float | None = None,
) -> int:
"""Speichert ein Transkript samt Segmenten, Woertern und Suchindex.
Vorhandene Daten der Datei werden ersetzt. Alles geschieht in einer
Transaktion, damit ein Abbruch keinen Teilzustand hinterlaesst.
"""
now = utc_now_iso()
with self.conn:
self.conn.execute("DELETE FROM transcripts WHERE audio_id = ?", (audio_id,))
self.conn.execute("DELETE FROM search_index WHERE audio_id = ?", (audio_id,))
self.conn.execute("DELETE FROM search_trigram WHERE audio_id = ?", (audio_id,))
cursor = self.conn.execute(
"""
INSERT INTO transcripts (audio_id, language, model, full_text, created_at)
VALUES (?, ?, ?, ?, ?)
""",
(audio_id, language, model, full_text, now),
)
transcript_id = int(cursor.lastrowid)
for segment in segments:
cursor = self.conn.execute(
"""
INSERT INTO segments (transcript_id, start_time, end_time, text, confidence)
VALUES (?, ?, ?, ?, ?)
""",
(
transcript_id,
float(segment.start),
float(segment.end),
segment.text,
getattr(segment, "confidence", None),
),
)
segment_id = int(cursor.lastrowid)
words = getattr(segment, "words", None) or []
if words:
self.conn.executemany(
"INSERT INTO words (segment_id, word, start_time, end_time) VALUES (?, ?, ?, ?)",
[
(
segment_id,
word.word,
getattr(word, "start", None),
getattr(word, "end", None),
)
for word in words
],
)
normalized = normalize_text(segment.text)
if normalized:
self.conn.execute(
"""
INSERT INTO search_index (text, audio_id, transcript_id, segment_id)
VALUES (?, ?, ?, ?)
""",
(normalized, audio_id, transcript_id, segment_id),
)
self.conn.execute(
"INSERT INTO search_trigram (text, audio_id, segment_id) VALUES (?, ?, ?)",
(normalized, audio_id, segment_id),
)
if duration is not None:
self.conn.execute(
"""
UPDATE audio_files
SET duration = COALESCE(duration, ?), status = 'done',
error_message = NULL, updated_at = ?
WHERE id = ?
""",
(float(duration), now, audio_id),
)
else:
self.conn.execute(
"""
UPDATE audio_files
SET status = 'done', error_message = NULL, updated_at = ?
WHERE id = ?
""",
(now, audio_id),
)
return transcript_id
def get_transcript(self, audio_id: int) -> sqlite3.Row | None:
return self.conn.execute(
"SELECT * FROM transcripts WHERE audio_id = ?", (audio_id,)
).fetchone()
def get_segments(self, audio_id: int) -> list[sqlite3.Row]:
return self.conn.execute(
"""
SELECT s.id, s.start_time, s.end_time, s.text, s.confidence
FROM segments s
JOIN transcripts t ON t.id = s.transcript_id
WHERE t.audio_id = ?
ORDER BY s.start_time, s.id
""",
(audio_id,),
).fetchall()
def get_words(self, segment_ids: Sequence[int]) -> dict[int, list[sqlite3.Row]]:
result: dict[int, list[sqlite3.Row]] = {}
ids = list(segment_ids)
for start in range(0, len(ids), 500):
chunk = ids[start : start + 500]
placeholders = ",".join("?" for _ in chunk)
rows = self.conn.execute(
f"""
SELECT id, segment_id, word, start_time, end_time
FROM words WHERE segment_id IN ({placeholders})
ORDER BY segment_id, start_time, id
""",
chunk,
).fetchall()
for row in rows:
result.setdefault(int(row["segment_id"]), []).append(row)
return result
def segment_texts(self, audio_id: int) -> list[str]:
return [row["text"] for row in self.get_segments(audio_id)]
def iter_audio_ids_with_transcripts(self, language: str | None = None) -> Iterator[int]:
sql = "SELECT DISTINCT audio_id FROM transcripts"
params: list[Any] = []
if language:
sql += " WHERE language = ?"
params.append(language)
sql += " ORDER BY audio_id"
cursor = self.conn.execute(sql, params)
for row in cursor:
yield int(row["audio_id"])
def transcript_languages(self) -> list[str]:
rows = self.conn.execute(
"SELECT DISTINCT language FROM transcripts WHERE language IS NOT NULL ORDER BY language"
).fetchall()
return [row["language"] for row in rows]
# ------------------------------------------------------------------
# Suche
# ------------------------------------------------------------------
def fts_candidates(self, match_query: str, limit: int) -> list[tuple[int, int]]:
"""Liefert (audio_id, segment_id) fuer einen FTS5-Ausdruck."""
if not match_query:
return []
try:
rows = self.conn.execute(
"""
SELECT audio_id, segment_id, bm25(search_index) AS rank
FROM search_index
WHERE search_index MATCH ?
ORDER BY rank
LIMIT ?
""",
(match_query, int(limit)),
).fetchall()
except sqlite3.OperationalError as exc:
logger.warning("FTS-Anfrage fehlgeschlagen (%s): %s", match_query, exc)
return []
return [(int(row["audio_id"]), int(row["segment_id"])) for row in rows]
def trigram_candidates(self, terms: Sequence[str], limit: int) -> list[tuple[int, int]]:
"""Teilstring-Kandidaten ueber den FTS5-Trigram-Index."""
results: list[tuple[int, int]] = []
seen: set[tuple[int, int]] = set()
for term in terms:
escaped = '"' + term.replace('"', '""') + '"'
try:
rows = self.conn.execute(
"""
SELECT audio_id, segment_id FROM search_trigram
WHERE search_trigram MATCH ?
LIMIT ?
""",
(escaped, int(limit)),
).fetchall()
except sqlite3.OperationalError as exc:
logger.warning("Trigramm-Anfrage fehlgeschlagen (%s): %s", term, exc)
continue
for row in rows:
key = (int(row["audio_id"]), int(row["segment_id"]))
if key not in seen:
seen.add(key)
results.append(key)
if len(results) >= limit:
return results
return results
def timeline(self, audio_id: int) -> tuple[list[sqlite3.Row], dict[int, list[sqlite3.Row]]]:
segments = self.get_segments(audio_id)
words = self.get_words([int(segment["id"]) for segment in segments])
return segments, words
def rebuild_search_index(self) -> int:
"""Baut den FTS-Index aus den gespeicherten Segmenten neu auf."""
with self.conn:
self.conn.execute("DELETE FROM search_index")
self.conn.execute("DELETE FROM search_trigram")
rows = self.conn.execute(
"""
SELECT s.id AS segment_id, t.audio_id AS audio_id, t.id AS transcript_id, s.text
FROM segments s
JOIN transcripts t ON t.id = s.transcript_id
ORDER BY s.id
"""
).fetchall()
index_rows = []
trigram_rows = []
for row in rows:
normalized = normalize_text(row["text"])
if not normalized:
continue
index_rows.append(
(normalized, int(row["audio_id"]), int(row["transcript_id"]), int(row["segment_id"]))
)
trigram_rows.append((normalized, int(row["audio_id"]), int(row["segment_id"])))
self.conn.executemany(
"INSERT INTO search_index (text, audio_id, transcript_id, segment_id) VALUES (?, ?, ?, ?)",
index_rows,
)
self.conn.executemany(
"INSERT INTO search_trigram (text, audio_id, segment_id) VALUES (?, ?, ?)",
trigram_rows,
)
return len(index_rows)
def delete_audio(self, audio_id: int) -> bool:
"""Entfernt eine Datei aus dem Index (die Audiodatei bleibt unberuehrt)."""
with self.conn:
self.conn.execute("DELETE FROM search_index WHERE audio_id = ?", (audio_id,))
self.conn.execute("DELETE FROM search_trigram WHERE audio_id = ?", (audio_id,))
cursor = self.conn.execute("DELETE FROM audio_files WHERE id = ?", (audio_id,))
return (cursor.rowcount or 0) > 0
def upsert_meta(self, key: str, value: str) -> None:
with self.conn:
self.conn.execute(
"INSERT INTO meta (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
(key, value),
)
def get_meta(self, key: str) -> str | None:
row = self.conn.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone()
return row["value"] if row else None
+80
View File
@@ -0,0 +1,80 @@
"""SQL-Schema fuer SjAudioSearch (SQLite + FTS5)."""
from __future__ import annotations
SCHEMA_VERSION = 1
TABLES_SQL = """
CREATE TABLE IF NOT EXISTS audio_files (
id INTEGER PRIMARY KEY AUTOINCREMENT,
path TEXT NOT NULL UNIQUE,
filename TEXT NOT NULL,
duration REAL,
file_size INTEGER NOT NULL,
modified_at REAL NOT NULL,
content_hash TEXT,
status TEXT NOT NULL DEFAULT 'pending'
CHECK (status IN ('pending', 'processing', 'done', 'error')),
duplicate_of INTEGER REFERENCES audio_files(id) ON DELETE SET NULL,
error_message TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_audio_status ON audio_files(status);
CREATE INDEX IF NOT EXISTS idx_audio_hash ON audio_files(content_hash);
CREATE INDEX IF NOT EXISTS idx_audio_size ON audio_files(file_size);
CREATE INDEX IF NOT EXISTS idx_audio_duplicate ON audio_files(duplicate_of);
CREATE TABLE IF NOT EXISTS transcripts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
audio_id INTEGER NOT NULL UNIQUE REFERENCES audio_files(id) ON DELETE CASCADE,
language TEXT,
model TEXT NOT NULL,
full_text TEXT NOT NULL,
created_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_transcripts_language ON transcripts(language);
CREATE TABLE IF NOT EXISTS segments (
id INTEGER PRIMARY KEY AUTOINCREMENT,
transcript_id INTEGER NOT NULL REFERENCES transcripts(id) ON DELETE CASCADE,
start_time REAL NOT NULL,
end_time REAL NOT NULL,
text TEXT NOT NULL,
confidence REAL
);
CREATE INDEX IF NOT EXISTS idx_segments_transcript ON segments(transcript_id, start_time);
CREATE TABLE IF NOT EXISTS words (
id INTEGER PRIMARY KEY AUTOINCREMENT,
segment_id INTEGER NOT NULL REFERENCES segments(id) ON DELETE CASCADE,
word TEXT NOT NULL,
start_time REAL,
end_time REAL
);
CREATE INDEX IF NOT EXISTS idx_words_segment ON words(segment_id);
CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
text,
audio_id UNINDEXED,
transcript_id UNINDEXED,
segment_id UNINDEXED,
tokenize = "unicode61 remove_diacritics 2"
);
CREATE VIRTUAL TABLE IF NOT EXISTS search_trigram USING fts5(
text,
audio_id UNINDEXED,
segment_id UNINDEXED,
tokenize = "trigram"
);
CREATE TABLE IF NOT EXISTS meta (
key TEXT PRIMARY KEY,
value TEXT
);
"""