Initial commit
This commit is contained in:
@@ -0,0 +1,186 @@
|
||||
"""Kontextaufbau fuer Suchtreffer.
|
||||
|
||||
Aus den gespeicherten Segmenten wird eine durchgehende Wort-Timeline gebaut.
|
||||
Dadurch gehen an Segmentgrenzen keine Informationen verloren, und der Kontext
|
||||
kann mitten in einem Segment beginnen und enden.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Sequence
|
||||
|
||||
from .fuzzy import normalize_text, tokenize_original
|
||||
|
||||
|
||||
@dataclass
|
||||
class TokenRef:
|
||||
"""Ein Wort im Transkript mit Zeitbezug und Herkunft."""
|
||||
|
||||
text: str
|
||||
norm: str
|
||||
start: float
|
||||
end: float
|
||||
segment_id: int
|
||||
precision: str # "word" oder "segment"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Timeline:
|
||||
"""Durchgehende Wortfolge einer Audiodatei."""
|
||||
|
||||
audio_id: int
|
||||
tokens: list[TokenRef] = field(default_factory=list)
|
||||
language: str | None = None
|
||||
model: str | None = None
|
||||
|
||||
@property
|
||||
def normalized_tokens(self) -> list[str]:
|
||||
return [token.norm for token in self.tokens]
|
||||
|
||||
@property
|
||||
def normalized_text(self) -> str:
|
||||
return " ".join(token.norm for token in self.tokens)
|
||||
|
||||
@property
|
||||
def original_text(self) -> str:
|
||||
return " ".join(token.text for token in self.tokens)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.tokens)
|
||||
|
||||
|
||||
def _distribute_times(
|
||||
segment_start: float, segment_end: float, token_count: int
|
||||
) -> list[tuple[float, float]]:
|
||||
"""Verteilt die Segmentzeit gleichmaessig auf die Woerter (Schaetzung)."""
|
||||
if token_count <= 0:
|
||||
return []
|
||||
span = max(0.0, float(segment_end) - float(segment_start))
|
||||
step = span / token_count if span else 0.0
|
||||
times: list[tuple[float, float]] = []
|
||||
for index in range(token_count):
|
||||
start = float(segment_start) + step * index
|
||||
end = float(segment_start) + step * (index + 1)
|
||||
times.append((round(start, 3), round(end, 3)))
|
||||
return times
|
||||
|
||||
|
||||
def build_timeline(
|
||||
audio_id: int,
|
||||
segments: Sequence[Any],
|
||||
words_by_segment: dict[int, Sequence[Any]] | None = None,
|
||||
*,
|
||||
language: str | None = None,
|
||||
model: str | None = None,
|
||||
) -> Timeline:
|
||||
"""Baut die Timeline aus Segmenten (sqlite3.Row oder Objekte mit Attributen)."""
|
||||
timeline = Timeline(audio_id=audio_id, language=language, model=model)
|
||||
words_by_segment = words_by_segment or {}
|
||||
|
||||
for segment in segments:
|
||||
if isinstance(segment, dict):
|
||||
segment_id = int(segment["id"])
|
||||
start = float(segment["start_time"])
|
||||
end = float(segment["end_time"])
|
||||
text = segment["text"]
|
||||
else:
|
||||
segment_id = int(segment["id"])
|
||||
start = float(segment["start_time"])
|
||||
end = float(segment["end_time"])
|
||||
text = segment["text"]
|
||||
|
||||
original_tokens = tokenize_original(text)
|
||||
if not original_tokens:
|
||||
continue
|
||||
|
||||
words = list(words_by_segment.get(segment_id, []))
|
||||
use_words = len(words) == len(original_tokens)
|
||||
|
||||
if use_words:
|
||||
for token, word in zip(original_tokens, words):
|
||||
word_start = word["start_time"] if not isinstance(word, dict) else word.get("start_time")
|
||||
word_end = word["end_time"] if not isinstance(word, dict) else word.get("end_time")
|
||||
if word_start is None or word_end is None:
|
||||
use_words = False
|
||||
break
|
||||
|
||||
if use_words:
|
||||
for token, word in zip(original_tokens, words):
|
||||
word_start = word["start_time"] if not isinstance(word, dict) else word["start_time"]
|
||||
word_end = word["end_time"] if not isinstance(word, dict) else word["end_time"]
|
||||
timeline.tokens.append(
|
||||
TokenRef(
|
||||
text=token,
|
||||
norm=normalize_text(token),
|
||||
start=float(word_start),
|
||||
end=float(word_end),
|
||||
segment_id=segment_id,
|
||||
precision="word",
|
||||
)
|
||||
)
|
||||
else:
|
||||
times = _distribute_times(start, end, len(original_tokens))
|
||||
for token, (token_start, token_end) in zip(original_tokens, times):
|
||||
timeline.tokens.append(
|
||||
TokenRef(
|
||||
text=token,
|
||||
norm=normalize_text(token),
|
||||
start=token_start,
|
||||
end=token_end,
|
||||
segment_id=segment_id,
|
||||
precision="segment",
|
||||
)
|
||||
)
|
||||
|
||||
return timeline
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchContext:
|
||||
"""Die drei Kontextabschnitte eines Treffers."""
|
||||
|
||||
before: str
|
||||
match: str
|
||||
after: str
|
||||
start_time: float
|
||||
end_time: float
|
||||
precision: str
|
||||
|
||||
def as_block(self, quote: str = '"') -> str:
|
||||
"""Formatiert den Kontext als zusammenhaengenden Textblock."""
|
||||
before = f"\u2026{self.before} " if self.before else ""
|
||||
after = f" {self.after}\u2026" if self.after else ""
|
||||
return f"{quote}{before}{self.match}{after}{quote}"
|
||||
|
||||
|
||||
def build_context(
|
||||
timeline: Timeline, start_index: int, end_index: int, context_words: int = 50
|
||||
) -> SearchContext:
|
||||
"""Schneidet Kontext um den Treffer ``[start_index, end_index)`` aus."""
|
||||
tokens = timeline.tokens
|
||||
if not tokens:
|
||||
return SearchContext("", "", "", 0.0, 0.0, "segment")
|
||||
|
||||
start_index = max(0, min(start_index, len(tokens)))
|
||||
end_index = max(start_index + 1, min(end_index, len(tokens)))
|
||||
|
||||
before_start = max(0, start_index - max(0, context_words))
|
||||
after_end = min(len(tokens), end_index + max(0, context_words))
|
||||
|
||||
before_tokens = tokens[before_start:start_index]
|
||||
match_tokens = tokens[start_index:end_index]
|
||||
after_tokens = tokens[end_index:after_end]
|
||||
|
||||
precision = "word" if all(token.precision == "word" for token in match_tokens) else "segment"
|
||||
start_time = match_tokens[0].start
|
||||
end_time = match_tokens[-1].end
|
||||
|
||||
return SearchContext(
|
||||
before=" ".join(token.text for token in before_tokens),
|
||||
match=" ".join(token.text for token in match_tokens),
|
||||
after=" ".join(token.text for token in after_tokens),
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
precision=precision,
|
||||
)
|
||||
Reference in New Issue
Block a user