eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Implemented deterministic transcript normalization before the LLM stages

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-22 07:22:40 -05:00
parent 445329de81
commit c1672af9b4
12 changed files with 719 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/test_cli.py
View File

@@ -21,4 +21,8 @@ def test_process_help_includes_glossary_pass_flag(capsys):
assert "--grammar-max-llm-passes" in output
assert "--glossary-confidence-threshold" in output
assert "--grammar-confidence-threshold" in output
assert "--normalize-max-segment-gap" in output
assert "--normalize-ellipsis-gap" in output
assert "--normalize-max-segment-duration" in output
assert "--normalize-max-segment-tokens" in output
assert "--confidence-threshold" not in output

49
tests/test_config.py
View File

@@ -10,6 +10,10 @@ from audita.config import (
DEFAULT_GRAMMAR_MAX_LLM_PASSES,
DEFAULT_MAX_RETRIES,
DEFAULT_MAX_SECTION_TOKENS,
DEFAULT_NORMALIZE_ELLIPSIS_GAP,
DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION,
DEFAULT_NORMALIZE_MAX_SEGMENT_GAP,
DEFAULT_NORMALIZE_MAX_SEGMENT_TOKENS,
DEFAULT_WORK_DIR,
)
from audita.errors import AuditaConfigError
@@ -26,6 +30,14 @@ def test_config_uses_defaults_with_api_key():
assert config.max_retries == DEFAULT_MAX_RETRIES
assert config.glossary_max_llm_passes == DEFAULT_GLOSSARY_MAX_LLM_PASSES
assert config.grammar_max_llm_passes == DEFAULT_GRAMMAR_MAX_LLM_PASSES
assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
assert config.normalize_max_segment_gap == 5.0
assert config.normalize_ellipsis_gap == DEFAULT_NORMALIZE_ELLIPSIS_GAP
assert config.normalize_ellipsis_gap == 2.0
assert config.normalize_max_segment_duration == DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION
assert config.normalize_max_segment_duration == 60.0
assert config.normalize_max_segment_tokens == DEFAULT_NORMALIZE_MAX_SEGMENT_TOKENS
assert config.normalize_max_segment_tokens == 2048
assert config.work_dir == Path(DEFAULT_WORK_DIR)
@@ -39,6 +51,10 @@ def test_config_env_overrides_defaults():
"AUDITA_MAX_RETRIES": "5",
"AUDITA_GLOSSARY_MAX_LLM_PASSES": "7",
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "4",
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "4.5",
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "1.5",
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0",
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "512",
"AUDITA_WORK_DIR": "/tmp/custom-audita",
}
)
@@ -49,6 +65,10 @@ def test_config_env_overrides_defaults():
assert config.max_retries == 5
assert config.glossary_max_llm_passes == 7
assert config.grammar_max_llm_passes == 4
assert config.normalize_max_segment_gap == 4.5
assert config.normalize_ellipsis_gap == 1.5
assert config.normalize_max_segment_duration == 45.0
assert config.normalize_max_segment_tokens == 512
assert config.work_dir == Path("/tmp/custom-audita")
@@ -60,6 +80,10 @@ def test_config_cli_overrides_env():
"AUDITA_MAX_RETRIES": "5",
"AUDITA_GLOSSARY_MAX_LLM_PASSES": "7",
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "6",
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "4.5",
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "1.5",
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0",
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "512",
"AUDITA_WORK_DIR": "/tmp/env-audita",
},
overrides=ConfigOverrides(
@@ -69,6 +93,10 @@ def test_config_cli_overrides_env():
max_retries=3,
glossary_max_llm_passes=2,
grammar_max_llm_passes=3,
normalize_max_segment_gap=3.0,
normalize_ellipsis_gap=1.0,
normalize_max_segment_duration=30.0,
normalize_max_segment_tokens=256,
work_dir=Path("/tmp/cli-audita"),
),
)
@@ -79,6 +107,10 @@ def test_config_cli_overrides_env():
assert config.max_retries == 3
assert config.glossary_max_llm_passes == 2
assert config.grammar_max_llm_passes == 3
assert config.normalize_max_segment_gap == 3.0
assert config.normalize_ellipsis_gap == 1.0
assert config.normalize_max_segment_duration == 30.0
assert config.normalize_max_segment_tokens == 256
assert config.work_dir == Path("/tmp/cli-audita")
@@ -126,3 +158,20 @@ def test_legacy_confidence_threshold_env_is_ignored():
assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
assert config.grammar_confidence_threshold == DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD
def test_config_rejects_invalid_normalization_values():
invalid_envs = [
{"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "-1"},
{"AUDITA_NORMALIZE_ELLIPSIS_GAP": "-1"},
{
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "1",
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "2",
},
{"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "0"},
{"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "0"},
{"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "nan"},
]
for env in invalid_envs:
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "key", **env})

153
tests/test_normalization.py Normal file
View File

@@ -0,0 +1,153 @@
from audita.normalization import normalize_transcript
from audita.schemas import parse_source_transcript_json
class WordEstimator:
def estimate_json(self, value):
return len(value[0]["original_text"].split())
def _normalize(raw, **overrides):
defaults = {
"max_segment_gap": 5.0,
"ellipsis_gap": 2.0,
"max_segment_duration": 60.0,
"max_segment_tokens": 2048,
}
defaults.update(overrides)
return normalize_transcript(parse_source_transcript_json(raw), **defaults)
def test_same_speaker_short_gap_merges_with_space():
result = _normalize(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hello"},
{"speaker": "Eric", "start": 2.0, "end": 3.0, "text": "there"}
]
"""
)
assert len(result.transcript) == 1
assert result.transcript[0].id == 1
assert result.transcript[0].text == "Hello there"
assert result.transcript[0].start == 0.0
assert result.transcript[0].end == 3.0
assert result.summary.merge_count == 1
def test_same_speaker_larger_allowed_gap_merges_with_ellipsis():
result = _normalize(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hello"},
{"speaker": "Eric", "start": 4.0, "end": 5.0, "text": "there"}
]
"""
)
assert result.transcript[0].text == "Hello ... there"
def test_different_speakers_do_not_merge():
result = _normalize(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hello"},
{"speaker": "Mike", "start": 1.5, "end": 2.0, "text": "there"}
]
"""
)
assert [segment.text for segment in result.transcript] == ["Hello", "there"]
assert result.summary.merge_count == 0
def test_gap_above_max_does_not_merge():
result = _normalize(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hello"},
{"speaker": "Eric", "start": 7.0, "end": 8.0, "text": "there"}
]
"""
)
assert [segment.text for segment in result.transcript] == ["Hello", "there"]
def test_overlapping_segments_do_not_merge():
result = _normalize(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 2.0, "text": "Hello"},
{"speaker": "Eric", "start": 1.5, "end": 3.0, "text": "there"}
]
"""
)
assert [segment.text for segment in result.transcript] == ["Hello", "there"]
def test_max_duration_prevents_merge():
result = _normalize(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 40.0, "text": "Hello"},
{"speaker": "Eric", "start": 45.0, "end": 50.0, "text": "there"}
]
""",
max_segment_duration=45.0,
)
assert [segment.text for segment in result.transcript] == ["Hello", "there"]
def test_max_token_limit_prevents_merge():
segments = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "one two"},
{"speaker": "Eric", "start": 2.0, "end": 3.0, "text": "three four"}
]
"""
)
result = normalize_transcript(
segments,
max_segment_gap=5.0,
ellipsis_gap=2.0,
max_segment_duration=60.0,
max_segment_tokens=3,
estimator=WordEstimator(),
)
assert [segment.text for segment in result.transcript] == ["one two", "three four"]
def test_shortest_gap_merges_first():
result = _normalize(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "A"},
{"speaker": "Eric", "start": 3.0, "end": 4.0, "text": "B"},
{"speaker": "Eric", "start": 4.5, "end": 5.0, "text": "C"}
]
""",
max_segment_duration=4.0,
)
assert [segment.text for segment in result.transcript] == ["A", "B C"]
def test_fresh_ids_are_assigned_chronologically_and_source_ids_are_discarded():
result = _normalize(
"""
[
{"id": 99, "speaker": "Mike", "start": 10.0, "end": 11.0, "text": "Second"},
{"id": 42, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "First"}
]
"""
)
assert [(segment.id, segment.text) for segment in result.transcript] == [(1, "First"), (2, "Second")]

70
tests/test_pipeline.py
View File

@@ -2,7 +2,7 @@ import json
from audita.config import AuditaConfig
from audita.pipeline import process_transcript
from audita.schemas import CorrectionCandidate, CorrectionSet, parse_glossary_yaml, parse_transcript_json
from audita.schemas import CorrectionCandidate, CorrectionSet, parse_glossary_yaml, parse_source_transcript_json
class FakeLLMClient:
@@ -42,11 +42,11 @@ def _glossary():
def _transcript():
return parse_transcript_json(
return parse_source_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 10.0, "end": 11.0, "text": "I ask Chontia."},
{"id": 2, "speaker": "Mike", "start": 0.0, "end": 1.0, "text": "Then Lyra."}
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I ask Chontia."},
{"speaker": "Mike", "start": 10.0, "end": 11.0, "text": "Then Lyra."}
]
"""
)
@@ -73,12 +73,44 @@ def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path):
llm_client=fake_client,
)
assert [segment.speaker for segment in revised] == ["Mike", "Eric"]
assert revised[1].text == "I ask Chauntea."
assert [segment.speaker for segment in revised] == ["Eric", "Mike"]
assert revised[0].text == "I ask Chauntea."
assert fake_client.calls == 2
assert list((tmp_path / "work").iterdir()) == []
def test_pipeline_normalizes_before_llm_prompts(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I ask"},
{"speaker": "Eric", "start": 2.0, "end": 3.0, "text": "Chontia."},
{"speaker": "Mike", "start": 10.0, "end": 11.0, "text": "Then Lyra."}
]
"""
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[]),
]
)
progress = []
process_transcript(
transcript,
_glossary(),
_config(tmp_path),
llm_client=fake_client,
progress=progress.append,
)
glossary_prompt = fake_client.messages[0][1]["content"]
glossary_payload = json.loads(glossary_prompt.split("Transcript section:\n", maxsplit=1)[1])
assert glossary_payload[0] == {"id": 1, "original_text": "I ask Chontia."}
assert any("Normalized transcript from 3 to 2 segments" in message for message in progress)
def test_pipeline_skips_bad_glossary_correction_and_preserves_diagnostics(tmp_path):
correction = CorrectionCandidate(
id=1,
@@ -102,7 +134,7 @@ def test_pipeline_skips_bad_glossary_correction_and_preserves_diagnostics(tmp_pa
progress=progress.append,
)
assert revised[1].text == "I ask Chontia."
assert revised[0].text == "I ask Chontia."
assert any("Skipping glossary correction for id 1" in message for message in progress)
preserved = list((tmp_path / "work").iterdir())
assert len(preserved) == 1
@@ -143,8 +175,8 @@ def test_pipeline_retries_skipped_segment_and_cleans_work_dir_when_fixed(tmp_pat
)
assert fake_client.calls == 3
assert [segment.speaker for segment in revised] == ["Mike", "Eric"]
assert revised[1].text == "I ask Chauntea."
assert [segment.speaker for segment in revised] == ["Eric", "Mike"]
assert revised[0].text == "I ask Chauntea."
assert list((tmp_path / "work").iterdir()) == []
@@ -215,7 +247,13 @@ def test_pipeline_writes_stage_metadata_for_unresolved_retries(tmp_path):
assert len(run_dirs) == 1
assert (run_dirs[0] / "glossary" / "pass-0001").exists()
assert (run_dirs[0] / "grammar" / "pass-0001").exists()
assert (run_dirs[0] / "normalization" / "source-transcript.json").exists()
assert (run_dirs[0] / "normalization" / "normalized-transcript.json").exists()
assert (run_dirs[0] / "normalization" / "summary.json").exists()
metadata = json.loads((run_dirs[0] / "metadata.json").read_text(encoding="utf-8"))
assert metadata["normalization"]["source_segment_count"] == 2
assert metadata["normalization"]["normalized_segment_count"] == 2
assert metadata["normalization"]["merge_count"] == 0
assert metadata["glossary_max_llm_passes"] == 2
assert metadata["grammar_max_llm_passes"] == 3
assert metadata["glossary_confidence_threshold"] == 0.8
@@ -227,11 +265,11 @@ def test_pipeline_writes_stage_metadata_for_unresolved_retries(tmp_path):
def test_grammar_stage_runs_after_glossary_and_sees_corrected_text(tmp_path):
transcript = parse_transcript_json(
transcript = parse_source_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 10.0, "end": 11.0, "text": "i ask Chontia."},
{"id": 2, "speaker": "Mike", "start": 0.0, "end": 1.0, "text": "Then Lyra."}
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "i ask Chontia."},
{"speaker": "Mike", "start": 10.0, "end": 11.0, "text": "Then Lyra."}
]
"""
)
@@ -264,14 +302,14 @@ def test_grammar_stage_runs_after_glossary_and_sees_corrected_text(tmp_path):
grammar_prompt = fake_client.messages[1][1]["content"]
grammar_payload = json.loads(grammar_prompt.split("Transcript section:\n", maxsplit=1)[1])
assert grammar_payload[0]["original_text"] == "i ask Chauntea."
assert revised[1].text == "I ask Chauntea."
assert revised[0].text == "I ask Chauntea."
def test_grammar_stage_retries_repeated_span_and_applies_unique_retry(tmp_path):
transcript = parse_transcript_json(
transcript = parse_source_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "there and there."}
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "there and there."}
]
"""
)
@@ -331,7 +369,7 @@ def test_below_threshold_grammar_corrections_are_not_retried(tmp_path):
)
assert fake_client.calls == 2
assert revised[1].text == "I ask Chontia."
assert revised[0].text == "I ask Chontia."
def test_unresolved_grammar_skip_preserves_diagnostics(tmp_path):

66
tests/test_validation.py
View File

@@ -1,7 +1,7 @@
import pytest
from audita.errors import AuditaValidationError
from audita.schemas import parse_glossary_yaml, parse_transcript_json
from audita.schemas import parse_glossary_yaml, parse_source_transcript_json, parse_transcript_json
def test_valid_transcript_parses():
@@ -102,6 +102,70 @@ def test_transcript_rejects_empty_input():
parse_transcript_json("[]")
def test_source_transcript_accepts_missing_ids():
segments = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"}
]
"""
)
assert segments[0].id is None
assert segments[0].speaker == "Eric"
def test_source_transcript_accepts_present_nonsequential_ids():
segments = parse_source_transcript_json(
"""
[
{"id": 10, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"},
{"id": 4, "speaker": "Mike", "start": 1.0, "end": 2.0, "text": "There"}
]
"""
)
assert [segment.id for segment in segments] == [10, 4]
def test_source_transcript_rejects_extra_fields():
with pytest.raises(AuditaValidationError):
parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi", "extra": true}
]
"""
)
def test_source_transcript_rejects_bad_timestamps():
with pytest.raises(AuditaValidationError):
parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 2.0, "end": 1.0, "text": "Hi"}
]
"""
)
def test_source_transcript_rejects_empty_values():
with pytest.raises(AuditaValidationError):
parse_source_transcript_json(
"""
[
{"speaker": "", "start": 0.0, "end": 1.0, "text": "Hi"}
]
"""
)
def test_source_transcript_rejects_invalid_json():
with pytest.raises(AuditaValidationError):
parse_source_transcript_json("{")
def test_valid_glossary_parses():
glossary = parse_glossary_yaml(
"""