eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Implemented deterministic transcript normalization before the LLM stages

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-22 07:22:40 -05:00
parent 445329de81
commit c1672af9b4
12 changed files with 719 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
tests/test_pipeline.py
View File

@@ -2,7 +2,7 @@ import json
from audita.config import AuditaConfig
from audita.pipeline import process_transcript
from audita.schemas import CorrectionCandidate, CorrectionSet, parse_glossary_yaml, parse_transcript_json
from audita.schemas import CorrectionCandidate, CorrectionSet, parse_glossary_yaml, parse_source_transcript_json
class FakeLLMClient:
@@ -42,11 +42,11 @@ def _glossary():
def _transcript():
return parse_transcript_json(
return parse_source_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 10.0, "end": 11.0, "text": "I ask Chontia."},
{"id": 2, "speaker": "Mike", "start": 0.0, "end": 1.0, "text": "Then Lyra."}
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I ask Chontia."},
{"speaker": "Mike", "start": 10.0, "end": 11.0, "text": "Then Lyra."}
]
"""
)
@@ -73,12 +73,44 @@ def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path):
llm_client=fake_client,
)
assert [segment.speaker for segment in revised] == ["Mike", "Eric"]
assert revised[1].text == "I ask Chauntea."
assert [segment.speaker for segment in revised] == ["Eric", "Mike"]
assert revised[0].text == "I ask Chauntea."
assert fake_client.calls == 2
assert list((tmp_path / "work").iterdir()) == []
def test_pipeline_normalizes_before_llm_prompts(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I ask"},
{"speaker": "Eric", "start": 2.0, "end": 3.0, "text": "Chontia."},
{"speaker": "Mike", "start": 10.0, "end": 11.0, "text": "Then Lyra."}
]
"""
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[]),
]
)
progress = []
process_transcript(
transcript,
_glossary(),
_config(tmp_path),
llm_client=fake_client,
progress=progress.append,
)
glossary_prompt = fake_client.messages[0][1]["content"]
glossary_payload = json.loads(glossary_prompt.split("Transcript section:\n", maxsplit=1)[1])
assert glossary_payload[0] == {"id": 1, "original_text": "I ask Chontia."}
assert any("Normalized transcript from 3 to 2 segments" in message for message in progress)
def test_pipeline_skips_bad_glossary_correction_and_preserves_diagnostics(tmp_path):
correction = CorrectionCandidate(
id=1,
@@ -102,7 +134,7 @@ def test_pipeline_skips_bad_glossary_correction_and_preserves_diagnostics(tmp_pa
progress=progress.append,
)
assert revised[1].text == "I ask Chontia."
assert revised[0].text == "I ask Chontia."
assert any("Skipping glossary correction for id 1" in message for message in progress)
preserved = list((tmp_path / "work").iterdir())
assert len(preserved) == 1
@@ -143,8 +175,8 @@ def test_pipeline_retries_skipped_segment_and_cleans_work_dir_when_fixed(tmp_pat
)
assert fake_client.calls == 3
assert [segment.speaker for segment in revised] == ["Mike", "Eric"]
assert revised[1].text == "I ask Chauntea."
assert [segment.speaker for segment in revised] == ["Eric", "Mike"]
assert revised[0].text == "I ask Chauntea."
assert list((tmp_path / "work").iterdir()) == []
@@ -215,7 +247,13 @@ def test_pipeline_writes_stage_metadata_for_unresolved_retries(tmp_path):
assert len(run_dirs) == 1
assert (run_dirs[0] / "glossary" / "pass-0001").exists()
assert (run_dirs[0] / "grammar" / "pass-0001").exists()
assert (run_dirs[0] / "normalization" / "source-transcript.json").exists()
assert (run_dirs[0] / "normalization" / "normalized-transcript.json").exists()
assert (run_dirs[0] / "normalization" / "summary.json").exists()
metadata = json.loads((run_dirs[0] / "metadata.json").read_text(encoding="utf-8"))
assert metadata["normalization"]["source_segment_count"] == 2
assert metadata["normalization"]["normalized_segment_count"] == 2
assert metadata["normalization"]["merge_count"] == 0
assert metadata["glossary_max_llm_passes"] == 2
assert metadata["grammar_max_llm_passes"] == 3
assert metadata["glossary_confidence_threshold"] == 0.8
@@ -227,11 +265,11 @@ def test_pipeline_writes_stage_metadata_for_unresolved_retries(tmp_path):
def test_grammar_stage_runs_after_glossary_and_sees_corrected_text(tmp_path):
transcript = parse_transcript_json(
transcript = parse_source_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 10.0, "end": 11.0, "text": "i ask Chontia."},
{"id": 2, "speaker": "Mike", "start": 0.0, "end": 1.0, "text": "Then Lyra."}
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "i ask Chontia."},
{"speaker": "Mike", "start": 10.0, "end": 11.0, "text": "Then Lyra."}
]
"""
)
@@ -264,14 +302,14 @@ def test_grammar_stage_runs_after_glossary_and_sees_corrected_text(tmp_path):
grammar_prompt = fake_client.messages[1][1]["content"]
grammar_payload = json.loads(grammar_prompt.split("Transcript section:\n", maxsplit=1)[1])
assert grammar_payload[0]["original_text"] == "i ask Chauntea."
assert revised[1].text == "I ask Chauntea."
assert revised[0].text == "I ask Chauntea."
def test_grammar_stage_retries_repeated_span_and_applies_unique_retry(tmp_path):
transcript = parse_transcript_json(
transcript = parse_source_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "there and there."}
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "there and there."}
]
"""
)
@@ -331,7 +369,7 @@ def test_below_threshold_grammar_corrections_are_not_retried(tmp_path):
)
assert fake_client.calls == 2
assert revised[1].text == "I ask Chontia."
assert revised[0].text == "I ask Chontia."
def test_unresolved_grammar_skip_preserves_diagnostics(tmp_path):