Simplified the json schema passed to the LLM, and implemented more forgiving error handling for LLM proposed corrections

This commit is contained in:
2026-04-21 11:14:56 -05:00
parent 8c80c942dd
commit 23532cade1
9 changed files with 216 additions and 97 deletions

View File

@@ -1,9 +1,6 @@
from pathlib import Path
import pytest
import json
from audita.config import AuditaConfig
from audita.errors import AuditaValidationError
from audita.pipeline import process_transcript
from audita.schemas import CorrectionCandidate, CorrectionSet, parse_glossary_yaml, parse_transcript_json
@@ -51,10 +48,7 @@ def _transcript():
def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path):
correction = CorrectionCandidate(
segment_index=0,
speaker="Eric",
start=0.0,
end=1.0,
segment_id=0,
original_text="I ask Chontia.",
corrected_text="I ask Chauntea.",
confidence=0.95,
@@ -73,27 +67,30 @@ def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path):
assert list((tmp_path / "work").iterdir()) == []
def test_pipeline_preserves_work_dir_on_failure(tmp_path):
def test_pipeline_skips_bad_correction_and_preserves_diagnostics(tmp_path):
correction = CorrectionCandidate(
segment_index=0,
speaker="Eric",
start=0.0,
end=1.0,
segment_id=0,
original_text="Different text.",
corrected_text="I ask Chauntea.",
confidence=0.95,
)
fake_client = FakeLLMClient([CorrectionSet(corrections=[correction])])
progress = []
with pytest.raises(AuditaValidationError):
process_transcript(
_transcript(),
_glossary(),
_config(tmp_path),
llm_client=fake_client,
)
revised = process_transcript(
_transcript(),
_glossary(),
_config(tmp_path),
llm_client=fake_client,
progress=progress.append,
)
assert revised[0].text == "I ask Chontia."
assert any("Skipping correction for segment 0" in message for message in progress)
preserved = list((tmp_path / "work").iterdir())
assert len(preserved) == 1
assert (Path(preserved[0]) / "section-0000.json").exists()
skipped_path = preserved[0] / "skipped-corrections.json"
assert skipped_path.exists()
diagnostics = json.loads(skipped_path.read_text(encoding="utf-8"))
assert diagnostics["skipped_corrections"][0]["segment_id"] == 0
assert "original_text" in diagnostics["skipped_corrections"][0]["reason"]