Simplified the json schema passed to the LLM, and implemented more forgiving error handling for LLM proposed corrections
This commit is contained in:
@@ -1,9 +1,6 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import json
|
||||
|
||||
from audita.config import AuditaConfig
|
||||
from audita.errors import AuditaValidationError
|
||||
from audita.pipeline import process_transcript
|
||||
from audita.schemas import CorrectionCandidate, CorrectionSet, parse_glossary_yaml, parse_transcript_json
|
||||
|
||||
@@ -51,10 +48,7 @@ def _transcript():
|
||||
|
||||
def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path):
|
||||
correction = CorrectionCandidate(
|
||||
segment_index=0,
|
||||
speaker="Eric",
|
||||
start=0.0,
|
||||
end=1.0,
|
||||
segment_id=0,
|
||||
original_text="I ask Chontia.",
|
||||
corrected_text="I ask Chauntea.",
|
||||
confidence=0.95,
|
||||
@@ -73,27 +67,30 @@ def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path):
|
||||
assert list((tmp_path / "work").iterdir()) == []
|
||||
|
||||
|
||||
def test_pipeline_preserves_work_dir_on_failure(tmp_path):
|
||||
def test_pipeline_skips_bad_correction_and_preserves_diagnostics(tmp_path):
|
||||
correction = CorrectionCandidate(
|
||||
segment_index=0,
|
||||
speaker="Eric",
|
||||
start=0.0,
|
||||
end=1.0,
|
||||
segment_id=0,
|
||||
original_text="Different text.",
|
||||
corrected_text="I ask Chauntea.",
|
||||
confidence=0.95,
|
||||
)
|
||||
fake_client = FakeLLMClient([CorrectionSet(corrections=[correction])])
|
||||
progress = []
|
||||
|
||||
with pytest.raises(AuditaValidationError):
|
||||
process_transcript(
|
||||
_transcript(),
|
||||
_glossary(),
|
||||
_config(tmp_path),
|
||||
llm_client=fake_client,
|
||||
)
|
||||
revised = process_transcript(
|
||||
_transcript(),
|
||||
_glossary(),
|
||||
_config(tmp_path),
|
||||
llm_client=fake_client,
|
||||
progress=progress.append,
|
||||
)
|
||||
|
||||
assert revised[0].text == "I ask Chontia."
|
||||
assert any("Skipping correction for segment 0" in message for message in progress)
|
||||
preserved = list((tmp_path / "work").iterdir())
|
||||
assert len(preserved) == 1
|
||||
assert (Path(preserved[0]) / "section-0000.json").exists()
|
||||
|
||||
skipped_path = preserved[0] / "skipped-corrections.json"
|
||||
assert skipped_path.exists()
|
||||
diagnostics = json.loads(skipped_path.read_text(encoding="utf-8"))
|
||||
assert diagnostics["skipped_corrections"][0]["segment_id"] == 0
|
||||
assert "original_text" in diagnostics["skipped_corrections"][0]["reason"]
|
||||
|
||||
Reference in New Issue
Block a user