Simplified the json schema passed to the LLM, and implemented more forgiving error handling for LLM proposed corrections
This commit is contained in:
@@ -20,69 +20,95 @@ def test_apply_corrections_uses_threshold_and_sorts_chronologically():
|
||||
transcript = _transcript()
|
||||
corrections = [
|
||||
CorrectionCandidate(
|
||||
segment_index=0,
|
||||
speaker="Eric",
|
||||
start=10.0,
|
||||
end=11.0,
|
||||
segment_id=0,
|
||||
original_text="I ask Chontia.",
|
||||
corrected_text="I ask Chauntea.",
|
||||
confidence=0.8,
|
||||
)
|
||||
]
|
||||
|
||||
revised = apply_corrections(transcript, corrections, confidence_threshold=0.8)
|
||||
result = apply_corrections(transcript, corrections, confidence_threshold=0.8)
|
||||
|
||||
assert [segment.speaker for segment in revised] == ["Mike", "Eric"]
|
||||
assert revised[1].text == "I ask Chauntea."
|
||||
assert [segment.speaker for segment in result.transcript] == ["Mike", "Eric"]
|
||||
assert result.transcript[1].text == "I ask Chauntea."
|
||||
assert result.skipped == []
|
||||
|
||||
|
||||
def test_apply_corrections_ignores_below_threshold():
|
||||
transcript = _transcript()
|
||||
corrections = [
|
||||
CorrectionCandidate(
|
||||
segment_index=0,
|
||||
speaker="Eric",
|
||||
start=10.0,
|
||||
end=11.0,
|
||||
segment_id=0,
|
||||
original_text="I ask Chontia.",
|
||||
corrected_text="I ask Chauntea.",
|
||||
confidence=0.79,
|
||||
)
|
||||
]
|
||||
|
||||
revised = apply_corrections(transcript, corrections, confidence_threshold=0.8)
|
||||
result = apply_corrections(transcript, corrections, confidence_threshold=0.8)
|
||||
|
||||
assert revised[1].text == "I ask Chontia."
|
||||
assert result.transcript[1].text == "I ask Chontia."
|
||||
assert result.skipped == []
|
||||
|
||||
|
||||
def test_apply_corrections_rejects_duplicate_targets():
|
||||
def test_apply_corrections_skips_duplicate_targets():
|
||||
transcript = _transcript()
|
||||
correction = CorrectionCandidate(
|
||||
segment_index=0,
|
||||
speaker="Eric",
|
||||
start=10.0,
|
||||
end=11.0,
|
||||
first = CorrectionCandidate(
|
||||
segment_id=0,
|
||||
original_text="I ask Chontia.",
|
||||
corrected_text="I ask Chauntea.",
|
||||
confidence=0.8,
|
||||
)
|
||||
second = CorrectionCandidate(
|
||||
segment_id=0,
|
||||
original_text="I ask Chontia.",
|
||||
corrected_text="I ask Something Else.",
|
||||
confidence=0.9,
|
||||
)
|
||||
|
||||
with pytest.raises(AuditaValidationError):
|
||||
apply_corrections(transcript, [correction, correction], confidence_threshold=0.8)
|
||||
result = apply_corrections(transcript, [first, second], confidence_threshold=0.8)
|
||||
|
||||
assert result.transcript[1].text == "I ask Chauntea."
|
||||
assert len(result.skipped) == 1
|
||||
assert result.skipped[0].segment_id == 0
|
||||
assert "duplicate" in result.skipped[0].reason
|
||||
|
||||
|
||||
def test_apply_corrections_rejects_mismatched_original_text():
|
||||
def test_apply_corrections_skips_mismatched_original_text():
|
||||
transcript = _transcript()
|
||||
correction = CorrectionCandidate(
|
||||
segment_index=0,
|
||||
speaker="Eric",
|
||||
start=10.0,
|
||||
end=11.0,
|
||||
segment_id=0,
|
||||
original_text="Different text.",
|
||||
corrected_text="I ask Chauntea.",
|
||||
confidence=0.8,
|
||||
)
|
||||
|
||||
with pytest.raises(AuditaValidationError):
|
||||
apply_corrections(transcript, [correction], confidence_threshold=0.8)
|
||||
result = apply_corrections(transcript, [correction], confidence_threshold=0.8)
|
||||
|
||||
assert result.transcript[1].text == "I ask Chontia."
|
||||
assert len(result.skipped) == 1
|
||||
assert result.skipped[0].segment_id == 0
|
||||
assert result.skipped[0].actual_text == "I ask Chontia."
|
||||
assert "original_text" in result.skipped[0].reason
|
||||
|
||||
|
||||
def test_apply_corrections_skips_missing_segment_id():
|
||||
transcript = _transcript()
|
||||
correction = CorrectionCandidate(
|
||||
segment_id=99,
|
||||
original_text="Missing.",
|
||||
corrected_text="Still missing.",
|
||||
confidence=0.8,
|
||||
)
|
||||
|
||||
result = apply_corrections(transcript, [correction], confidence_threshold=0.8)
|
||||
|
||||
assert [segment.text for segment in result.transcript] == ["Then Lyra.", "I ask Chontia."]
|
||||
assert len(result.skipped) == 1
|
||||
assert result.skipped[0].segment_id == 99
|
||||
assert "does not exist" in result.skipped[0].reason
|
||||
|
||||
|
||||
def test_apply_corrections_rejects_invalid_threshold():
|
||||
with pytest.raises(AuditaValidationError):
|
||||
apply_corrections(_transcript(), [], confidence_threshold=1.1)
|
||||
|
||||
Reference in New Issue
Block a user