eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Implemented a second-stage validator for grammar corrections that allows homophone-related changes

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-23 11:25:35 -05:00
parent dee6ae4067
commit c4e2db75f1
13 changed files with 638 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
tests/test_pipeline.py
View File

@@ -7,6 +7,8 @@ from audita.errors import AuditaError
from audita.pipeline import process_transcript
from audita.schemas import (
CorrectionCandidate,
GrammarSpokenFormValidationDecision,
GrammarSpokenFormValidationSet,
CorrectionSet,
GrammarValidationDecision,
GrammarValidationSet,
@@ -16,13 +18,16 @@ from audita.schemas import (
class FakeLLMClient:
def __init__(self, responses, validation_responses=None):
def __init__(self, responses, validation_responses=None, spoken_form_validation_responses=None):
self.responses = list(responses)
self.validation_responses = list(validation_responses or [])
self.spoken_form_validation_responses = list(spoken_form_validation_responses or [])
self.calls = 0
self.validation_calls = 0
self.spoken_form_validation_calls = 0
self.messages = []
self.validation_messages = []
self.spoken_form_validation_messages = []
def create_corrections(self, messages, config):
self.calls += 1
@@ -36,6 +41,13 @@ class FakeLLMClient:
raise AssertionError("Unexpected grammar validation request.")
return self.validation_responses.pop(0)
def create_grammar_spoken_form_validations(self, messages, config):
self.spoken_form_validation_calls += 1
self.spoken_form_validation_messages.append(messages)
if not self.spoken_form_validation_responses:
raise AssertionError("Unexpected spoken-form validation request.")
return self.spoken_form_validation_responses.pop(0)
def _config(
tmp_path,
@@ -43,6 +55,7 @@ def _config(
grammar_max_llm_passes=3,
grammar_validation_enabled=False,
grammar_validation_confidence_threshold=0.8,
grammar_spoken_form_validation_confidence_threshold=0.8,
):
return AuditaConfig(
api_key="key",
@@ -54,6 +67,7 @@ def _config(
grammar_max_llm_passes=grammar_max_llm_passes,
grammar_validation_enabled=grammar_validation_enabled,
grammar_validation_confidence_threshold=grammar_validation_confidence_threshold,
grammar_spoken_form_validation_confidence_threshold=grammar_spoken_form_validation_confidence_threshold,
work_dir=tmp_path / "work",
)
@@ -698,6 +712,7 @@ def test_pipeline_writes_stage_metadata_for_unresolved_retries(tmp_path):
assert metadata["grammar_confidence_threshold"] == 0.8
assert metadata["grammar_validation_enabled"] is False
assert metadata["grammar_validation_confidence_threshold"] == 0.8
assert metadata["grammar_spoken_form_validation_confidence_threshold"] == 0.8
assert [item["stage"] for item in metadata["stages"]] == ["glossary", "grammar"]
assert [item["pass_number"] for item in metadata["stages"][0]["passes"]] == [1, 2]
assert metadata["stages"][0]["passes"][0]["retry_segment_count"] == 1
@@ -766,12 +781,21 @@ def test_grammar_validation_rejects_semantic_change(tmp_path):
confidence=0.99,
reason="This reverses visible to invisible.",
)
spoken_form_validation = GrammarSpokenFormValidationDecision(
correction_index=0,
is_likely_spoken_form_correction=False,
confidence=0.98,
reason="This is a semantic reversal, not a likely spoken-form correction.",
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[grammar_correction]),
],
validation_responses=[GrammarValidationSet(validations=[validation])],
spoken_form_validation_responses=[
GrammarSpokenFormValidationSet(validations=[spoken_form_validation])
],
)
revised = process_transcript(
@@ -783,20 +807,24 @@ def test_grammar_validation_rejects_semantic_change(tmp_path):
assert revised[0].text == "He became visible."
assert fake_client.validation_calls == 1
assert fake_client.spoken_form_validation_calls == 1
run_dirs = list((tmp_path / "work").iterdir())
assert len(run_dirs) == 1
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
skipped = diagnostics["skipped_corrections"][0]
assert skipped["stage"] == "grammar"
assert skipped["reason"] == "grammar validation rejected semantic change"
assert skipped["validation_confidence"] == 0.99
assert skipped["validation_reason"] == "This reverses visible to invisible."
assert skipped["validation_confidence"] == 0.98
assert skipped["validation_reason"] == "This is a semantic reversal, not a likely spoken-form correction."
metadata = json.loads((run_dirs[0] / "metadata.json").read_text(encoding="utf-8"))
grammar_pass = metadata["stages"][1]["passes"][0]
assert grammar_pass["validation_candidate_count"] == 1
assert grammar_pass["validation_approved_count"] == 0
assert grammar_pass["validation_rejected_count"] == 1
assert grammar_pass["validation_bypassed_count"] == 0
assert grammar_pass["spoken_form_validation_candidate_count"] == 1
assert grammar_pass["spoken_form_validation_approved_count"] == 0
assert grammar_pass["spoken_form_validation_rejected_count"] == 1
def test_grammar_validation_accepts_meaning_preserving_fix(tmp_path):
@@ -836,6 +864,57 @@ def test_grammar_validation_accepts_meaning_preserving_fix(tmp_path):
assert revised[0].text == "Keep in mind."
assert fake_client.validation_calls == 1
assert fake_client.spoken_form_validation_calls == 0
assert list((tmp_path / "work").iterdir()) == []
def test_grammar_validation_rescues_likely_spoken_form_fix(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."}
]
"""
)
grammar_correction = CorrectionCandidate(
id=1,
original_text="dam",
corrected_text="damn",
confidence=0.95,
)
meaning_validation = GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=False,
confidence=0.98,
reason="Written meaning changes from a barrier to a curse word.",
)
spoken_form_validation = GrammarSpokenFormValidationDecision(
correction_index=0,
is_likely_spoken_form_correction=True,
confidence=0.97,
reason="The phrase strongly suggests the intended spoken word was the expletive.",
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[grammar_correction]),
],
validation_responses=[GrammarValidationSet(validations=[meaning_validation])],
spoken_form_validation_responses=[
GrammarSpokenFormValidationSet(validations=[spoken_form_validation])
],
)
revised = process_transcript(
transcript,
_glossary(),
_config(tmp_path, grammar_validation_enabled=True),
llm_client=fake_client,
)
assert revised[0].text == "ChatGPT still can't really do that with a damn."
assert fake_client.validation_calls == 1
assert fake_client.spoken_form_validation_calls == 1
assert list((tmp_path / "work").iterdir()) == []
@@ -877,6 +956,7 @@ def test_grammar_validation_bypasses_protected_vocabulary_correction(tmp_path):
assert revised[0].text == "The Jesters arrived."
assert fake_client.validation_calls == 0
assert fake_client.spoken_form_validation_calls == 0
assert list((tmp_path / "work").iterdir()) == []
@@ -910,6 +990,7 @@ def test_disabled_grammar_validation_preserves_current_behavior(tmp_path):
assert revised[0].text == "He became invisible."
assert fake_client.validation_calls == 0
assert fake_client.spoken_form_validation_calls == 0
assert list((tmp_path / "work").iterdir()) == []
@@ -949,6 +1030,54 @@ def test_missing_grammar_validation_decision_fails_and_preserves_diagnostics(tmp
assert (run_dirs[0] / "grammar" / "pass-0001" / "validation-response-0000.json").exists()
def test_missing_spoken_form_validation_decision_fails_and_preserves_diagnostics(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."}
]
"""
)
grammar_correction = CorrectionCandidate(
id=1,
original_text="dam",
corrected_text="damn",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[grammar_correction]),
],
validation_responses=[
GrammarValidationSet(
validations=[
GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=False,
confidence=0.99,
reason="Written meaning changes.",
)
]
)
],
spoken_form_validation_responses=[GrammarSpokenFormValidationSet(validations=[])],
)
with pytest.raises(AuditaError):
process_transcript(
transcript,
_glossary(),
_config(tmp_path, grammar_validation_enabled=True),
llm_client=fake_client,
)
run_dirs = list((tmp_path / "work").iterdir())
assert len(run_dirs) == 1
assert (run_dirs[0] / "grammar" / "pass-0001" / "spoken-form-validation-prompt-0000.json").exists()
assert (run_dirs[0] / "grammar" / "pass-0001" / "spoken-form-validation-response-0000.json").exists()
def test_grammar_stage_cannot_reverse_glossary_protected_term(tmp_path):
transcript = parse_source_transcript_json(
"""