eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Implemented a second-stage validator for grammar corrections that allows homophone-related changes

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-23 11:25:35 -05:00
parent dee6ae4067
commit c4e2db75f1
13 changed files with 638 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/test_cli.py
View File

@@ -23,6 +23,7 @@ def test_process_help_includes_glossary_pass_flag(capsys):
assert "--grammar-confidence-threshold" in output
assert "--grammar-validation-enabled" in output
assert "--grammar-validation-confidence-threshold" in output
assert "--grammar-spoken-form-validation-confidence-threshold" in output
assert "--normalize-max-segment-gap" in output
assert "--normalize-ellipsis-gap" in output
assert "--normalize-max-segment-duration" in output

18
tests/test_config.py
View File

@@ -8,6 +8,7 @@ from audita.config import (
DEFAULT_GLOSSARY_MAX_LLM_PASSES,
DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD,
DEFAULT_GRAMMAR_MAX_LLM_PASSES,
DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD,
DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD,
DEFAULT_GRAMMAR_VALIDATION_ENABLED,
DEFAULT_MAX_RETRIES,
@@ -36,6 +37,11 @@ def test_config_uses_defaults_with_api_key():
assert config.grammar_validation_enabled is True
assert config.grammar_validation_confidence_threshold == DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD
assert config.grammar_validation_confidence_threshold == 0.8
assert (
config.grammar_spoken_form_validation_confidence_threshold
== DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD
)
assert config.grammar_spoken_form_validation_confidence_threshold == 0.8
assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
assert config.normalize_max_segment_gap == 5.0
assert config.normalize_ellipsis_gap == DEFAULT_NORMALIZE_ELLIPSIS_GAP
@@ -58,6 +64,7 @@ def test_config_env_overrides_defaults():
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "4",
"AUDITA_GRAMMAR_VALIDATION_ENABLED": "false",
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "0.91",
"AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD": "0.87",
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "4.5",
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "1.5",
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0",
@@ -74,6 +81,7 @@ def test_config_env_overrides_defaults():
assert config.grammar_max_llm_passes == 4
assert config.grammar_validation_enabled is False
assert config.grammar_validation_confidence_threshold == 0.91
assert config.grammar_spoken_form_validation_confidence_threshold == 0.87
assert config.normalize_max_segment_gap == 4.5
assert config.normalize_ellipsis_gap == 1.5
assert config.normalize_max_segment_duration == 45.0
@@ -91,6 +99,7 @@ def test_config_cli_overrides_env():
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "6",
"AUDITA_GRAMMAR_VALIDATION_ENABLED": "false",
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "0.91",
"AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD": "0.87",
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "4.5",
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "1.5",
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0",
@@ -106,6 +115,7 @@ def test_config_cli_overrides_env():
grammar_max_llm_passes=3,
grammar_validation_enabled=True,
grammar_validation_confidence_threshold=0.75,
grammar_spoken_form_validation_confidence_threshold=0.72,
normalize_max_segment_gap=3.0,
normalize_ellipsis_gap=1.0,
normalize_max_segment_duration=30.0,
@@ -122,6 +132,7 @@ def test_config_cli_overrides_env():
assert config.grammar_max_llm_passes == 3
assert config.grammar_validation_enabled is True
assert config.grammar_validation_confidence_threshold == 0.75
assert config.grammar_spoken_form_validation_confidence_threshold == 0.72
assert config.normalize_max_segment_gap == 3.0
assert config.normalize_ellipsis_gap == 1.0
assert config.normalize_max_segment_duration == 30.0
@@ -168,6 +179,13 @@ def test_config_rejects_invalid_stage_thresholds():
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "1.1"}
)
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={
"OPENROUTER_API_KEY": "key",
"AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD": "-0.1",
}
)
def test_config_rejects_invalid_grammar_validation_enabled():

135
tests/test_pipeline.py
View File

@@ -7,6 +7,8 @@ from audita.errors import AuditaError
from audita.pipeline import process_transcript
from audita.schemas import (
CorrectionCandidate,
GrammarSpokenFormValidationDecision,
GrammarSpokenFormValidationSet,
CorrectionSet,
GrammarValidationDecision,
GrammarValidationSet,
@@ -16,13 +18,16 @@ from audita.schemas import (
class FakeLLMClient:
def __init__(self, responses, validation_responses=None):
def __init__(self, responses, validation_responses=None, spoken_form_validation_responses=None):
self.responses = list(responses)
self.validation_responses = list(validation_responses or [])
self.spoken_form_validation_responses = list(spoken_form_validation_responses or [])
self.calls = 0
self.validation_calls = 0
self.spoken_form_validation_calls = 0
self.messages = []
self.validation_messages = []
self.spoken_form_validation_messages = []
def create_corrections(self, messages, config):
self.calls += 1
@@ -36,6 +41,13 @@ class FakeLLMClient:
raise AssertionError("Unexpected grammar validation request.")
return self.validation_responses.pop(0)
def create_grammar_spoken_form_validations(self, messages, config):
self.spoken_form_validation_calls += 1
self.spoken_form_validation_messages.append(messages)
if not self.spoken_form_validation_responses:
raise AssertionError("Unexpected spoken-form validation request.")
return self.spoken_form_validation_responses.pop(0)
def _config(
tmp_path,
@@ -43,6 +55,7 @@ def _config(
grammar_max_llm_passes=3,
grammar_validation_enabled=False,
grammar_validation_confidence_threshold=0.8,
grammar_spoken_form_validation_confidence_threshold=0.8,
):
return AuditaConfig(
api_key="key",
@@ -54,6 +67,7 @@ def _config(
grammar_max_llm_passes=grammar_max_llm_passes,
grammar_validation_enabled=grammar_validation_enabled,
grammar_validation_confidence_threshold=grammar_validation_confidence_threshold,
grammar_spoken_form_validation_confidence_threshold=grammar_spoken_form_validation_confidence_threshold,
work_dir=tmp_path / "work",
)
@@ -698,6 +712,7 @@ def test_pipeline_writes_stage_metadata_for_unresolved_retries(tmp_path):
assert metadata["grammar_confidence_threshold"] == 0.8
assert metadata["grammar_validation_enabled"] is False
assert metadata["grammar_validation_confidence_threshold"] == 0.8
assert metadata["grammar_spoken_form_validation_confidence_threshold"] == 0.8
assert [item["stage"] for item in metadata["stages"]] == ["glossary", "grammar"]
assert [item["pass_number"] for item in metadata["stages"][0]["passes"]] == [1, 2]
assert metadata["stages"][0]["passes"][0]["retry_segment_count"] == 1
@@ -766,12 +781,21 @@ def test_grammar_validation_rejects_semantic_change(tmp_path):
confidence=0.99,
reason="This reverses visible to invisible.",
)
spoken_form_validation = GrammarSpokenFormValidationDecision(
correction_index=0,
is_likely_spoken_form_correction=False,
confidence=0.98,
reason="This is a semantic reversal, not a likely spoken-form correction.",
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[grammar_correction]),
],
validation_responses=[GrammarValidationSet(validations=[validation])],
spoken_form_validation_responses=[
GrammarSpokenFormValidationSet(validations=[spoken_form_validation])
],
)
revised = process_transcript(
@@ -783,20 +807,24 @@ def test_grammar_validation_rejects_semantic_change(tmp_path):
assert revised[0].text == "He became visible."
assert fake_client.validation_calls == 1
assert fake_client.spoken_form_validation_calls == 1
run_dirs = list((tmp_path / "work").iterdir())
assert len(run_dirs) == 1
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
skipped = diagnostics["skipped_corrections"][0]
assert skipped["stage"] == "grammar"
assert skipped["reason"] == "grammar validation rejected semantic change"
assert skipped["validation_confidence"] == 0.99
assert skipped["validation_reason"] == "This reverses visible to invisible."
assert skipped["validation_confidence"] == 0.98
assert skipped["validation_reason"] == "This is a semantic reversal, not a likely spoken-form correction."
metadata = json.loads((run_dirs[0] / "metadata.json").read_text(encoding="utf-8"))
grammar_pass = metadata["stages"][1]["passes"][0]
assert grammar_pass["validation_candidate_count"] == 1
assert grammar_pass["validation_approved_count"] == 0
assert grammar_pass["validation_rejected_count"] == 1
assert grammar_pass["validation_bypassed_count"] == 0
assert grammar_pass["spoken_form_validation_candidate_count"] == 1
assert grammar_pass["spoken_form_validation_approved_count"] == 0
assert grammar_pass["spoken_form_validation_rejected_count"] == 1
def test_grammar_validation_accepts_meaning_preserving_fix(tmp_path):
@@ -836,6 +864,57 @@ def test_grammar_validation_accepts_meaning_preserving_fix(tmp_path):
assert revised[0].text == "Keep in mind."
assert fake_client.validation_calls == 1
assert fake_client.spoken_form_validation_calls == 0
assert list((tmp_path / "work").iterdir()) == []
def test_grammar_validation_rescues_likely_spoken_form_fix(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."}
]
"""
)
grammar_correction = CorrectionCandidate(
id=1,
original_text="dam",
corrected_text="damn",
confidence=0.95,
)
meaning_validation = GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=False,
confidence=0.98,
reason="Written meaning changes from a barrier to a curse word.",
)
spoken_form_validation = GrammarSpokenFormValidationDecision(
correction_index=0,
is_likely_spoken_form_correction=True,
confidence=0.97,
reason="The phrase strongly suggests the intended spoken word was the expletive.",
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[grammar_correction]),
],
validation_responses=[GrammarValidationSet(validations=[meaning_validation])],
spoken_form_validation_responses=[
GrammarSpokenFormValidationSet(validations=[spoken_form_validation])
],
)
revised = process_transcript(
transcript,
_glossary(),
_config(tmp_path, grammar_validation_enabled=True),
llm_client=fake_client,
)
assert revised[0].text == "ChatGPT still can't really do that with a damn."
assert fake_client.validation_calls == 1
assert fake_client.spoken_form_validation_calls == 1
assert list((tmp_path / "work").iterdir()) == []
@@ -877,6 +956,7 @@ def test_grammar_validation_bypasses_protected_vocabulary_correction(tmp_path):
assert revised[0].text == "The Jesters arrived."
assert fake_client.validation_calls == 0
assert fake_client.spoken_form_validation_calls == 0
assert list((tmp_path / "work").iterdir()) == []
@@ -910,6 +990,7 @@ def test_disabled_grammar_validation_preserves_current_behavior(tmp_path):
assert revised[0].text == "He became invisible."
assert fake_client.validation_calls == 0
assert fake_client.spoken_form_validation_calls == 0
assert list((tmp_path / "work").iterdir()) == []
@@ -949,6 +1030,54 @@ def test_missing_grammar_validation_decision_fails_and_preserves_diagnostics(tmp
assert (run_dirs[0] / "grammar" / "pass-0001" / "validation-response-0000.json").exists()
def test_missing_spoken_form_validation_decision_fails_and_preserves_diagnostics(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."}
]
"""
)
grammar_correction = CorrectionCandidate(
id=1,
original_text="dam",
corrected_text="damn",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[grammar_correction]),
],
validation_responses=[
GrammarValidationSet(
validations=[
GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=False,
confidence=0.99,
reason="Written meaning changes.",
)
]
)
],
spoken_form_validation_responses=[GrammarSpokenFormValidationSet(validations=[])],
)
with pytest.raises(AuditaError):
process_transcript(
transcript,
_glossary(),
_config(tmp_path, grammar_validation_enabled=True),
llm_client=fake_client,
)
run_dirs = list((tmp_path / "work").iterdir())
assert len(run_dirs) == 1
assert (run_dirs[0] / "grammar" / "pass-0001" / "spoken-form-validation-prompt-0000.json").exists()
assert (run_dirs[0] / "grammar" / "pass-0001" / "spoken-form-validation-response-0000.json").exists()
def test_grammar_stage_cannot_reverse_glossary_protected_term(tmp_path):
transcript = parse_source_transcript_json(
"""

25
tests/test_prompts.py
View File

@@ -4,6 +4,7 @@ from audita.chunking import chunk_transcript
from audita.prompts import (
build_glossary_correction_messages,
build_grammar_correction_messages,
build_grammar_spoken_form_validation_messages,
build_grammar_validation_messages,
)
from audita.schemas import parse_glossary_yaml, parse_transcript_json
@@ -194,6 +195,28 @@ def test_grammar_validation_prompt_rejects_semantic_changes():
assert "became visible" in prompt_text
assert "became invisible" in prompt_text
assert "reverses the meaning" in prompt_text
assert "spelling and homophone fixes only when" in prompt_text
assert "do not try to rescue likely homophone or transcription fixes" in prompt_text
assert "handled in a separate spoken-form validation step" in prompt_text
assert "correction_index" in prompt_text
assert "is_meaning_preserving" in prompt_text
def test_grammar_spoken_form_validation_prompt_allows_homophone_rescue():
messages = build_grammar_spoken_form_validation_messages(
[
{
"correction_index": 0,
"id": 1,
"original_segment_text": "ChatGPT still can't really do that with a dam.",
"corrected_segment_text": "ChatGPT still can't really do that with a damn.",
"original_text": "dam",
"corrected_text": "damn",
}
]
)
prompt_text = "\n".join(message["content"] for message in messages)
assert "likely homophone, spoken-form, or transcription fix" in prompt_text
assert '"dam" to "damn"' in prompt_text
assert '"became visible" to "became invisible"' in prompt_text
assert "is_likely_spoken_form_correction" in prompt_text

197
tests/test_semantic_validation.py
View File

@@ -1,16 +1,22 @@
import pytest
from audita.errors import AuditaLLMError
from audita.semantic_validation import select_grammar_validation_candidates
from audita.semantic_validation import filter_with_grammar_validations
from audita.protection import ProtectedVocabulary
from audita.schemas import (
CorrectionCandidate,
GrammarSpokenFormValidationDecision,
GrammarSpokenFormValidationSet,
GrammarValidationDecision,
GrammarValidationSet,
parse_glossary_yaml,
parse_transcript_json,
)
from audita.semantic_validation import (
filter_with_meaning_preserving_validations,
filter_with_spoken_form_validations,
keep_corrections_with_indexes,
select_grammar_validation_candidates,
)
def _transcript():
@@ -118,7 +124,106 @@ def test_meaning_sensitive_substitution_requires_validation():
assert bypassed_count == 0
def test_validation_rejects_duplicate_and_unknown_decisions():
def test_meaning_preserving_validation_approves_without_rescue():
correction = CorrectionCandidate(
id=1,
original_text="bind",
corrected_text="mind",
confidence=0.95,
)
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Keep in bind."}
]
"""
)
candidates, _ = select_grammar_validation_candidates(
transcript,
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
result = filter_with_meaning_preserving_validations(
candidates,
GrammarValidationSet(
validations=[
GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=True,
confidence=0.95,
reason="This preserves the intended meaning.",
)
]
),
confidence_threshold=0.8,
)
assert result.approved_correction_indexes == [0]
assert result.rescue_candidates == []
assert result.approved_count == 1
assert result.rejected_count == 0
def test_spoken_form_validation_can_rescue_homophone_fix():
correction = CorrectionCandidate(
id=1,
original_text="dam",
corrected_text="damn",
confidence=0.95,
)
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."}
]
"""
)
candidates, _ = select_grammar_validation_candidates(
transcript,
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
meaning_result = filter_with_meaning_preserving_validations(
candidates,
GrammarValidationSet(
validations=[
GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=False,
confidence=0.99,
reason="Written meaning changes from a barrier to a curse word.",
)
]
),
confidence_threshold=0.8,
)
spoken_form_result = filter_with_spoken_form_validations(
meaning_result.rescue_candidates,
GrammarSpokenFormValidationSet(
validations=[
GrammarSpokenFormValidationDecision(
correction_index=0,
is_likely_spoken_form_correction=True,
confidence=0.95,
reason="The surrounding phrase strongly supports the intended spoken phrase with a curse word.",
)
]
),
confidence_threshold=0.8,
)
kept = keep_corrections_with_indexes([correction], spoken_form_result.approved_correction_indexes)
assert meaning_result.approved_correction_indexes == []
assert kept == [correction]
assert spoken_form_result.skipped == []
def test_spoken_form_validation_rejects_non_homophone_semantic_change():
correction = CorrectionCandidate(
id=1,
original_text="visible",
@@ -132,30 +237,90 @@ def test_validation_rejects_duplicate_and_unknown_decisions():
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
decision = GrammarValidationDecision(
meaning_result = filter_with_meaning_preserving_validations(
candidates,
GrammarValidationSet(
validations=[
GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=False,
confidence=0.99,
reason="This reverses visible to invisible.",
)
]
),
confidence_threshold=0.8,
)
spoken_form_result = filter_with_spoken_form_validations(
meaning_result.rescue_candidates,
GrammarSpokenFormValidationSet(
validations=[
GrammarSpokenFormValidationDecision(
correction_index=0,
is_likely_spoken_form_correction=False,
confidence=0.99,
reason="This is a semantic reversal, not a likely spoken-form transcription error.",
)
]
),
confidence_threshold=0.8,
)
assert spoken_form_result.approved_correction_indexes == []
assert spoken_form_result.rejected_count == 1
assert spoken_form_result.skipped[0].reason == "grammar validation rejected semantic change"
assert spoken_form_result.skipped[0].validation_reason == (
"This is a semantic reversal, not a likely spoken-form transcription error."
)
def test_validation_rejects_duplicate_unknown_and_missing_decisions():
correction = CorrectionCandidate(
id=1,
original_text="visible",
corrected_text="invisible",
confidence=0.95,
)
candidates, _ = select_grammar_validation_candidates(
_transcript(),
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
meaning_decision = GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=True,
confidence=0.95,
reason="Preserves meaning.",
)
spoken_form_decision = GrammarSpokenFormValidationDecision(
correction_index=0,
is_likely_spoken_form_correction=True,
confidence=0.95,
reason="Likely spoken-form correction.",
)
with pytest.raises(AuditaLLMError):
filter_with_grammar_validations(
[correction],
filter_with_meaning_preserving_validations(
candidates,
GrammarValidationSet(validations=[decision, decision]),
GrammarValidationSet(validations=[meaning_decision, meaning_decision]),
confidence_threshold=0.8,
)
with pytest.raises(AuditaLLMError):
filter_with_grammar_validations(
[correction],
filter_with_meaning_preserving_validations(
candidates,
GrammarValidationSet(
GrammarValidationSet(validations=[]),
confidence_threshold=0.8,
)
with pytest.raises(AuditaLLMError):
filter_with_spoken_form_validations(
candidates,
GrammarSpokenFormValidationSet(
validations=[
GrammarValidationDecision(
GrammarSpokenFormValidationDecision(
correction_index=99,
is_meaning_preserving=True,
is_likely_spoken_form_correction=True,
confidence=0.95,
reason="Unknown.",
)
@@ -163,3 +328,9 @@ def test_validation_rejects_duplicate_and_unknown_decisions():
),
confidence_threshold=0.8,
)
with pytest.raises(AuditaLLMError):
filter_with_spoken_form_validations(
candidates,
GrammarSpokenFormValidationSet(validations=[spoken_form_decision, spoken_form_decision]),
confidence_threshold=0.8,
)