eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Replaced the deterministic GrammarOnlyValidator with an LLM-backed grammar_only_guard

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-28 22:29:28 -05:00
parent bbbef37d9a
commit e71e5b8faf
6 changed files with 314 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
tests/test_llm_validators.py
View File

@@ -16,6 +16,7 @@ from audita.validators import (
from audita.validators.base import ValidationContext
from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
from audita.validators.prompts import (
build_grammar_only_messages,
build_meaning_reversal_messages,
build_spoken_form_plausibility_messages,
build_spoken_word_messages,
@@ -346,7 +347,7 @@ def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)]
def test_grammar_only_validator_allows_formatting_only_changes(tmp_path):
def test_grammar_only_validator_approves_conservative_grammar_cleanup(tmp_path):
transcript = parse_transcript_json(
"""
[
@@ -385,9 +386,35 @@ def test_grammar_only_validator_allows_formatting_only_changes(tmp_path):
confidence=0.95,
),
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.98,
"reason": "Conservative punctuation and capitalization cleanup.",
},
{
"correction_index": 1,
"approved": True,
"confidence": 0.95,
"reason": "Conservative apostrophe insertion within grammar cleanup.",
},
{
"correction_index": 2,
"approved": True,
"confidence": 0.97,
"reason": "Conservative spacing cleanup.",
},
]
}
]
)
result = GrammarOnlyValidator("grammar_only_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
@@ -395,6 +422,10 @@ def test_grammar_only_validator_allows_formatting_only_changes(tmp_path):
(1, True),
(2, True),
]
prompt_text = client.calls[0]["messages"][1]["content"]
assert "whole-word article cleanup" in prompt_text
assert "embedded within an otherwise grammatical revision" in prompt_text
assert "filler cleanup" in prompt_text
def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path):
@@ -426,9 +457,29 @@ def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path):
confidence=0.95,
),
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.96,
"reason": "Allowed whole-word article cleanup.",
},
{
"correction_index": 1,
"approved": True,
"confidence": 0.93,
"reason": "Allowed whole-word article cleanup.",
},
]
}
]
)
result = GrammarOnlyValidator("grammar_only_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
@@ -437,7 +488,7 @@ def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path):
]
def test_grammar_only_validator_rejects_word_level_changes(tmp_path):
def test_grammar_only_validator_rejects_out_of_scope_rewrites(tmp_path):
transcript = parse_transcript_json(
"""
[
@@ -496,16 +547,91 @@ def test_grammar_only_validator_rejects_word_level_changes(tmp_path):
confidence=0.95,
),
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": False,
"confidence": 0.99,
"reason": "Unrelated word substitution rather than conservative grammar cleanup.",
},
{
"correction_index": 1,
"approved": False,
"confidence": 0.99,
"reason": "Free-standing homophone rewrite rather than conservative grammar cleanup.",
},
{
"correction_index": 2,
"approved": False,
"confidence": 1.0,
"reason": "Spoken-word filler cleanup is out of scope for the grammar module.",
},
{
"correction_index": 3,
"approved": False,
"confidence": 1.0,
"reason": "Spoken-word repetition cleanup is out of scope for the grammar module.",
},
{
"correction_index": 4,
"approved": False,
"confidence": 0.98,
"reason": "Possessive rewrite goes beyond conservative grammar cleanup.",
},
]
}
]
)
result = GrammarOnlyValidator("grammar_only_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [decision.approved for decision in result.decisions] == [False, False, False, False, False]
assert all(
decision.reason == "correction is not limited to punctuation, capitalization, and spacing"
for decision in result.decisions
def test_grammar_only_validator_allows_embedded_homophone_fix_within_grammar_revision(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="grammar",
module_key="grammar",
id=1,
original_text="question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't",
corrected_text="question: If he dies, does he stay there? The way that, yeah, the way that it's written, it's like, so if it stops, then he goes down; but what if it doesn't?",
confidence=0.95,
)
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.94,
"reason": "Primarily a grammatical revision with an embedded likely mistranscription recovery.",
}
]
}
]
)
result = GrammarOnlyValidator("grammar_only_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)]
def test_non_empty_segment_validator_rejects_empty_and_whitespace_only_segments(tmp_path):
@@ -962,6 +1088,29 @@ def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path):
assert all("corrected_segment_text" in payload for payload in chunk_calls[0]["payloads"])
def test_grammar_validation_prompt_is_scoped_to_conservative_cleanup():
messages = build_grammar_only_messages(
[
{
"correction_index": 0,
"id": 1,
"original_text": "dam",
"corrected_text": "damn",
"confidence": 0.95,
"original_segment_text": "ChatGPT still can't do that with a dam.",
"corrected_segment_text": "ChatGPT still can't do that with a damn.",
}
]
)
combined = messages[0]["content"] + messages[1]["content"]
assert "whole-word article cleanup" in combined
assert "embedded within an otherwise grammatical revision" in combined
assert "free-standing homophone" in combined
assert "spoken-word dysfluency cleanup" in combined
assert "original_segment_text" in messages[1]["content"]
def test_llm_validators_process_batches_concurrently_and_preserve_proposal_order(monkeypatch, tmp_path):
transcript = parse_transcript_json(
"""

121
tests/test_module_proposals.py
View File

@@ -603,6 +603,16 @@ def test_process_transcript_result_runs_grammar_module_with_full_validator_chain
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.98,
"reason": "Conservative grammar cleanup.",
}
]
},
{
"validations": [
{
@@ -627,6 +637,7 @@ def test_process_transcript_result_runs_grammar_module_with_full_validator_chain
assert result.transcript[0].text == "Hello world."
assert [call["stage_name"] for call in client.calls] == [
"grammar:proposal",
"grammar:grammar_only_guard",
"grammar:meaning_reversal_review",
]
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
@@ -678,6 +689,16 @@ def test_process_transcript_result_grammar_module_applies_indefinite_article_cle
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.97,
"reason": "Allowed article cleanup.",
}
]
},
{
"validations": [
{
@@ -702,6 +723,7 @@ def test_process_transcript_result_grammar_module_applies_indefinite_article_cle
assert result.transcript[0].text == "Give me an intelligence saving throw."
assert [call["stage_name"] for call in client.calls] == [
"grammar:proposal",
"grammar:grammar_only_guard",
"grammar:meaning_reversal_review",
]
@@ -798,6 +820,16 @@ def test_process_transcript_result_grammar_module_still_rejects_homophone_style_
"confidence": 0.95,
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": False,
"confidence": 0.99,
"reason": "Free-standing homophone rewrite rather than conservative grammar cleanup.",
}
]
}
]
)
@@ -811,9 +843,94 @@ def test_process_transcript_result_grammar_module_still_rejects_homophone_style_
)
assert result.transcript[0].text == "ChatGPT still can't really do that with a dam."
assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"]
assert [call["stage_name"] for call in client.calls] == [
"grammar:proposal",
"grammar:grammar_only_guard",
]
assert result.report.skipped_corrections[0].source == "validator:grammar_only_guard"
assert result.report.skipped_corrections[0].reason == "correction is not limited to punctuation, capitalization, and spacing"
assert "grammar cleanup" in result.report.skipped_corrections[0].reason
def test_process_transcript_result_grammar_module_allows_embedded_homophone_fix_with_grammar_cleanup(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't"}
]
"""
)
base_config = AuditaConfig.from_sources(env={})
config = AuditaConfig(
api_key=base_config.api_key,
llm_concurrency=base_config.llm_concurrency,
module_keys=base_config.module_keys,
model=base_config.model,
base_url=base_config.base_url,
max_retries=base_config.max_retries,
max_section_tokens=base_config.max_section_tokens,
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
grammar_confidence_threshold=base_config.grammar_confidence_threshold,
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold,
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
normalize_max_segment_tokens=base_config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="always",
)
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't",
"corrected_text": "question: If he dies, does he stay there? The way that, yeah, the way that it's written, it's like, so if it stops, then he goes down; but what if it doesn't?",
"confidence": 0.95,
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.95,
"reason": "Primarily a grammatical revision with an embedded likely mistranscription recovery.",
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.99,
"reason": "Does not reverse the segment meaning.",
}
]
},
]
)
result = process_transcript_result(
transcript,
_glossary(),
config,
module_keys=["grammar"],
llm_client=client,
)
assert result.transcript[0].text == (
"question: If he dies, does he stay there? The way that, yeah, the way that it's written, "
"it's like, so if it stops, then he goes down; but what if it doesn't?"
)
assert [call["stage_name"] for call in client.calls] == [
"grammar:proposal",
"grammar:grammar_only_guard",
"grammar:meaning_reversal_review",
]
def test_process_transcript_result_rejects_spoken_word_whole_segment_deletion_before_llm_validators(tmp_path):