From e71e5b8faf5139693fc2379ff227173c9969dea1 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 28 Apr 2026 22:29:28 -0500 Subject: [PATCH] Replaced the deterministic GrammarOnlyValidator with an LLM-backed grammar_only_guard --- src/audita/validators/__init__.py | 3 +- src/audita/validators/deterministic.py | 58 --------- src/audita/validators/llm.py | 13 +- src/audita/validators/prompts.py | 25 ++++ tests/test_llm_validators.py | 165 +++++++++++++++++++++++-- tests/test_module_proposals.py | 121 +++++++++++++++++- 6 files changed, 314 insertions(+), 71 deletions(-) diff --git a/src/audita/validators/__init__.py b/src/audita/validators/__init__.py index f094e53..4840576 100644 --- a/src/audita/validators/__init__.py +++ b/src/audita/validators/__init__.py @@ -1,14 +1,13 @@ from .base import ValidationContext, ValidationDecision, ValidationResult, Validator from .deterministic import ( GlossaryStageProtectedGlossaryTermsValidator, - GrammarOnlyValidator, IdenticalTextValidator, NonEmptySegmentValidator, OriginalTextPresentValidator, ProposalConfidenceValidator, ProtectedGlossaryTermsValidator, ) -from .llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator +from .llm import GrammarOnlyValidator, MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator from .protection import ProtectedVocabulary __all__ = [ diff --git a/src/audita/validators/deterministic.py b/src/audita/validators/deterministic.py index 227b7f0..3b900f8 100644 --- a/src/audita/validators/deterministic.py +++ b/src/audita/validators/deterministic.py @@ -1,4 +1,3 @@ -import string from dataclasses import dataclass from audita.framework.proposals import ProposalPreviewError, preview_proposal @@ -155,60 +154,3 @@ class NonEmptySegmentValidator: execution_kind=self.execution_kind, decisions=decisions, ) - - -_GRAMMAR_PUNCTUATION = set(string.punctuation) | {"—", "–", "…", "“", "”", "‘", "’"} - - -@dataclass(frozen=True) -class GrammarOnlyValidator: - name: str - execution_kind: str = "deterministic" - - def validate(self, context: ValidationContext) -> ValidationResult: - return ValidationResult( - validator_name=self.name, - execution_kind=self.execution_kind, - decisions=[ - _grammar_validation_decision(proposal.proposal_index, proposal.original_text, proposal.corrected_text) - for proposal in context.proposals - ], - ) - - -def _grammar_validation_decision(proposal_index: int, original_text: str, corrected_text: str) -> ValidationDecision: - approved = _grammar_semantic_key(original_text) == _grammar_semantic_key(corrected_text) or ( - _grammar_article_semantic_key(original_text) == _grammar_article_semantic_key(corrected_text) - ) - return ValidationDecision( - proposal_index=proposal_index, - approved=approved, - reason=None if approved else "correction is not limited to punctuation, capitalization, and spacing", - ) - - -def _grammar_semantic_key(text: str) -> str: - return "".join( - character.casefold() - for character in text - if not character.isspace() and character not in _GRAMMAR_PUNCTUATION - ) - - -def _grammar_article_semantic_key(text: str) -> tuple[str, ...]: - return tuple("__article__" if token in {"a", "an"} else token for token in _grammar_word_tokens(text)) - - -def _grammar_word_tokens(text: str) -> tuple[str, ...]: - tokens: list[str] = [] - current: list[str] = [] - for character in text.casefold(): - if character.isalnum(): - current.append(character) - continue - if current: - tokens.append("".join(current)) - current = [] - if current: - tokens.append("".join(current)) - return tuple(tokens) diff --git a/src/audita/validators/llm.py b/src/audita/validators/llm.py index 3c96390..283d433 100644 --- a/src/audita/validators/llm.py +++ b/src/audita/validators/llm.py @@ -13,7 +13,12 @@ from audita.core.errors import AuditaLLMError from audita.framework.proposals import ProposalPreview, ProposalPreviewError, preview_proposal from .base import ValidationContext, ValidationDecision, ValidationResult -from .prompts import build_meaning_reversal_messages, build_spoken_form_plausibility_messages, build_spoken_word_messages +from .prompts import ( + build_grammar_only_messages, + build_meaning_reversal_messages, + build_spoken_form_plausibility_messages, + build_spoken_word_messages, +) class _LLMValidationDecisionModel(BaseModel): @@ -164,5 +169,11 @@ class SpokenWordValidator(_BaseLLMValidator): prompt_builder: PromptBuilder = build_spoken_word_messages +@dataclass(frozen=True) +class GrammarOnlyValidator(_BaseLLMValidator): + name: str = "grammar_only_guard" + prompt_builder: PromptBuilder = build_grammar_only_messages + + def _write_json(path: Path, payload: dict) -> None: path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") diff --git a/src/audita/validators/prompts.py b/src/audita/validators/prompts.py index 13c5386..1e22281 100644 --- a/src/audita/validators/prompts.py +++ b/src/audita/validators/prompts.py @@ -81,3 +81,28 @@ def build_spoken_word_messages(validation_payload: List[dict]) -> List[Message]: f"Corrections to validate:\n{payload_json}" ) return [{"role": "system", "content": system}, {"role": "user", "content": user}] + + +def build_grammar_only_messages(validation_payload: List[dict]) -> List[Message]: + payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2) + system = ( + "You are Audita, a conservative grammar-scope validation assistant. " + "Evaluate whether each proposed correction is acceptable for the grammar module. " + "Approve only low-risk transcript cleanup that is primarily grammatical in nature and preserves substantive meaning." + ) + user = ( + "Review these proposed transcript corrections and decide whether each one is acceptable for the grammar module.\n\n" + "Rules:\n" + "- Return one validation decision for every correction_index in the input.\n" + "- Approve conservative cleanup that primarily performs punctuation, capitalization, spacing, or whole-word article cleanup such as \"a\" <-> \"an\".\n" + "- You may also approve a likely homophone or mistranscription fix when it is embedded within an otherwise grammatical revision and the overall correction is still a low-risk transcript cleanup.\n" + "- Approve embedded recovery such as formatting cleanup plus a likely transcription fix when the full corrected segment remains conservative and contextually well supported.\n" + "- Reject free-standing homophone or mistranscription rewrites when they are not part of an otherwise grammatical cleanup.\n" + "- Reject filler cleanup, repetition cleanup, spoken-word dysfluency cleanup, stylistic polishing, broad paraphrase, and unrelated content substitutions.\n" + "- Reject corrections that go beyond conservative grammar-stage cleanup, even if some punctuation or capitalization cleanup is also present.\n" + "- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n" + "- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" + "- confidence must be between 0.0 and 1.0.\n\n" + f"Corrections to validate:\n{payload_json}" + ) + return [{"role": "system", "content": system}, {"role": "user", "content": user}] diff --git a/tests/test_llm_validators.py b/tests/test_llm_validators.py index a4dee2e..730aa6f 100644 --- a/tests/test_llm_validators.py +++ b/tests/test_llm_validators.py @@ -16,6 +16,7 @@ from audita.validators import ( from audita.validators.base import ValidationContext from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator from audita.validators.prompts import ( + build_grammar_only_messages, build_meaning_reversal_messages, build_spoken_form_plausibility_messages, build_spoken_word_messages, @@ -346,7 +347,7 @@ def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)] -def test_grammar_only_validator_allows_formatting_only_changes(tmp_path): +def test_grammar_only_validator_approves_conservative_grammar_cleanup(tmp_path): transcript = parse_transcript_json( """ [ @@ -385,9 +386,35 @@ def test_grammar_only_validator_allows_formatting_only_changes(tmp_path): confidence=0.95, ), ] + client = FakeStructuredLLMClient( + [ + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.98, + "reason": "Conservative punctuation and capitalization cleanup.", + }, + { + "correction_index": 1, + "approved": True, + "confidence": 0.95, + "reason": "Conservative apostrophe insertion within grammar cleanup.", + }, + { + "correction_index": 2, + "approved": True, + "confidence": 0.97, + "reason": "Conservative spacing cleanup.", + }, + ] + } + ] + ) result = GrammarOnlyValidator("grammar_only_guard").validate( - _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path) + _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) ) assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [ @@ -395,6 +422,10 @@ def test_grammar_only_validator_allows_formatting_only_changes(tmp_path): (1, True), (2, True), ] + prompt_text = client.calls[0]["messages"][1]["content"] + assert "whole-word article cleanup" in prompt_text + assert "embedded within an otherwise grammatical revision" in prompt_text + assert "filler cleanup" in prompt_text def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path): @@ -426,9 +457,29 @@ def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path): confidence=0.95, ), ] + client = FakeStructuredLLMClient( + [ + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.96, + "reason": "Allowed whole-word article cleanup.", + }, + { + "correction_index": 1, + "approved": True, + "confidence": 0.93, + "reason": "Allowed whole-word article cleanup.", + }, + ] + } + ] + ) result = GrammarOnlyValidator("grammar_only_guard").validate( - _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path) + _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) ) assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [ @@ -437,7 +488,7 @@ def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path): ] -def test_grammar_only_validator_rejects_word_level_changes(tmp_path): +def test_grammar_only_validator_rejects_out_of_scope_rewrites(tmp_path): transcript = parse_transcript_json( """ [ @@ -496,16 +547,91 @@ def test_grammar_only_validator_rejects_word_level_changes(tmp_path): confidence=0.95, ), ] + client = FakeStructuredLLMClient( + [ + { + "validations": [ + { + "correction_index": 0, + "approved": False, + "confidence": 0.99, + "reason": "Unrelated word substitution rather than conservative grammar cleanup.", + }, + { + "correction_index": 1, + "approved": False, + "confidence": 0.99, + "reason": "Free-standing homophone rewrite rather than conservative grammar cleanup.", + }, + { + "correction_index": 2, + "approved": False, + "confidence": 1.0, + "reason": "Spoken-word filler cleanup is out of scope for the grammar module.", + }, + { + "correction_index": 3, + "approved": False, + "confidence": 1.0, + "reason": "Spoken-word repetition cleanup is out of scope for the grammar module.", + }, + { + "correction_index": 4, + "approved": False, + "confidence": 0.98, + "reason": "Possessive rewrite goes beyond conservative grammar cleanup.", + }, + ] + } + ] + ) result = GrammarOnlyValidator("grammar_only_guard").validate( - _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path) + _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) ) assert [decision.approved for decision in result.decisions] == [False, False, False, False, False] - assert all( - decision.reason == "correction is not limited to punctuation, capitalization, and spacing" - for decision in result.decisions + + +def test_grammar_only_validator_allows_embedded_homophone_fix_within_grammar_revision(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't"} + ] + """ ) + proposals = [ + CorrectionProposal( + proposal_index=0, + module_instance="grammar", + module_key="grammar", + id=1, + original_text="question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't", + corrected_text="question: If he dies, does he stay there? The way that, yeah, the way that it's written, it's like, so if it stops, then he goes down; but what if it doesn't?", + confidence=0.95, + ) + ] + client = FakeStructuredLLMClient( + [ + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.94, + "reason": "Primarily a grammatical revision with an embedded likely mistranscription recovery.", + } + ] + } + ] + ) + + result = GrammarOnlyValidator("grammar_only_guard").validate( + _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) + ) + + assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)] def test_non_empty_segment_validator_rejects_empty_and_whitespace_only_segments(tmp_path): @@ -962,6 +1088,29 @@ def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path): assert all("corrected_segment_text" in payload for payload in chunk_calls[0]["payloads"]) +def test_grammar_validation_prompt_is_scoped_to_conservative_cleanup(): + messages = build_grammar_only_messages( + [ + { + "correction_index": 0, + "id": 1, + "original_text": "dam", + "corrected_text": "damn", + "confidence": 0.95, + "original_segment_text": "ChatGPT still can't do that with a dam.", + "corrected_segment_text": "ChatGPT still can't do that with a damn.", + } + ] + ) + + combined = messages[0]["content"] + messages[1]["content"] + assert "whole-word article cleanup" in combined + assert "embedded within an otherwise grammatical revision" in combined + assert "free-standing homophone" in combined + assert "spoken-word dysfluency cleanup" in combined + assert "original_segment_text" in messages[1]["content"] + + def test_llm_validators_process_batches_concurrently_and_preserve_proposal_order(monkeypatch, tmp_path): transcript = parse_transcript_json( """ diff --git a/tests/test_module_proposals.py b/tests/test_module_proposals.py index 74eb08f..dbea95d 100644 --- a/tests/test_module_proposals.py +++ b/tests/test_module_proposals.py @@ -603,6 +603,16 @@ def test_process_transcript_result_runs_grammar_module_with_full_validator_chain } ] }, + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.98, + "reason": "Conservative grammar cleanup.", + } + ] + }, { "validations": [ { @@ -627,6 +637,7 @@ def test_process_transcript_result_runs_grammar_module_with_full_validator_chain assert result.transcript[0].text == "Hello world." assert [call["stage_name"] for call in client.calls] == [ "grammar:proposal", + "grammar:grammar_only_guard", "grammar:meaning_reversal_review", ] assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ @@ -678,6 +689,16 @@ def test_process_transcript_result_grammar_module_applies_indefinite_article_cle } ] }, + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.97, + "reason": "Allowed article cleanup.", + } + ] + }, { "validations": [ { @@ -702,6 +723,7 @@ def test_process_transcript_result_grammar_module_applies_indefinite_article_cle assert result.transcript[0].text == "Give me an intelligence saving throw." assert [call["stage_name"] for call in client.calls] == [ "grammar:proposal", + "grammar:grammar_only_guard", "grammar:meaning_reversal_review", ] @@ -798,6 +820,16 @@ def test_process_transcript_result_grammar_module_still_rejects_homophone_style_ "confidence": 0.95, } ] + }, + { + "validations": [ + { + "correction_index": 0, + "approved": False, + "confidence": 0.99, + "reason": "Free-standing homophone rewrite rather than conservative grammar cleanup.", + } + ] } ] ) @@ -811,9 +843,94 @@ def test_process_transcript_result_grammar_module_still_rejects_homophone_style_ ) assert result.transcript[0].text == "ChatGPT still can't really do that with a dam." - assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"] + assert [call["stage_name"] for call in client.calls] == [ + "grammar:proposal", + "grammar:grammar_only_guard", + ] assert result.report.skipped_corrections[0].source == "validator:grammar_only_guard" - assert result.report.skipped_corrections[0].reason == "correction is not limited to punctuation, capitalization, and spacing" + assert "grammar cleanup" in result.report.skipped_corrections[0].reason + + +def test_process_transcript_result_grammar_module_allows_embedded_homophone_fix_with_grammar_cleanup(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't"} + ] + """ + ) + base_config = AuditaConfig.from_sources(env={}) + config = AuditaConfig( + api_key=base_config.api_key, + llm_concurrency=base_config.llm_concurrency, + module_keys=base_config.module_keys, + model=base_config.model, + base_url=base_config.base_url, + max_retries=base_config.max_retries, + max_section_tokens=base_config.max_section_tokens, + glossary_confidence_threshold=base_config.glossary_confidence_threshold, + grammar_confidence_threshold=base_config.grammar_confidence_threshold, + homophones_confidence_threshold=base_config.homophones_confidence_threshold, + spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, + normalize_max_segment_gap=base_config.normalize_max_segment_gap, + normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, + normalize_max_segment_duration=base_config.normalize_max_segment_duration, + normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, + work_dir=tmp_path / "work", + work_dir_retention="always", + ) + client = FakeStructuredLLMClient( + [ + { + "corrections": [ + { + "id": 1, + "original_text": "question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't", + "corrected_text": "question: If he dies, does he stay there? The way that, yeah, the way that it's written, it's like, so if it stops, then he goes down; but what if it doesn't?", + "confidence": 0.95, + } + ] + }, + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.95, + "reason": "Primarily a grammatical revision with an embedded likely mistranscription recovery.", + } + ] + }, + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.99, + "reason": "Does not reverse the segment meaning.", + } + ] + }, + ] + ) + + result = process_transcript_result( + transcript, + _glossary(), + config, + module_keys=["grammar"], + llm_client=client, + ) + + assert result.transcript[0].text == ( + "question: If he dies, does he stay there? The way that, yeah, the way that it's written, " + "it's like, so if it stops, then he goes down; but what if it doesn't?" + ) + assert [call["stage_name"] for call in client.calls] == [ + "grammar:proposal", + "grammar:grammar_only_guard", + "grammar:meaning_reversal_review", + ] def test_process_transcript_result_rejects_spoken_word_whole_segment_deletion_before_llm_validators(tmp_path):