Replaced the deterministic GrammarOnlyValidator with an LLM-backed grammar_only_guard

src/audita/validators/__init__.py

@@ -1,14 +1,13 @@
 from .base import ValidationContext, ValidationDecision, ValidationResult, Validator
 from .deterministic import (
     GlossaryStageProtectedGlossaryTermsValidator,
-    GrammarOnlyValidator,
     IdenticalTextValidator,
     NonEmptySegmentValidator,
     OriginalTextPresentValidator,
     ProposalConfidenceValidator,
     ProtectedGlossaryTermsValidator,
 )
-from .llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
+from .llm import GrammarOnlyValidator, MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
 from .protection import ProtectedVocabulary
 
 __all__ = [

src/audita/validators/deterministic.py

@@ -1,4 +1,3 @@
-import string
 from dataclasses import dataclass
 
 from audita.framework.proposals import ProposalPreviewError, preview_proposal
@@ -155,60 +154,3 @@ class NonEmptySegmentValidator:
             execution_kind=self.execution_kind,
             decisions=decisions,
         )
-
-
-_GRAMMAR_PUNCTUATION = set(string.punctuation) | {"—", "–", "…", "“", "”", "‘", "’"}
-
-
-@dataclass(frozen=True)
-class GrammarOnlyValidator:
-    name: str
-    execution_kind: str = "deterministic"
-
-    def validate(self, context: ValidationContext) -> ValidationResult:
-        return ValidationResult(
-            validator_name=self.name,
-            execution_kind=self.execution_kind,
-            decisions=[
-                _grammar_validation_decision(proposal.proposal_index, proposal.original_text, proposal.corrected_text)
-                for proposal in context.proposals
-            ],
-        )
-
-
-def _grammar_validation_decision(proposal_index: int, original_text: str, corrected_text: str) -> ValidationDecision:
-    approved = _grammar_semantic_key(original_text) == _grammar_semantic_key(corrected_text) or (
-        _grammar_article_semantic_key(original_text) == _grammar_article_semantic_key(corrected_text)
-    )
-    return ValidationDecision(
-        proposal_index=proposal_index,
-        approved=approved,
-        reason=None if approved else "correction is not limited to punctuation, capitalization, and spacing",
-    )
-
-
-def _grammar_semantic_key(text: str) -> str:
-    return "".join(
-        character.casefold()
-        for character in text
-        if not character.isspace() and character not in _GRAMMAR_PUNCTUATION
-    )
-
-
-def _grammar_article_semantic_key(text: str) -> tuple[str, ...]:
-    return tuple("__article__" if token in {"a", "an"} else token for token in _grammar_word_tokens(text))
-
-
-def _grammar_word_tokens(text: str) -> tuple[str, ...]:
-    tokens: list[str] = []
-    current: list[str] = []
-    for character in text.casefold():
-        if character.isalnum():
-            current.append(character)
-            continue
-        if current:
-            tokens.append("".join(current))
-            current = []
-    if current:
-        tokens.append("".join(current))
-    return tuple(tokens)

src/audita/validators/llm.py

@@ -13,7 +13,12 @@ from audita.core.errors import AuditaLLMError
 from audita.framework.proposals import ProposalPreview, ProposalPreviewError, preview_proposal
 
 from .base import ValidationContext, ValidationDecision, ValidationResult
-from .prompts import build_meaning_reversal_messages, build_spoken_form_plausibility_messages, build_spoken_word_messages
+from .prompts import (
+    build_grammar_only_messages,
+    build_meaning_reversal_messages,
+    build_spoken_form_plausibility_messages,
+    build_spoken_word_messages,
+)
 
 
 class _LLMValidationDecisionModel(BaseModel):
@@ -164,5 +169,11 @@ class SpokenWordValidator(_BaseLLMValidator):
     prompt_builder: PromptBuilder = build_spoken_word_messages
 
 
+@dataclass(frozen=True)
+class GrammarOnlyValidator(_BaseLLMValidator):
+    name: str = "grammar_only_guard"
+    prompt_builder: PromptBuilder = build_grammar_only_messages
+
+
 def _write_json(path: Path, payload: dict) -> None:
     path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")

src/audita/validators/prompts.py

@@ -81,3 +81,28 @@ def build_spoken_word_messages(validation_payload: List[dict]) -> List[Message]:
         f"Corrections to validate:\n{payload_json}"
     )
     return [{"role": "system", "content": system}, {"role": "user", "content": user}]
+
+
+def build_grammar_only_messages(validation_payload: List[dict]) -> List[Message]:
+    payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2)
+    system = (
+        "You are Audita, a conservative grammar-scope validation assistant. "
+        "Evaluate whether each proposed correction is acceptable for the grammar module. "
+        "Approve only low-risk transcript cleanup that is primarily grammatical in nature and preserves substantive meaning."
+    )
+    user = (
+        "Review these proposed transcript corrections and decide whether each one is acceptable for the grammar module.\n\n"
+        "Rules:\n"
+        "- Return one validation decision for every correction_index in the input.\n"
+        "- Approve conservative cleanup that primarily performs punctuation, capitalization, spacing, or whole-word article cleanup such as \"a\" <-> \"an\".\n"
+        "- You may also approve a likely homophone or mistranscription fix when it is embedded within an otherwise grammatical revision and the overall correction is still a low-risk transcript cleanup.\n"
+        "- Approve embedded recovery such as formatting cleanup plus a likely transcription fix when the full corrected segment remains conservative and contextually well supported.\n"
+        "- Reject free-standing homophone or mistranscription rewrites when they are not part of an otherwise grammatical cleanup.\n"
+        "- Reject filler cleanup, repetition cleanup, spoken-word dysfluency cleanup, stylistic polishing, broad paraphrase, and unrelated content substitutions.\n"
+        "- Reject corrections that go beyond conservative grammar-stage cleanup, even if some punctuation or capitalization cleanup is also present.\n"
+        "- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n"
+        "- Each returned validation must contain only correction_index, approved, confidence, and reason.\n"
+        "- confidence must be between 0.0 and 1.0.\n\n"
+        f"Corrections to validate:\n{payload_json}"
+    )
+    return [{"role": "system", "content": system}, {"role": "user", "content": user}]

tests/test_llm_validators.py

@@ -16,6 +16,7 @@ from audita.validators import (
 from audita.validators.base import ValidationContext
 from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
 from audita.validators.prompts import (
+    build_grammar_only_messages,
     build_meaning_reversal_messages,
     build_spoken_form_plausibility_messages,
     build_spoken_word_messages,
@@ -346,7 +347,7 @@ def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp
     assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)]
 
 
-def test_grammar_only_validator_allows_formatting_only_changes(tmp_path):
+def test_grammar_only_validator_approves_conservative_grammar_cleanup(tmp_path):
     transcript = parse_transcript_json(
         """
         [
@@ -385,9 +386,35 @@ def test_grammar_only_validator_allows_formatting_only_changes(tmp_path):
             confidence=0.95,
         ),
     ]
+    client = FakeStructuredLLMClient(
+        [
+            {
+                "validations": [
+                    {
+                        "correction_index": 0,
+                        "approved": True,
+                        "confidence": 0.98,
+                        "reason": "Conservative punctuation and capitalization cleanup.",
+                    },
+                    {
+                        "correction_index": 1,
+                        "approved": True,
+                        "confidence": 0.95,
+                        "reason": "Conservative apostrophe insertion within grammar cleanup.",
+                    },
+                    {
+                        "correction_index": 2,
+                        "approved": True,
+                        "confidence": 0.97,
+                        "reason": "Conservative spacing cleanup.",
+                    },
+                ]
+            }
+        ]
+    )
 
     result = GrammarOnlyValidator("grammar_only_guard").validate(
-        _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
+        _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
     )
 
     assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
@@ -395,6 +422,10 @@ def test_grammar_only_validator_allows_formatting_only_changes(tmp_path):
         (1, True),
         (2, True),
     ]
+    prompt_text = client.calls[0]["messages"][1]["content"]
+    assert "whole-word article cleanup" in prompt_text
+    assert "embedded within an otherwise grammatical revision" in prompt_text
+    assert "filler cleanup" in prompt_text
 
 
 def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path):
@@ -426,9 +457,29 @@ def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path):
             confidence=0.95,
         ),
     ]
+    client = FakeStructuredLLMClient(
+        [
+            {
+                "validations": [
+                    {
+                        "correction_index": 0,
+                        "approved": True,
+                        "confidence": 0.96,
+                        "reason": "Allowed whole-word article cleanup.",
+                    },
+                    {
+                        "correction_index": 1,
+                        "approved": True,
+                        "confidence": 0.93,
+                        "reason": "Allowed whole-word article cleanup.",
+                    },
+                ]
+            }
+        ]
+    )
 
     result = GrammarOnlyValidator("grammar_only_guard").validate(
-        _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
+        _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
     )
 
     assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
@@ -437,7 +488,7 @@ def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path):
     ]
 
 
-def test_grammar_only_validator_rejects_word_level_changes(tmp_path):
+def test_grammar_only_validator_rejects_out_of_scope_rewrites(tmp_path):
     transcript = parse_transcript_json(
         """
         [
@@ -496,16 +547,91 @@ def test_grammar_only_validator_rejects_word_level_changes(tmp_path):
             confidence=0.95,
         ),
     ]
+    client = FakeStructuredLLMClient(
+        [
+            {
+                "validations": [
+                    {
+                        "correction_index": 0,
+                        "approved": False,
+                        "confidence": 0.99,
+                        "reason": "Unrelated word substitution rather than conservative grammar cleanup.",
+                    },
+                    {
+                        "correction_index": 1,
+                        "approved": False,
+                        "confidence": 0.99,
+                        "reason": "Free-standing homophone rewrite rather than conservative grammar cleanup.",
+                    },
+                    {
+                        "correction_index": 2,
+                        "approved": False,
+                        "confidence": 1.0,
+                        "reason": "Spoken-word filler cleanup is out of scope for the grammar module.",
+                    },
+                    {
+                        "correction_index": 3,
+                        "approved": False,
+                        "confidence": 1.0,
+                        "reason": "Spoken-word repetition cleanup is out of scope for the grammar module.",
+                    },
+                    {
+                        "correction_index": 4,
+                        "approved": False,
+                        "confidence": 0.98,
+                        "reason": "Possessive rewrite goes beyond conservative grammar cleanup.",
+                    },
+                ]
+            }
+        ]
+    )
 
     result = GrammarOnlyValidator("grammar_only_guard").validate(
-        _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
+        _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
     )
 
     assert [decision.approved for decision in result.decisions] == [False, False, False, False, False]
-    assert all(
+
-        decision.reason == "correction is not limited to punctuation, capitalization, and spacing"
+
-        for decision in result.decisions
+def test_grammar_only_validator_allows_embedded_homophone_fix_within_grammar_revision(tmp_path):
+    transcript = parse_transcript_json(
+        """
+        [
+          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't"}
+        ]
+        """
     )
+    proposals = [
+        CorrectionProposal(
+            proposal_index=0,
+            module_instance="grammar",
+            module_key="grammar",
+            id=1,
+            original_text="question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't",
+            corrected_text="question: If he dies, does he stay there? The way that, yeah, the way that it's written, it's like, so if it stops, then he goes down; but what if it doesn't?",
+            confidence=0.95,
+        )
+    ]
+    client = FakeStructuredLLMClient(
+        [
+            {
+                "validations": [
+                    {
+                        "correction_index": 0,
+                        "approved": True,
+                        "confidence": 0.94,
+                        "reason": "Primarily a grammatical revision with an embedded likely mistranscription recovery.",
+                    }
+                ]
+            }
+        ]
+    )
+
+    result = GrammarOnlyValidator("grammar_only_guard").validate(
+        _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
+    )
+
+    assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)]
 
 
 def test_non_empty_segment_validator_rejects_empty_and_whitespace_only_segments(tmp_path):
@@ -962,6 +1088,29 @@ def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path):
     assert all("corrected_segment_text" in payload for payload in chunk_calls[0]["payloads"])
 
 
+def test_grammar_validation_prompt_is_scoped_to_conservative_cleanup():
+    messages = build_grammar_only_messages(
+        [
+            {
+                "correction_index": 0,
+                "id": 1,
+                "original_text": "dam",
+                "corrected_text": "damn",
+                "confidence": 0.95,
+                "original_segment_text": "ChatGPT still can't do that with a dam.",
+                "corrected_segment_text": "ChatGPT still can't do that with a damn.",
+            }
+        ]
+    )
+
+    combined = messages[0]["content"] + messages[1]["content"]
+    assert "whole-word article cleanup" in combined
+    assert "embedded within an otherwise grammatical revision" in combined
+    assert "free-standing homophone" in combined
+    assert "spoken-word dysfluency cleanup" in combined
+    assert "original_segment_text" in messages[1]["content"]
+
+
 def test_llm_validators_process_batches_concurrently_and_preserve_proposal_order(monkeypatch, tmp_path):
     transcript = parse_transcript_json(
         """

tests/test_module_proposals.py

@@ -603,6 +603,16 @@ def test_process_transcript_result_runs_grammar_module_with_full_validator_chain
                     }
                 ]
             },
+            {
+                "validations": [
+                    {
+                        "correction_index": 0,
+                        "approved": True,
+                        "confidence": 0.98,
+                        "reason": "Conservative grammar cleanup.",
+                    }
+                ]
+            },
             {
                 "validations": [
                     {
@@ -627,6 +637,7 @@ def test_process_transcript_result_runs_grammar_module_with_full_validator_chain
     assert result.transcript[0].text == "Hello world."
     assert [call["stage_name"] for call in client.calls] == [
         "grammar:proposal",
+        "grammar:grammar_only_guard",
         "grammar:meaning_reversal_review",
     ]
     assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
@@ -678,6 +689,16 @@ def test_process_transcript_result_grammar_module_applies_indefinite_article_cle
                     }
                 ]
             },
+            {
+                "validations": [
+                    {
+                        "correction_index": 0,
+                        "approved": True,
+                        "confidence": 0.97,
+                        "reason": "Allowed article cleanup.",
+                    }
+                ]
+            },
             {
                 "validations": [
                     {
@@ -702,6 +723,7 @@ def test_process_transcript_result_grammar_module_applies_indefinite_article_cle
     assert result.transcript[0].text == "Give me an intelligence saving throw."
     assert [call["stage_name"] for call in client.calls] == [
         "grammar:proposal",
+        "grammar:grammar_only_guard",
         "grammar:meaning_reversal_review",
     ]
 
@@ -798,6 +820,16 @@ def test_process_transcript_result_grammar_module_still_rejects_homophone_style_
                         "confidence": 0.95,
                     }
                 ]
+            },
+            {
+                "validations": [
+                    {
+                        "correction_index": 0,
+                        "approved": False,
+                        "confidence": 0.99,
+                        "reason": "Free-standing homophone rewrite rather than conservative grammar cleanup.",
+                    }
+                ]
             }
         ]
     )
@@ -811,9 +843,94 @@ def test_process_transcript_result_grammar_module_still_rejects_homophone_style_
     )
 
     assert result.transcript[0].text == "ChatGPT still can't really do that with a dam."
-    assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"]
+    assert [call["stage_name"] for call in client.calls] == [
+        "grammar:proposal",
+        "grammar:grammar_only_guard",
+    ]
     assert result.report.skipped_corrections[0].source == "validator:grammar_only_guard"
-    assert result.report.skipped_corrections[0].reason == "correction is not limited to punctuation, capitalization, and spacing"
+    assert "grammar cleanup" in result.report.skipped_corrections[0].reason
+
+
+def test_process_transcript_result_grammar_module_allows_embedded_homophone_fix_with_grammar_cleanup(tmp_path):
+    transcript = parse_source_transcript_json(
+        """
+        [
+          {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't"}
+        ]
+        """
+    )
+    base_config = AuditaConfig.from_sources(env={})
+    config = AuditaConfig(
+        api_key=base_config.api_key,
+        llm_concurrency=base_config.llm_concurrency,
+        module_keys=base_config.module_keys,
+        model=base_config.model,
+        base_url=base_config.base_url,
+        max_retries=base_config.max_retries,
+        max_section_tokens=base_config.max_section_tokens,
+        glossary_confidence_threshold=base_config.glossary_confidence_threshold,
+        grammar_confidence_threshold=base_config.grammar_confidence_threshold,
+        homophones_confidence_threshold=base_config.homophones_confidence_threshold,
+        spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold,
+        normalize_max_segment_gap=base_config.normalize_max_segment_gap,
+        normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
+        normalize_max_segment_duration=base_config.normalize_max_segment_duration,
+        normalize_max_segment_tokens=base_config.normalize_max_segment_tokens,
+        work_dir=tmp_path / "work",
+        work_dir_retention="always",
+    )
+    client = FakeStructuredLLMClient(
+        [
+            {
+                "corrections": [
+                    {
+                        "id": 1,
+                        "original_text": "question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't",
+                        "corrected_text": "question: If he dies, does he stay there? The way that, yeah, the way that it's written, it's like, so if it stops, then he goes down; but what if it doesn't?",
+                        "confidence": 0.95,
+                    }
+                ]
+            },
+            {
+                "validations": [
+                    {
+                        "correction_index": 0,
+                        "approved": True,
+                        "confidence": 0.95,
+                        "reason": "Primarily a grammatical revision with an embedded likely mistranscription recovery.",
+                    }
+                ]
+            },
+            {
+                "validations": [
+                    {
+                        "correction_index": 0,
+                        "approved": True,
+                        "confidence": 0.99,
+                        "reason": "Does not reverse the segment meaning.",
+                    }
+                ]
+            },
+        ]
+    )
+
+    result = process_transcript_result(
+        transcript,
+        _glossary(),
+        config,
+        module_keys=["grammar"],
+        llm_client=client,
+    )
+
+    assert result.transcript[0].text == (
+        "question: If he dies, does he stay there? The way that, yeah, the way that it's written, "
+        "it's like, so if it stops, then he goes down; but what if it doesn't?"
+    )
+    assert [call["stage_name"] for call in client.calls] == [
+        "grammar:proposal",
+        "grammar:grammar_only_guard",
+        "grammar:meaning_reversal_review",
+    ]
 
 
 def test_process_transcript_result_rejects_spoken_word_whole_segment_deletion_before_llm_validators(tmp_path):