From 9ab5a6012371f82b610b30e74b7681a5cbd6a827 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 1 May 2026 07:01:51 -0500 Subject: [PATCH] Implemented a shared, more permissive editorial validatior for the grammar and spoken_word modules --- src/audita/validators/__init__.py | 3 +- src/audita/validators/llm.py | 13 +++++-- src/audita/validators/prompts.py | 50 ++++++++---------------- tests/test_llm_validators.py | 65 +++++++++++++++++++------------ tests/test_module_proposals.py | 13 +++---- 5 files changed, 75 insertions(+), 69 deletions(-) diff --git a/src/audita/validators/__init__.py b/src/audita/validators/__init__.py index 4840576..740a97c 100644 --- a/src/audita/validators/__init__.py +++ b/src/audita/validators/__init__.py @@ -7,7 +7,7 @@ from .deterministic import ( ProposalConfidenceValidator, ProtectedGlossaryTermsValidator, ) -from .llm import GrammarOnlyValidator, MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator +from .llm import EditorialValidator, GrammarOnlyValidator, MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator from .protection import ProtectedVocabulary __all__ = [ @@ -21,6 +21,7 @@ __all__ = [ "ProtectedGlossaryTermsValidator", "GlossaryStageProtectedGlossaryTermsValidator", "NonEmptySegmentValidator", + "EditorialValidator", "GrammarOnlyValidator", "ProtectedVocabulary", "SpokenFormPlausibilityValidator", diff --git a/src/audita/validators/llm.py b/src/audita/validators/llm.py index bc62f08..f351bef 100644 --- a/src/audita/validators/llm.py +++ b/src/audita/validators/llm.py @@ -14,10 +14,9 @@ from audita.framework.proposals import ProposalPreview, ProposalPreviewError, pr from .base import ValidationContext, ValidationDecision, ValidationResult from .prompts import ( - build_grammar_only_messages, + build_editorial_messages, build_meaning_reversal_messages, build_spoken_form_plausibility_messages, - build_spoken_word_messages, ) @@ -174,16 +173,22 @@ class MeaningReversalValidator(_BaseLLMValidator): prompt_builder: PromptBuilder = build_meaning_reversal_messages +@dataclass(frozen=True) +class EditorialValidator(_BaseLLMValidator): + name: str = "editorial_review" + prompt_builder: PromptBuilder = build_editorial_messages + + @dataclass(frozen=True) class SpokenWordValidator(_BaseLLMValidator): name: str = "spoken_word_review" - prompt_builder: PromptBuilder = build_spoken_word_messages + prompt_builder: PromptBuilder = build_editorial_messages @dataclass(frozen=True) class GrammarOnlyValidator(_BaseLLMValidator): name: str = "grammar_only_guard" - prompt_builder: PromptBuilder = build_grammar_only_messages + prompt_builder: PromptBuilder = build_editorial_messages def _write_json(path: Path, payload: dict) -> None: diff --git a/src/audita/validators/prompts.py b/src/audita/validators/prompts.py index 3d077e1..5f753f6 100644 --- a/src/audita/validators/prompts.py +++ b/src/audita/validators/prompts.py @@ -58,25 +58,27 @@ def build_meaning_reversal_messages(validation_payload: List[dict]) -> List[Mess return [{"role": "system", "content": system}, {"role": "user", "content": user}] -def build_spoken_word_messages(validation_payload: List[dict]) -> List[Message]: +def build_editorial_messages(validation_payload: List[dict]) -> List[Message]: payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2) system = ( - "You are Audita, a conservative spoken-word cleanup validation assistant. " - "Evaluate whether each proposed correction is a reasonable cleanup of repeated words, repeated short phrases, " - "filler words, hesitation artifacts, or similar spoken dysfluencies. " - "Approve only low-risk cleanup that preserves the segment's substantive meaning." + "You are Audita, a conservative editorial validation assistant. " + "Evaluate whether each proposed correction is editorial in nature and preserves the segment's underlying meaning. " + "Editorial revisions may include dysfluency cleanup, punctuation changes, capitalization changes, " + "homophone or mistranscription corrections, and similar low-risk editorial cleanup." ) user = ( - "Review these proposed transcript corrections and decide whether each one is a valid spoken-word cleanup.\n\n" + "Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision.\n\n" "Rules:\n" "- Return one validation decision for every correction_index in the input.\n" - "- Approve cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies.\n" + "- Approve editorial revisions that preserve the underlying meaning of the segment.\n" + "- Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies.\n" + "- Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning.\n" + "- Approve low-risk homophone or mistranscription corrections when they are contextually well supported.\n" + "- Approve conservative combinations of these editorial changes when the overall revision remains meaning-preserving.\n" "- Reject repetition cleanup when the repetition plausibly serves urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency.\n" "- Phrases such as \"Help! Help! Help!\", \"Stop! Stop! Stop!\", \"No! No! No!\", \"Yes! Yes! Yes!\", and \"Go! Go! Go!\" are often intentional emphasis and should usually be preserved.\n" "- Approve repetition cleanup only when local context supports it as accidental repetition, hesitation, or verbal restart.\n" - "- Approve minor punctuation, spacing, or capitalization cleanup only when it is plausibly part of removing a dysfluency.\n" - "- Reject free-standing stylistic polishing, readability edits, paraphrases, and general rewriting.\n" - "- Reject edits that materially change the segment's substantive meaning, even if they are not literal antonyms.\n" + "- Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning.\n" "- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n" "- If a correction includes categories, treat them as additional segment context.\n" "- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" @@ -87,26 +89,8 @@ def build_spoken_word_messages(validation_payload: List[dict]) -> List[Message]: def build_grammar_only_messages(validation_payload: List[dict]) -> List[Message]: - payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2) - system = ( - "You are Audita, a conservative grammar-scope validation assistant. " - "Evaluate whether each proposed correction is acceptable for the grammar module. " - "Approve only low-risk transcript cleanup that is primarily grammatical in nature and preserves substantive meaning." - ) - user = ( - "Review these proposed transcript corrections and decide whether each one is acceptable for the grammar module.\n\n" - "Rules:\n" - "- Return one validation decision for every correction_index in the input.\n" - "- Approve conservative cleanup that primarily performs punctuation, capitalization, spacing, or whole-word article cleanup such as \"a\" <-> \"an\".\n" - "- You may also approve a likely homophone or mistranscription fix when it is embedded within an otherwise grammatical revision and the overall correction is still a low-risk transcript cleanup.\n" - "- Approve embedded recovery such as formatting cleanup plus a likely transcription fix when the full corrected segment remains conservative and contextually well supported.\n" - "- Reject free-standing homophone or mistranscription rewrites when they are not part of an otherwise grammatical cleanup.\n" - "- Reject filler cleanup, repetition cleanup, spoken-word dysfluency cleanup, stylistic polishing, broad paraphrase, and unrelated content substitutions.\n" - "- Reject corrections that go beyond conservative grammar-stage cleanup, even if some punctuation or capitalization cleanup is also present.\n" - "- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n" - "- If a correction includes categories, treat them as additional segment context.\n" - "- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" - "- confidence must be between 0.0 and 1.0.\n\n" - f"Corrections to validate:\n{payload_json}" - ) - return [{"role": "system", "content": system}, {"role": "user", "content": user}] + return build_editorial_messages(validation_payload) + + +def build_spoken_word_messages(validation_payload: List[dict]) -> List[Message]: + return build_editorial_messages(validation_payload) diff --git a/tests/test_llm_validators.py b/tests/test_llm_validators.py index 163d6f1..557fbcc 100644 --- a/tests/test_llm_validators.py +++ b/tests/test_llm_validators.py @@ -8,6 +8,7 @@ from audita.core.errors import AuditaLLMError from audita.core.schemas import parse_glossary_yaml, parse_transcript_json from audita.framework.models import CorrectionProposal, ModuleRunSpec from audita.validators import ( + EditorialValidator, GrammarOnlyValidator, IdenticalTextValidator, NonEmptySegmentValidator, @@ -312,6 +313,8 @@ def test_spoken_word_validator_approves_cleanup_and_rejects_rewrite(tmp_path): prompt_text = client.calls[0]["messages"][1]["content"] assert "I think we should go." in prompt_text assert "go now" in prompt_text + assert "acceptable editorial revision" in prompt_text + assert "homophone or mistranscription corrections" in prompt_text def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp_path): @@ -355,7 +358,7 @@ def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)] -def test_grammar_only_validator_approves_conservative_grammar_cleanup(tmp_path): +def test_grammar_only_validator_approves_editorial_cleanup(tmp_path): transcript = parse_transcript_json( """ [ @@ -431,9 +434,9 @@ def test_grammar_only_validator_approves_conservative_grammar_cleanup(tmp_path): (2, True), ] prompt_text = client.calls[0]["messages"][1]["content"] - assert "whole-word article cleanup" in prompt_text - assert "embedded within an otherwise grammatical revision" in prompt_text - assert "filler cleanup" in prompt_text + assert "acceptable editorial revision" in prompt_text + assert "article cleanup" in prompt_text + assert "dysfluency cleanup" in prompt_text def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path): @@ -496,7 +499,7 @@ def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path): ] -def test_grammar_only_validator_rejects_out_of_scope_rewrites(tmp_path): +def test_grammar_only_validator_allows_editorial_overlap_and_rejects_meaning_change(tmp_path): transcript = parse_transcript_json( """ [ @@ -563,31 +566,31 @@ def test_grammar_only_validator_rejects_out_of_scope_rewrites(tmp_path): "correction_index": 0, "approved": False, "confidence": 0.99, - "reason": "Unrelated word substitution rather than conservative grammar cleanup.", + "reason": "This changes meaning rather than preserving it through editorial cleanup.", }, { "correction_index": 1, - "approved": False, - "confidence": 0.99, - "reason": "Free-standing homophone rewrite rather than conservative grammar cleanup.", + "approved": True, + "confidence": 0.93, + "reason": "Low-risk editorial homophone correction that preserves meaning.", }, { "correction_index": 2, - "approved": False, - "confidence": 1.0, - "reason": "Spoken-word filler cleanup is out of scope for the grammar module.", + "approved": True, + "confidence": 0.95, + "reason": "Editorial dysfluency cleanup that preserves meaning.", }, { "correction_index": 3, - "approved": False, - "confidence": 1.0, - "reason": "Spoken-word repetition cleanup is out of scope for the grammar module.", + "approved": True, + "confidence": 0.95, + "reason": "Editorial repetition cleanup that preserves meaning.", }, { "correction_index": 4, - "approved": False, - "confidence": 0.98, - "reason": "Possessive rewrite goes beyond conservative grammar cleanup.", + "approved": True, + "confidence": 0.91, + "reason": "Conservative editorial possessive correction that preserves meaning.", }, ] } @@ -598,7 +601,7 @@ def test_grammar_only_validator_rejects_out_of_scope_rewrites(tmp_path): _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) ) - assert [decision.approved for decision in result.decisions] == [False, False, False, False, False] + assert [decision.approved for decision in result.decisions] == [False, True, True, True, True] def test_grammar_only_validator_allows_embedded_homophone_fix_within_grammar_revision(tmp_path): @@ -852,6 +855,20 @@ def test_original_text_present_validator_rejects_missing_spans_and_allows_presen }, "unknown correction_index", ), + ( + EditorialValidator("editorial_review"), + { + "validations": [ + { + "correction_index": 99, + "approved": True, + "confidence": 0.9, + "reason": "unknown", + } + ] + }, + "unknown correction_index", + ), ( SpokenWordValidator("spoken_word_review"), { @@ -1099,7 +1116,7 @@ def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path): assert all("corrected_segment_text" in payload for payload in chunk_calls[0]["payloads"]) -def test_grammar_validation_prompt_is_scoped_to_conservative_cleanup(): +def test_grammar_validation_prompt_is_scoped_to_editorial_cleanup(): messages = build_grammar_only_messages( [ { @@ -1115,10 +1132,10 @@ def test_grammar_validation_prompt_is_scoped_to_conservative_cleanup(): ) combined = messages[0]["content"] + messages[1]["content"] - assert "whole-word article cleanup" in combined - assert "embedded within an otherwise grammatical revision" in combined - assert "free-standing homophone" in combined - assert "spoken-word dysfluency cleanup" in combined + assert "acceptable editorial revision" in combined + assert "article cleanup" in combined + assert "homophone or mistranscription corrections" in combined + assert "dysfluency cleanup" in combined assert "original_segment_text" in messages[1]["content"] diff --git a/tests/test_module_proposals.py b/tests/test_module_proposals.py index a875723..88572ce 100644 --- a/tests/test_module_proposals.py +++ b/tests/test_module_proposals.py @@ -860,7 +860,7 @@ def test_process_transcript_result_rejects_grammar_below_threshold_before_later_ assert result.report.modules[0].validators[3].candidate_count == 0 -def test_process_transcript_result_grammar_module_still_rejects_homophone_style_proposals(tmp_path): +def test_process_transcript_result_grammar_module_allows_editorial_homophone_style_proposals(tmp_path): transcript = parse_source_transcript_json( """ [ @@ -902,9 +902,9 @@ def test_process_transcript_result_grammar_module_still_rejects_homophone_style_ "validations": [ { "correction_index": 0, - "approved": False, - "confidence": 0.99, - "reason": "Free-standing homophone rewrite rather than conservative grammar cleanup.", + "approved": True, + "confidence": 0.94, + "reason": "A low-risk editorial homophone correction that preserves meaning.", } ] }, @@ -929,14 +929,13 @@ def test_process_transcript_result_grammar_module_still_rejects_homophone_style_ llm_client=client, ) - assert result.transcript[0].text == "ChatGPT still can't really do that with a dam." + assert result.transcript[0].text == "ChatGPT still can't really do that with a damn." assert [call["stage_name"] for call in client.calls] == [ "grammar:proposal", "grammar:grammar_only_guard", "grammar:meaning_reversal_review", ] - assert result.report.skipped_corrections[0].source == "validator:grammar_only_guard" - assert "grammar cleanup" in result.report.skipped_corrections[0].reason + assert result.report.skipped_corrections == [] def test_process_transcript_result_grammar_module_allows_embedded_homophone_fix_with_grammar_cleanup(tmp_path):