eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Implemented a shared, more permissive editorial validatior for the grammar and spoken_word modules

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-01 07:01:51 -05:00
parent 0d5bdd1c84
commit 9ab5a60123
5 changed files with 75 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/audita/validators/__init__.py
View File

@@ -7,7 +7,7 @@ from .deterministic import (
ProposalConfidenceValidator, ProposalConfidenceValidator,
ProtectedGlossaryTermsValidator, ProtectedGlossaryTermsValidator,
) )
from .llm import GrammarOnlyValidator, MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator from .llm import EditorialValidator, GrammarOnlyValidator, MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
from .protection import ProtectedVocabulary from .protection import ProtectedVocabulary
__all__ = [ __all__ = [
@@ -21,6 +21,7 @@ __all__ = [
"ProtectedGlossaryTermsValidator", "ProtectedGlossaryTermsValidator",
"GlossaryStageProtectedGlossaryTermsValidator", "GlossaryStageProtectedGlossaryTermsValidator",
"NonEmptySegmentValidator", "NonEmptySegmentValidator",
"EditorialValidator",
"GrammarOnlyValidator", "GrammarOnlyValidator",
"ProtectedVocabulary", "ProtectedVocabulary",
"SpokenFormPlausibilityValidator", "SpokenFormPlausibilityValidator",

13
src/audita/validators/llm.py
View File

@@ -14,10 +14,9 @@ from audita.framework.proposals import ProposalPreview, ProposalPreviewError, pr
from .base import ValidationContext, ValidationDecision, ValidationResult from .base import ValidationContext, ValidationDecision, ValidationResult
from .prompts import ( from .prompts import (
build_grammar_only_messages, build_editorial_messages,
build_meaning_reversal_messages, build_meaning_reversal_messages,
build_spoken_form_plausibility_messages, build_spoken_form_plausibility_messages,
build_spoken_word_messages,
) )
@@ -174,16 +173,22 @@ class MeaningReversalValidator(_BaseLLMValidator):
prompt_builder: PromptBuilder = build_meaning_reversal_messages prompt_builder: PromptBuilder = build_meaning_reversal_messages
@dataclass(frozen=True)
class EditorialValidator(_BaseLLMValidator):
name: str = "editorial_review"
prompt_builder: PromptBuilder = build_editorial_messages
@dataclass(frozen=True) @dataclass(frozen=True)
class SpokenWordValidator(_BaseLLMValidator): class SpokenWordValidator(_BaseLLMValidator):
name: str = "spoken_word_review" name: str = "spoken_word_review"
prompt_builder: PromptBuilder = build_spoken_word_messages prompt_builder: PromptBuilder = build_editorial_messages
@dataclass(frozen=True) @dataclass(frozen=True)
class GrammarOnlyValidator(_BaseLLMValidator): class GrammarOnlyValidator(_BaseLLMValidator):
name: str = "grammar_only_guard" name: str = "grammar_only_guard"
prompt_builder: PromptBuilder = build_grammar_only_messages prompt_builder: PromptBuilder = build_editorial_messages
def _write_json(path: Path, payload: dict) -> None: def _write_json(path: Path, payload: dict) -> None:

50
src/audita/validators/prompts.py
View File

@@ -58,25 +58,27 @@ def build_meaning_reversal_messages(validation_payload: List[dict]) -> List[Mess
return [{"role": "system", "content": system}, {"role": "user", "content": user}] return [{"role": "system", "content": system}, {"role": "user", "content": user}]
def build_spoken_word_messages(validation_payload: List[dict]) -> List[Message]: def build_editorial_messages(validation_payload: List[dict]) -> List[Message]:
payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2) payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2)
system = ( system = (
"You are Audita, a conservative spoken-word cleanup validation assistant. " "You are Audita, a conservative editorial validation assistant. "
"Evaluate whether each proposed correction is a reasonable cleanup of repeated words, repeated short phrases, " "Evaluate whether each proposed correction is editorial in nature and preserves the segment's underlying meaning. "
"filler words, hesitation artifacts, or similar spoken dysfluencies. " "Editorial revisions may include dysfluency cleanup, punctuation changes, capitalization changes, "
"Approve only low-risk cleanup that preserves the segment's substantive meaning." "homophone or mistranscription corrections, and similar low-risk editorial cleanup."
) )
user = ( user = (
"Review these proposed transcript corrections and decide whether each one is a valid spoken-word cleanup.\n\n" "Review these proposed transcript corrections and decide whether each one is an acceptable editorial revision.\n\n"
"Rules:\n" "Rules:\n"
"- Return one validation decision for every correction_index in the input.\n" "- Return one validation decision for every correction_index in the input.\n"
"- Approve cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies.\n" "- Approve editorial revisions that preserve the underlying meaning of the segment.\n"
"- Approve dysfluency cleanup, including cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies.\n"
"- Approve punctuation, spacing, capitalization, and article cleanup when they preserve meaning.\n"
"- Approve low-risk homophone or mistranscription corrections when they are contextually well supported.\n"
"- Approve conservative combinations of these editorial changes when the overall revision remains meaning-preserving.\n"
"- Reject repetition cleanup when the repetition plausibly serves urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency.\n" "- Reject repetition cleanup when the repetition plausibly serves urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency.\n"
"- Phrases such as \"Help! Help! Help!\", \"Stop! Stop! Stop!\", \"No! No! No!\", \"Yes! Yes! Yes!\", and \"Go! Go! Go!\" are often intentional emphasis and should usually be preserved.\n" "- Phrases such as \"Help! Help! Help!\", \"Stop! Stop! Stop!\", \"No! No! No!\", \"Yes! Yes! Yes!\", and \"Go! Go! Go!\" are often intentional emphasis and should usually be preserved.\n"
"- Approve repetition cleanup only when local context supports it as accidental repetition, hesitation, or verbal restart.\n" "- Approve repetition cleanup only when local context supports it as accidental repetition, hesitation, or verbal restart.\n"
"- Approve minor punctuation, spacing, or capitalization cleanup only when it is plausibly part of removing a dysfluency.\n" "- Reject broad paraphrase, substantive semantic changes, substantive meaning changes, and revisions that materially change, obscure, distort, or reverse the segment's meaning.\n"
"- Reject free-standing stylistic polishing, readability edits, paraphrases, and general rewriting.\n"
"- Reject edits that materially change the segment's substantive meaning, even if they are not literal antonyms.\n"
"- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n" "- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n"
"- If a correction includes categories, treat them as additional segment context.\n" "- If a correction includes categories, treat them as additional segment context.\n"
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" "- Each returned validation must contain only correction_index, approved, confidence, and reason.\n"
@@ -87,26 +89,8 @@ def build_spoken_word_messages(validation_payload: List[dict]) -> List[Message]:
def build_grammar_only_messages(validation_payload: List[dict]) -> List[Message]: def build_grammar_only_messages(validation_payload: List[dict]) -> List[Message]:
payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2) return build_editorial_messages(validation_payload)
system = (
"You are Audita, a conservative grammar-scope validation assistant. "
"Evaluate whether each proposed correction is acceptable for the grammar module. " def build_spoken_word_messages(validation_payload: List[dict]) -> List[Message]:
"Approve only low-risk transcript cleanup that is primarily grammatical in nature and preserves substantive meaning." return build_editorial_messages(validation_payload)
)
user = (
"Review these proposed transcript corrections and decide whether each one is acceptable for the grammar module.\n\n"
"Rules:\n"
"- Return one validation decision for every correction_index in the input.\n"
"- Approve conservative cleanup that primarily performs punctuation, capitalization, spacing, or whole-word article cleanup such as \"a\" <-> \"an\".\n"
"- You may also approve a likely homophone or mistranscription fix when it is embedded within an otherwise grammatical revision and the overall correction is still a low-risk transcript cleanup.\n"
"- Approve embedded recovery such as formatting cleanup plus a likely transcription fix when the full corrected segment remains conservative and contextually well supported.\n"
"- Reject free-standing homophone or mistranscription rewrites when they are not part of an otherwise grammatical cleanup.\n"
"- Reject filler cleanup, repetition cleanup, spoken-word dysfluency cleanup, stylistic polishing, broad paraphrase, and unrelated content substitutions.\n"
"- Reject corrections that go beyond conservative grammar-stage cleanup, even if some punctuation or capitalization cleanup is also present.\n"
"- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n"
"- If a correction includes categories, treat them as additional segment context.\n"
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n"
"- confidence must be between 0.0 and 1.0.\n\n"
f"Corrections to validate:\n{payload_json}"
)
return [{"role": "system", "content": system}, {"role": "user", "content": user}]

65
tests/test_llm_validators.py
View File

@@ -8,6 +8,7 @@ from audita.core.errors import AuditaLLMError
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
from audita.framework.models import CorrectionProposal, ModuleRunSpec from audita.framework.models import CorrectionProposal, ModuleRunSpec
from audita.validators import ( from audita.validators import (
EditorialValidator,
GrammarOnlyValidator, GrammarOnlyValidator,
IdenticalTextValidator, IdenticalTextValidator,
NonEmptySegmentValidator, NonEmptySegmentValidator,
@@ -312,6 +313,8 @@ def test_spoken_word_validator_approves_cleanup_and_rejects_rewrite(tmp_path):
prompt_text = client.calls[0]["messages"][1]["content"] prompt_text = client.calls[0]["messages"][1]["content"]
assert "I think we should go." in prompt_text assert "I think we should go." in prompt_text
assert "go now" in prompt_text assert "go now" in prompt_text
assert "acceptable editorial revision" in prompt_text
assert "homophone or mistranscription corrections" in prompt_text
def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp_path): def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp_path):
@@ -355,7 +358,7 @@ def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)] assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)]
def test_grammar_only_validator_approves_conservative_grammar_cleanup(tmp_path): def test_grammar_only_validator_approves_editorial_cleanup(tmp_path):
transcript = parse_transcript_json( transcript = parse_transcript_json(
""" """
[ [
@@ -431,9 +434,9 @@ def test_grammar_only_validator_approves_conservative_grammar_cleanup(tmp_path):
(2, True), (2, True),
] ]
prompt_text = client.calls[0]["messages"][1]["content"] prompt_text = client.calls[0]["messages"][1]["content"]
assert "whole-word article cleanup" in prompt_text assert "acceptable editorial revision" in prompt_text
assert "embedded within an otherwise grammatical revision" in prompt_text assert "article cleanup" in prompt_text
assert "filler cleanup" in prompt_text assert "dysfluency cleanup" in prompt_text
def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path): def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path):
@@ -496,7 +499,7 @@ def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path):
] ]
def test_grammar_only_validator_rejects_out_of_scope_rewrites(tmp_path): def test_grammar_only_validator_allows_editorial_overlap_and_rejects_meaning_change(tmp_path):
transcript = parse_transcript_json( transcript = parse_transcript_json(
""" """
[ [
@@ -563,31 +566,31 @@ def test_grammar_only_validator_rejects_out_of_scope_rewrites(tmp_path):
"correction_index": 0, "correction_index": 0,
"approved": False, "approved": False,
"confidence": 0.99, "confidence": 0.99,
"reason": "Unrelated word substitution rather than conservative grammar cleanup.", "reason": "This changes meaning rather than preserving it through editorial cleanup.",
}, },
{ {
"correction_index": 1, "correction_index": 1,
"approved": False, "approved": True,
"confidence": 0.99, "confidence": 0.93,
"reason": "Free-standing homophone rewrite rather than conservative grammar cleanup.", "reason": "Low-risk editorial homophone correction that preserves meaning.",
}, },
{ {
"correction_index": 2, "correction_index": 2,
"approved": False, "approved": True,
"confidence": 1.0, "confidence": 0.95,
"reason": "Spoken-word filler cleanup is out of scope for the grammar module.", "reason": "Editorial dysfluency cleanup that preserves meaning.",
}, },
{ {
"correction_index": 3, "correction_index": 3,
"approved": False, "approved": True,
"confidence": 1.0, "confidence": 0.95,
"reason": "Spoken-word repetition cleanup is out of scope for the grammar module.", "reason": "Editorial repetition cleanup that preserves meaning.",
}, },
{ {
"correction_index": 4, "correction_index": 4,
"approved": False, "approved": True,
"confidence": 0.98, "confidence": 0.91,
"reason": "Possessive rewrite goes beyond conservative grammar cleanup.", "reason": "Conservative editorial possessive correction that preserves meaning.",
}, },
] ]
} }
@@ -598,7 +601,7 @@ def test_grammar_only_validator_rejects_out_of_scope_rewrites(tmp_path):
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
) )
assert [decision.approved for decision in result.decisions] == [False, False, False, False, False] assert [decision.approved for decision in result.decisions] == [False, True, True, True, True]
def test_grammar_only_validator_allows_embedded_homophone_fix_within_grammar_revision(tmp_path): def test_grammar_only_validator_allows_embedded_homophone_fix_within_grammar_revision(tmp_path):
@@ -852,6 +855,20 @@ def test_original_text_present_validator_rejects_missing_spans_and_allows_presen
}, },
"unknown correction_index", "unknown correction_index",
), ),
(
EditorialValidator("editorial_review"),
{
"validations": [
{
"correction_index": 99,
"approved": True,
"confidence": 0.9,
"reason": "unknown",
}
]
},
"unknown correction_index",
),
( (
SpokenWordValidator("spoken_word_review"), SpokenWordValidator("spoken_word_review"),
{ {
@@ -1099,7 +1116,7 @@ def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path):
assert all("corrected_segment_text" in payload for payload in chunk_calls[0]["payloads"]) assert all("corrected_segment_text" in payload for payload in chunk_calls[0]["payloads"])
def test_grammar_validation_prompt_is_scoped_to_conservative_cleanup(): def test_grammar_validation_prompt_is_scoped_to_editorial_cleanup():
messages = build_grammar_only_messages( messages = build_grammar_only_messages(
[ [
{ {
@@ -1115,10 +1132,10 @@ def test_grammar_validation_prompt_is_scoped_to_conservative_cleanup():
) )
combined = messages[0]["content"] + messages[1]["content"] combined = messages[0]["content"] + messages[1]["content"]
assert "whole-word article cleanup" in combined assert "acceptable editorial revision" in combined
assert "embedded within an otherwise grammatical revision" in combined assert "article cleanup" in combined
assert "free-standing homophone" in combined assert "homophone or mistranscription corrections" in combined
assert "spoken-word dysfluency cleanup" in combined assert "dysfluency cleanup" in combined
assert "original_segment_text" in messages[1]["content"] assert "original_segment_text" in messages[1]["content"]

13
tests/test_module_proposals.py
View File

@@ -860,7 +860,7 @@ def test_process_transcript_result_rejects_grammar_below_threshold_before_later_
assert result.report.modules[0].validators[3].candidate_count == 0 assert result.report.modules[0].validators[3].candidate_count == 0
def test_process_transcript_result_grammar_module_still_rejects_homophone_style_proposals(tmp_path): def test_process_transcript_result_grammar_module_allows_editorial_homophone_style_proposals(tmp_path):
transcript = parse_source_transcript_json( transcript = parse_source_transcript_json(
""" """
[ [
@@ -902,9 +902,9 @@ def test_process_transcript_result_grammar_module_still_rejects_homophone_style_
"validations": [ "validations": [
{ {
"correction_index": 0, "correction_index": 0,
"approved": False, "approved": True,
"confidence": 0.99, "confidence": 0.94,
"reason": "Free-standing homophone rewrite rather than conservative grammar cleanup.", "reason": "A low-risk editorial homophone correction that preserves meaning.",
} }
] ]
}, },
@@ -929,14 +929,13 @@ def test_process_transcript_result_grammar_module_still_rejects_homophone_style_
llm_client=client, llm_client=client,
) )
assert result.transcript[0].text == "ChatGPT still can't really do that with a dam." assert result.transcript[0].text == "ChatGPT still can't really do that with a damn."
assert [call["stage_name"] for call in client.calls] == [ assert [call["stage_name"] for call in client.calls] == [
"grammar:proposal", "grammar:proposal",
"grammar:grammar_only_guard", "grammar:grammar_only_guard",
"grammar:meaning_reversal_review", "grammar:meaning_reversal_review",
] ]
assert result.report.skipped_corrections[0].source == "validator:grammar_only_guard" assert result.report.skipped_corrections == []
assert "grammar cleanup" in result.report.skipped_corrections[0].reason
def test_process_transcript_result_grammar_module_allows_embedded_homophone_fix_with_grammar_cleanup(tmp_path): def test_process_transcript_result_grammar_module_allows_embedded_homophone_fix_with_grammar_cleanup(tmp_path):