eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Implemented a shared, more permissive editorial validatior for the grammar and spoken_word modules

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-01 07:01:51 -05:00
parent 0d5bdd1c84
commit 9ab5a60123
5 changed files with 75 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
tests/test_llm_validators.py
View File

@@ -8,6 +8,7 @@ from audita.core.errors import AuditaLLMError
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
from audita.framework.models import CorrectionProposal, ModuleRunSpec
from audita.validators import (
EditorialValidator,
GrammarOnlyValidator,
IdenticalTextValidator,
NonEmptySegmentValidator,
@@ -312,6 +313,8 @@ def test_spoken_word_validator_approves_cleanup_and_rejects_rewrite(tmp_path):
prompt_text = client.calls[0]["messages"][1]["content"]
assert "I think we should go." in prompt_text
assert "go now" in prompt_text
assert "acceptable editorial revision" in prompt_text
assert "homophone or mistranscription corrections" in prompt_text
def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp_path):
@@ -355,7 +358,7 @@ def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)]
def test_grammar_only_validator_approves_conservative_grammar_cleanup(tmp_path):
def test_grammar_only_validator_approves_editorial_cleanup(tmp_path):
transcript = parse_transcript_json(
"""
[
@@ -431,9 +434,9 @@ def test_grammar_only_validator_approves_conservative_grammar_cleanup(tmp_path):
(2, True),
]
prompt_text = client.calls[0]["messages"][1]["content"]
assert "whole-word article cleanup" in prompt_text
assert "embedded within an otherwise grammatical revision" in prompt_text
assert "filler cleanup" in prompt_text
assert "acceptable editorial revision" in prompt_text
assert "article cleanup" in prompt_text
assert "dysfluency cleanup" in prompt_text
def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path):
@@ -496,7 +499,7 @@ def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path):
]
def test_grammar_only_validator_rejects_out_of_scope_rewrites(tmp_path):
def test_grammar_only_validator_allows_editorial_overlap_and_rejects_meaning_change(tmp_path):
transcript = parse_transcript_json(
"""
[
@@ -563,31 +566,31 @@ def test_grammar_only_validator_rejects_out_of_scope_rewrites(tmp_path):
"correction_index": 0,
"approved": False,
"confidence": 0.99,
"reason": "Unrelated word substitution rather than conservative grammar cleanup.",
"reason": "This changes meaning rather than preserving it through editorial cleanup.",
},
{
"correction_index": 1,
"approved": False,
"confidence": 0.99,
"reason": "Free-standing homophone rewrite rather than conservative grammar cleanup.",
"approved": True,
"confidence": 0.93,
"reason": "Low-risk editorial homophone correction that preserves meaning.",
},
{
"correction_index": 2,
"approved": False,
"confidence": 1.0,
"reason": "Spoken-word filler cleanup is out of scope for the grammar module.",
"approved": True,
"confidence": 0.95,
"reason": "Editorial dysfluency cleanup that preserves meaning.",
},
{
"correction_index": 3,
"approved": False,
"confidence": 1.0,
"reason": "Spoken-word repetition cleanup is out of scope for the grammar module.",
"approved": True,
"confidence": 0.95,
"reason": "Editorial repetition cleanup that preserves meaning.",
},
{
"correction_index": 4,
"approved": False,
"confidence": 0.98,
"reason": "Possessive rewrite goes beyond conservative grammar cleanup.",
"approved": True,
"confidence": 0.91,
"reason": "Conservative editorial possessive correction that preserves meaning.",
},
]
}
@@ -598,7 +601,7 @@ def test_grammar_only_validator_rejects_out_of_scope_rewrites(tmp_path):
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [decision.approved for decision in result.decisions] == [False, False, False, False, False]
assert [decision.approved for decision in result.decisions] == [False, True, True, True, True]
def test_grammar_only_validator_allows_embedded_homophone_fix_within_grammar_revision(tmp_path):
@@ -852,6 +855,20 @@ def test_original_text_present_validator_rejects_missing_spans_and_allows_presen
},
"unknown correction_index",
),
(
EditorialValidator("editorial_review"),
{
"validations": [
{
"correction_index": 99,
"approved": True,
"confidence": 0.9,
"reason": "unknown",
}
]
},
"unknown correction_index",
),
(
SpokenWordValidator("spoken_word_review"),
{
@@ -1099,7 +1116,7 @@ def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path):
assert all("corrected_segment_text" in payload for payload in chunk_calls[0]["payloads"])
def test_grammar_validation_prompt_is_scoped_to_conservative_cleanup():
def test_grammar_validation_prompt_is_scoped_to_editorial_cleanup():
messages = build_grammar_only_messages(
[
{
@@ -1115,10 +1132,10 @@ def test_grammar_validation_prompt_is_scoped_to_conservative_cleanup():
)
combined = messages[0]["content"] + messages[1]["content"]
assert "whole-word article cleanup" in combined
assert "embedded within an otherwise grammatical revision" in combined
assert "free-standing homophone" in combined
assert "spoken-word dysfluency cleanup" in combined
assert "acceptable editorial revision" in combined
assert "article cleanup" in combined
assert "homophone or mistranscription corrections" in combined
assert "dysfluency cleanup" in combined
assert "original_segment_text" in messages[1]["content"]