eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Implemented the grammar LLM review module

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-25 09:06:06 -05:00
parent 52d29f7228
commit 4724cd16c8
12 changed files with 471 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
tests/test_llm_validators.py
View File

@@ -5,6 +5,7 @@ from audita.core.config import AuditaConfig
from audita.core.errors import AuditaLLMError
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
from audita.framework.models import CorrectionProposal, ModuleRunSpec
from audita.validators import GrammarOnlyValidator
from audita.validators.base import ValidationContext
from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
from audita.validators.prompts import (
@@ -309,6 +310,118 @@ def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)]
def test_grammar_only_validator_allows_formatting_only_changes(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello there"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "cant we go"},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "Hello,world"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="grammar",
module_key="grammar",
id=1,
original_text="hello there",
corrected_text="Hello there.",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="grammar",
module_key="grammar",
id=2,
original_text="cant",
corrected_text="can't",
confidence=0.95,
),
CorrectionProposal(
proposal_index=2,
module_instance="grammar",
module_key="grammar",
id=3,
original_text="Hello,world",
corrected_text="Hello, world",
confidence=0.95,
),
]
result = GrammarOnlyValidator("grammar_only_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
(0, True),
(1, True),
(2, True),
]
def test_grammar_only_validator_rejects_word_level_changes(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "their plan"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "dam"},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "uh"},
{"id": 4, "speaker": "A", "start": 3.0, "end": 4.0, "text": "I I agree"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="grammar",
module_key="grammar",
id=1,
original_text="their",
corrected_text="there",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="grammar",
module_key="grammar",
id=2,
original_text="dam",
corrected_text="damn",
confidence=0.95,
),
CorrectionProposal(
proposal_index=2,
module_instance="grammar",
module_key="grammar",
id=3,
original_text="uh",
corrected_text="",
confidence=0.95,
),
CorrectionProposal(
proposal_index=3,
module_instance="grammar",
module_key="grammar",
id=4,
original_text="I I",
corrected_text="I",
confidence=0.95,
),
]
result = GrammarOnlyValidator("grammar_only_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
)
assert [decision.approved for decision in result.decisions] == [False, False, False, False]
assert all(
decision.reason == "correction is not limited to punctuation, capitalization, and spacing"
for decision in result.decisions
)
@pytest.mark.parametrize(
("validator", "payload", "message_fragment"),
[