eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Implemented the grammar LLM review module

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-25 09:06:06 -05:00
parent 52d29f7228
commit 4724cd16c8
12 changed files with 471 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
tests/test_llm_validators.py
View File

@@ -5,6 +5,7 @@ from audita.core.config import AuditaConfig
from audita.core.errors import AuditaLLMError
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
from audita.framework.models import CorrectionProposal, ModuleRunSpec
from audita.validators import GrammarOnlyValidator
from audita.validators.base import ValidationContext
from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
from audita.validators.prompts import (
@@ -309,6 +310,118 @@ def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)]
def test_grammar_only_validator_allows_formatting_only_changes(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello there"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "cant we go"},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "Hello,world"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="grammar",
module_key="grammar",
id=1,
original_text="hello there",
corrected_text="Hello there.",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="grammar",
module_key="grammar",
id=2,
original_text="cant",
corrected_text="can't",
confidence=0.95,
),
CorrectionProposal(
proposal_index=2,
module_instance="grammar",
module_key="grammar",
id=3,
original_text="Hello,world",
corrected_text="Hello, world",
confidence=0.95,
),
]
result = GrammarOnlyValidator("grammar_only_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
(0, True),
(1, True),
(2, True),
]
def test_grammar_only_validator_rejects_word_level_changes(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "their plan"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "dam"},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "uh"},
{"id": 4, "speaker": "A", "start": 3.0, "end": 4.0, "text": "I I agree"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="grammar",
module_key="grammar",
id=1,
original_text="their",
corrected_text="there",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="grammar",
module_key="grammar",
id=2,
original_text="dam",
corrected_text="damn",
confidence=0.95,
),
CorrectionProposal(
proposal_index=2,
module_instance="grammar",
module_key="grammar",
id=3,
original_text="uh",
corrected_text="",
confidence=0.95,
),
CorrectionProposal(
proposal_index=3,
module_instance="grammar",
module_key="grammar",
id=4,
original_text="I I",
corrected_text="I",
confidence=0.95,
),
]
result = GrammarOnlyValidator("grammar_only_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
)
assert [decision.approved for decision in result.decisions] == [False, False, False, False]
assert all(
decision.reason == "correction is not limited to punctuation, capitalization, and spacing"
for decision in result.decisions
)
@pytest.mark.parametrize(
("validator", "payload", "message_fragment"),
[

208
tests/test_module_proposals.py
View File

@@ -5,9 +5,14 @@ from audita.core.config import AuditaConfig
from audita.core.errors import AuditaLLMError
from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json, parse_transcript_json
from audita.framework.models import ModuleContext, ModuleRunSpec
from audita.modules.grammar import GrammarModule
from audita.modules.glossary import GlossaryModule
from audita.modules.homophones import HomophonesModule
from audita.modules.prompts import build_homophones_proposal_messages, build_spoken_word_proposal_messages
from audita.modules.prompts import (
build_grammar_proposal_messages,
build_homophones_proposal_messages,
build_spoken_word_proposal_messages,
)
from audita.modules.spoken_word import SpokenWordModule
from audita.pipeline import process_transcript_result
@@ -175,6 +180,70 @@ def test_spoken_word_prompt_is_explicitly_scoped_to_dysfluency_cleanup():
assert '"id": 1' in messages[1]["content"]
def test_grammar_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
]
"""
)
section = chunk_transcript(transcript, max_section_tokens=1000)[0]
module = GrammarModule()
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "hello world",
"corrected_text": "Hello world.",
"confidence": 0.95,
}
]
}
]
)
context = ModuleContext(
run_spec=ModuleRunSpec(instance_name="grammar", module_key="grammar", module=module),
glossary=_glossary(),
config=AuditaConfig.from_sources(env={}),
run_dir=tmp_path,
llm_client=client,
)
proposals = list(module.propose(section, context))
assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [
(1, "hello world", "Hello world.", 0.95)
]
assert (tmp_path / "prompt-0000.json").exists()
assert (tmp_path / "corrections-0000.json").exists()
prompt_text = client.calls[0]["messages"][1]["content"]
assert "punctuation, capitalization, and spacing" in prompt_text
assert "exact text span" in prompt_text
assert "word substitutions" in prompt_text
def test_grammar_prompt_is_explicitly_scoped_to_formatting_cleanup():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
]
"""
)
section = chunk_transcript(transcript, max_section_tokens=1000)[0]
messages = build_grammar_proposal_messages(section, _glossary())
combined = messages[0]["content"] + messages[1]["content"]
assert "punctuation, capitalization, and spacing" in combined
assert "word substitutions" in combined
assert "homophone fixes" in combined
assert '"id": 1' in messages[1]["content"]
def test_process_transcript_result_uses_injected_fake_client_and_applies_sequential_module_updates(tmp_path):
transcript = parse_source_transcript_json(
"""
@@ -194,7 +263,9 @@ def test_process_transcript_result_uses_injected_fake_client_and_applies_sequent
max_retries=config.max_retries,
max_section_tokens=config.max_section_tokens,
glossary_confidence_threshold=config.glossary_confidence_threshold,
grammar_confidence_threshold=config.grammar_confidence_threshold,
homophones_confidence_threshold=config.homophones_confidence_threshold,
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
normalize_max_segment_gap=config.normalize_max_segment_gap,
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
normalize_max_segment_duration=config.normalize_max_segment_duration,
@@ -266,6 +337,7 @@ def test_process_transcript_result_uses_injected_fake_client_and_applies_sequent
},
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
@@ -281,6 +353,7 @@ def test_process_transcript_result_uses_injected_fake_client_and_applies_sequent
"homophones:meaning_reversal_review",
"glossary_2:proposal",
"spoken_word:proposal",
"grammar:proposal",
]
assert "There were Jesters at the dam." in client.calls[3]["messages"][1]["content"]
@@ -301,7 +374,9 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_
max_retries=base_config.max_retries,
max_section_tokens=base_config.max_section_tokens,
glossary_confidence_threshold=0.96,
grammar_confidence_threshold=base_config.grammar_confidence_threshold,
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold,
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
@@ -324,6 +399,7 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_
{"corrections": []},
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
@@ -335,6 +411,7 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_
"homophones:proposal",
"glossary_2:proposal",
"spoken_word:proposal",
"grammar:proposal",
]
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
assert result.report.skipped_corrections[0].source == "validator:proposal_confidence_guard"
@@ -358,6 +435,7 @@ def test_process_transcript_result_runs_spoken_word_module_with_full_validator_c
max_retries=base_config.max_retries,
max_section_tokens=base_config.max_section_tokens,
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
grammar_confidence_threshold=base_config.grammar_confidence_threshold,
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold,
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
@@ -440,6 +518,7 @@ def test_process_transcript_result_rejects_spoken_word_below_threshold_before_ll
max_retries=base_config.max_retries,
max_section_tokens=base_config.max_section_tokens,
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
grammar_confidence_threshold=base_config.grammar_confidence_threshold,
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
spoken_word_confidence_threshold=0.96,
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
@@ -476,3 +555,130 @@ def test_process_transcript_result_rejects_spoken_word_below_threshold_before_ll
assert [call["stage_name"] for call in client.calls] == ["spoken_word:proposal"]
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
assert result.report.modules[0].validators[1].candidate_count == 0
def test_process_transcript_result_runs_grammar_module_with_full_validator_chain(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
]
"""
)
base_config = AuditaConfig.from_sources(env={})
config = AuditaConfig(
api_key=base_config.api_key,
model=base_config.model,
base_url=base_config.base_url,
max_retries=base_config.max_retries,
max_section_tokens=base_config.max_section_tokens,
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
grammar_confidence_threshold=base_config.grammar_confidence_threshold,
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold,
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
normalize_max_segment_tokens=base_config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="always",
)
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "hello world",
"corrected_text": "Hello world.",
"confidence": 0.95,
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.99,
"reason": "Does not reverse the segment meaning.",
}
]
},
]
)
result = process_transcript_result(
transcript,
_glossary(),
config,
module_keys=["grammar"],
llm_client=client,
)
assert result.transcript[0].text == "Hello world."
assert [call["stage_name"] for call in client.calls] == [
"grammar:proposal",
"grammar:meaning_reversal_review",
]
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"grammar_only_guard",
"meaning_reversal_review",
]
def test_process_transcript_result_rejects_grammar_below_threshold_before_later_validators(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
]
"""
)
base_config = AuditaConfig.from_sources(env={})
config = AuditaConfig(
api_key=base_config.api_key,
model=base_config.model,
base_url=base_config.base_url,
max_retries=base_config.max_retries,
max_section_tokens=base_config.max_section_tokens,
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
grammar_confidence_threshold=0.96,
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold,
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
normalize_max_segment_tokens=base_config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="always",
)
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "hello world",
"corrected_text": "Hello world.",
"confidence": 0.95,
}
]
}
]
)
result = process_transcript_result(
transcript,
_glossary(),
config,
module_keys=["grammar"],
llm_client=client,
)
assert result.transcript[0].text == "hello world"
assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"]
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
assert result.report.modules[0].validators[1].candidate_count == 0

1
tests/test_new_cli.py
View File

@@ -26,6 +26,7 @@ def test_process_help_exposes_framework_flags(capsys):
assert "--max-retries" in output
assert "--max-section-tokens" in output
assert "--glossary-confidence-threshold" in output
assert "--grammar-confidence-threshold" in output
assert "--homophones-confidence-threshold" in output
assert "--spoken-word-confidence-threshold" in output
assert "--work-dir-retention" in output

6
tests/test_new_config.py
View File

@@ -4,6 +4,7 @@ from audita.core.config import (
AuditaConfig,
ConfigOverrides,
DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD,
DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD,
DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD,
DEFAULT_NORMALIZE_MAX_SEGMENT_GAP,
DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD,
@@ -19,6 +20,7 @@ def test_default_config_allows_missing_api_key():
assert config.api_key is None
assert config.module_keys == DEFAULT_MODULE_KEYS
assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
assert config.grammar_confidence_threshold == DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD
assert config.homophones_confidence_threshold == DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD
assert config.spoken_word_confidence_threshold == DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD
assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
@@ -58,17 +60,20 @@ def test_threshold_overrides_take_precedence():
config = AuditaConfig.from_sources(
env={
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.6",
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.65",
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.7",
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.75",
},
overrides=ConfigOverrides(
glossary_confidence_threshold=0.85,
grammar_confidence_threshold=0.88,
homophones_confidence_threshold=0.9,
spoken_word_confidence_threshold=0.95,
),
)
assert config.glossary_confidence_threshold == 0.85
assert config.grammar_confidence_threshold == 0.88
assert config.homophones_confidence_threshold == 0.9
assert config.spoken_word_confidence_threshold == 0.95
@@ -90,6 +95,7 @@ def test_invalid_module_sequences_are_rejected(value):
"env_name",
[
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD",
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD",
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD",
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD",
],

27
tests/test_new_pipeline.py
View File

@@ -61,6 +61,7 @@ def test_process_transcript_runs_noop_framework(tmp_path):
{"corrections": []},
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
revised = process_transcript(
@@ -78,6 +79,7 @@ def test_process_transcript_runs_noop_framework(tmp_path):
"homophones:proposal",
"glossary_2:proposal",
"spoken_word:proposal",
"grammar:proposal",
]
@@ -93,7 +95,9 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
max_retries=config.max_retries,
max_section_tokens=config.max_section_tokens,
glossary_confidence_threshold=config.glossary_confidence_threshold,
grammar_confidence_threshold=config.grammar_confidence_threshold,
homophones_confidence_threshold=config.homophones_confidence_threshold,
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
normalize_max_segment_gap=config.normalize_max_segment_gap,
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
normalize_max_segment_duration=config.normalize_max_segment_duration,
@@ -108,6 +112,7 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
{"corrections": []},
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
@@ -135,6 +140,12 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
"spoken_word_review",
"meaning_reversal_review",
]
assert [validator["name"] for validator in result.report.modules[4].to_dict()["validators"]] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"grammar_only_guard",
"meaning_reversal_review",
]
def test_external_report_can_be_written(tmp_path):
@@ -145,6 +156,7 @@ def test_external_report_can_be_written(tmp_path):
{"corrections": []},
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
@@ -191,7 +203,12 @@ def test_default_module_specs_expose_final_validator_order():
"spoken_word_review",
"meaning_reversal_review",
]
assert [validator.name for validator in specs[4].module.validators()] == ["protected_glossary_guard"]
assert [validator.name for validator in specs[4].module.validators()] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"grammar_only_guard",
"meaning_reversal_review",
]
def test_process_transcript_result_missing_api_key_writes_failed_report(tmp_path):
@@ -206,7 +223,9 @@ def test_process_transcript_result_missing_api_key_writes_failed_report(tmp_path
max_retries=config.max_retries,
max_section_tokens=config.max_section_tokens,
glossary_confidence_threshold=config.glossary_confidence_threshold,
grammar_confidence_threshold=config.grammar_confidence_threshold,
homophones_confidence_threshold=config.homophones_confidence_threshold,
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
normalize_max_segment_gap=config.normalize_max_segment_gap,
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
normalize_max_segment_duration=config.normalize_max_segment_duration,
@@ -257,7 +276,9 @@ def test_process_transcript_result_preserves_partial_progress_when_later_module_
max_retries=config.max_retries,
max_section_tokens=config.max_section_tokens,
glossary_confidence_threshold=config.glossary_confidence_threshold,
grammar_confidence_threshold=config.grammar_confidence_threshold,
homophones_confidence_threshold=config.homophones_confidence_threshold,
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
normalize_max_segment_gap=config.normalize_max_segment_gap,
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
normalize_max_segment_duration=config.normalize_max_segment_duration,
@@ -330,7 +351,9 @@ def test_process_transcript_result_preserves_partial_skips_and_validator_diagnos
max_retries=config.max_retries,
max_section_tokens=config.max_section_tokens,
glossary_confidence_threshold=0.8,
grammar_confidence_threshold=config.grammar_confidence_threshold,
homophones_confidence_threshold=config.homophones_confidence_threshold,
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
normalize_max_segment_gap=config.normalize_max_segment_gap,
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
normalize_max_segment_duration=config.normalize_max_segment_duration,
@@ -404,7 +427,7 @@ def test_process_transcript_result_supports_grammar_only_module_override(tmp_pat
_glossary(),
config,
module_keys=["grammar"],
llm_client=FakeStructuredLLMClient([]),
llm_client=FakeStructuredLLMClient([{"corrections": []}]),
)
assert [segment.id for segment in result.transcript] == [1, 2]