eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Narrowed the grammar module LLM prompt and added support for a <-> an replacements

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-25 13:25:19 -05:00
parent 4724cd16c8
commit 5944f13404
4 changed files with 213 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
tests/test_llm_validators.py
View File

@@ -361,6 +361,46 @@ def test_grammar_only_validator_allows_formatting_only_changes(tmp_path):
]
def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "a intelligence saving throw"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "an owl"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="grammar",
module_key="grammar",
id=1,
original_text="a intelligence saving throw",
corrected_text="an intelligence saving throw",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="grammar",
module_key="grammar",
id=2,
original_text="an owl",
corrected_text="a owl",
confidence=0.95,
),
]
result = GrammarOnlyValidator("grammar_only_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
(0, True),
(1, True),
]
def test_grammar_only_validator_rejects_word_level_changes(tmp_path):
transcript = parse_transcript_json(
"""
@@ -368,7 +408,8 @@ def test_grammar_only_validator_rejects_word_level_changes(tmp_path):
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "their plan"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "dam"},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "uh"},
{"id": 4, "speaker": "A", "start": 3.0, "end": 4.0, "text": "I I agree"}
{"id": 4, "speaker": "A", "start": 3.0, "end": 4.0, "text": "I I agree"},
{"id": 5, "speaker": "A", "start": 4.0, "end": 5.0, "text": "Eric"}
]
"""
)
@@ -409,13 +450,22 @@ def test_grammar_only_validator_rejects_word_level_changes(tmp_path):
corrected_text="I",
confidence=0.95,
),
CorrectionProposal(
proposal_index=4,
module_instance="grammar",
module_key="grammar",
id=5,
original_text="Eric",
corrected_text="Eric's",
confidence=0.95,
),
]
result = GrammarOnlyValidator("grammar_only_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
)
assert [decision.approved for decision in result.decisions] == [False, False, False, False]
assert [decision.approved for decision in result.decisions] == [False, False, False, False, False]
assert all(
decision.reason == "correction is not limited to punctuation, capitalization, and spacing"
for decision in result.decisions

128
tests/test_module_proposals.py
View File

@@ -220,9 +220,10 @@ def test_grammar_module_propose_writes_diagnostics_and_returns_proposals_without
assert (tmp_path / "prompt-0000.json").exists()
assert (tmp_path / "corrections-0000.json").exists()
prompt_text = client.calls[0]["messages"][1]["content"]
assert "punctuation, capitalization, and spacing" in prompt_text
assert "punctuation, capitalization, spacing, and article cleanup" in prompt_text
assert "exact text span" in prompt_text
assert "word substitutions" in prompt_text
assert "later review stage" in prompt_text
def test_grammar_prompt_is_explicitly_scoped_to_formatting_cleanup():
@@ -238,9 +239,11 @@ def test_grammar_prompt_is_explicitly_scoped_to_formatting_cleanup():
messages = build_grammar_proposal_messages(section, _glossary())
combined = messages[0]["content"] + messages[1]["content"]
assert "punctuation, capitalization, and spacing" in combined
assert "punctuation, capitalization, spacing, and article cleanup" in combined
assert "word substitutions" in combined
assert "homophone fixes" in combined
assert "later review stage" in combined
assert "mistranscription" in combined
assert '"id": 1' in messages[1]["content"]
@@ -629,6 +632,72 @@ def test_process_transcript_result_runs_grammar_module_with_full_validator_chain
]
def test_process_transcript_result_grammar_module_applies_indefinite_article_cleanup(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Give me a intelligence saving throw."}
]
"""
)
base_config = AuditaConfig.from_sources(env={})
config = AuditaConfig(
api_key=base_config.api_key,
model=base_config.model,
base_url=base_config.base_url,
max_retries=base_config.max_retries,
max_section_tokens=base_config.max_section_tokens,
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
grammar_confidence_threshold=base_config.grammar_confidence_threshold,
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold,
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
normalize_max_segment_tokens=base_config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="always",
)
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "a intelligence saving throw",
"corrected_text": "an intelligence saving throw",
"confidence": 0.95,
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.99,
"reason": "Does not reverse the segment meaning.",
}
]
},
]
)
result = process_transcript_result(
transcript,
_glossary(),
config,
module_keys=["grammar"],
llm_client=client,
)
assert result.transcript[0].text == "Give me an intelligence saving throw."
assert [call["stage_name"] for call in client.calls] == [
"grammar:proposal",
"grammar:meaning_reversal_review",
]
def test_process_transcript_result_rejects_grammar_below_threshold_before_later_validators(tmp_path):
transcript = parse_source_transcript_json(
"""
@@ -682,3 +751,58 @@ def test_process_transcript_result_rejects_grammar_below_threshold_before_later_
assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"]
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
assert result.report.modules[0].validators[1].candidate_count == 0
def test_process_transcript_result_grammar_module_still_rejects_homophone_style_proposals(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."}
]
"""
)
base_config = AuditaConfig.from_sources(env={})
config = AuditaConfig(
api_key=base_config.api_key,
model=base_config.model,
base_url=base_config.base_url,
max_retries=base_config.max_retries,
max_section_tokens=base_config.max_section_tokens,
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
grammar_confidence_threshold=base_config.grammar_confidence_threshold,
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold,
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
normalize_max_segment_tokens=base_config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="always",
)
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "dam",
"corrected_text": "damn",
"confidence": 0.95,
}
]
}
]
)
result = process_transcript_result(
transcript,
_glossary(),
config,
module_keys=["grammar"],
llm_client=client,
)
assert result.transcript[0].text == "ChatGPT still can't really do that with a dam."
assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"]
assert result.report.skipped_corrections[0].source == "validator:grammar_only_guard"
assert result.report.skipped_corrections[0].reason == "correction is not limited to punctuation, capitalization, and spacing"