eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Implemented a validation step to confirm that grammatical changes do not change substantive meaning

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-22 15:58:05 -05:00
parent 87c47517a0
commit d7788da1f4
15 changed files with 849 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/test_cli.py
View File

@@ -21,6 +21,8 @@ def test_process_help_includes_glossary_pass_flag(capsys):
assert "--grammar-max-llm-passes" in output
assert "--glossary-confidence-threshold" in output
assert "--grammar-confidence-threshold" in output
assert "--grammar-validation-enabled" in output
assert "--grammar-validation-confidence-threshold" in output
assert "--normalize-max-segment-gap" in output
assert "--normalize-ellipsis-gap" in output
assert "--normalize-max-segment-duration" in output

27
tests/test_config.py
View File

@@ -8,6 +8,8 @@ from audita.config import (
DEFAULT_GLOSSARY_MAX_LLM_PASSES,
DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD,
DEFAULT_GRAMMAR_MAX_LLM_PASSES,
DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD,
DEFAULT_GRAMMAR_VALIDATION_ENABLED,
DEFAULT_MAX_RETRIES,
DEFAULT_MAX_SECTION_TOKENS,
DEFAULT_NORMALIZE_ELLIPSIS_GAP,
@@ -30,6 +32,10 @@ def test_config_uses_defaults_with_api_key():
assert config.max_retries == DEFAULT_MAX_RETRIES
assert config.glossary_max_llm_passes == DEFAULT_GLOSSARY_MAX_LLM_PASSES
assert config.grammar_max_llm_passes == DEFAULT_GRAMMAR_MAX_LLM_PASSES
assert config.grammar_validation_enabled == DEFAULT_GRAMMAR_VALIDATION_ENABLED
assert config.grammar_validation_enabled is True
assert config.grammar_validation_confidence_threshold == DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD
assert config.grammar_validation_confidence_threshold == 0.8
assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
assert config.normalize_max_segment_gap == 5.0
assert config.normalize_ellipsis_gap == DEFAULT_NORMALIZE_ELLIPSIS_GAP
@@ -50,6 +56,8 @@ def test_config_env_overrides_defaults():
"AUDITA_MAX_RETRIES": "5",
"AUDITA_GLOSSARY_MAX_LLM_PASSES": "7",
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "4",
"AUDITA_GRAMMAR_VALIDATION_ENABLED": "false",
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "0.91",
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "4.5",
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "1.5",
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0",
@@ -64,6 +72,8 @@ def test_config_env_overrides_defaults():
assert config.max_retries == 5
assert config.glossary_max_llm_passes == 7
assert config.grammar_max_llm_passes == 4
assert config.grammar_validation_enabled is False
assert config.grammar_validation_confidence_threshold == 0.91
assert config.normalize_max_segment_gap == 4.5
assert config.normalize_ellipsis_gap == 1.5
assert config.normalize_max_segment_duration == 45.0
@@ -79,6 +89,8 @@ def test_config_cli_overrides_env():
"AUDITA_MAX_RETRIES": "5",
"AUDITA_GLOSSARY_MAX_LLM_PASSES": "7",
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "6",
"AUDITA_GRAMMAR_VALIDATION_ENABLED": "false",
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "0.91",
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "4.5",
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "1.5",
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0",
@@ -92,6 +104,8 @@ def test_config_cli_overrides_env():
max_retries=3,
glossary_max_llm_passes=2,
grammar_max_llm_passes=3,
grammar_validation_enabled=True,
grammar_validation_confidence_threshold=0.75,
normalize_max_segment_gap=3.0,
normalize_ellipsis_gap=1.0,
normalize_max_segment_duration=30.0,
@@ -106,6 +120,8 @@ def test_config_cli_overrides_env():
assert config.max_retries == 3
assert config.glossary_max_llm_passes == 2
assert config.grammar_max_llm_passes == 3
assert config.grammar_validation_enabled is True
assert config.grammar_validation_confidence_threshold == 0.75
assert config.normalize_max_segment_gap == 3.0
assert config.normalize_ellipsis_gap == 1.0
assert config.normalize_max_segment_duration == 30.0
@@ -148,6 +164,17 @@ def test_config_rejects_invalid_stage_thresholds():
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "-0.1"}
)
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "1.1"}
)
def test_config_rejects_invalid_grammar_validation_enabled():
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GRAMMAR_VALIDATION_ENABLED": "maybe"}
)
def test_legacy_confidence_threshold_env_is_ignored():

239
tests/test_pipeline.py
View File

@@ -1,23 +1,49 @@
import json
import pytest
from audita.config import AuditaConfig
from audita.errors import AuditaError
from audita.pipeline import process_transcript
from audita.schemas import CorrectionCandidate, CorrectionSet, parse_glossary_yaml, parse_source_transcript_json
from audita.schemas import (
CorrectionCandidate,
CorrectionSet,
GrammarValidationDecision,
GrammarValidationSet,
parse_glossary_yaml,
parse_source_transcript_json,
)
class FakeLLMClient:
def __init__(self, responses):
def __init__(self, responses, validation_responses=None):
self.responses = list(responses)
self.validation_responses = list(validation_responses or [])
self.calls = 0
self.validation_calls = 0
self.messages = []
self.validation_messages = []
def create_corrections(self, messages, config):
self.calls += 1
self.messages.append(messages)
return self.responses.pop(0)
def create_grammar_validations(self, messages, config):
self.validation_calls += 1
self.validation_messages.append(messages)
if not self.validation_responses:
raise AssertionError("Unexpected grammar validation request.")
return self.validation_responses.pop(0)
def _config(tmp_path, glossary_max_llm_passes=3, grammar_max_llm_passes=3):
def _config(
tmp_path,
glossary_max_llm_passes=3,
grammar_max_llm_passes=3,
grammar_validation_enabled=False,
grammar_validation_confidence_threshold=0.8,
):
return AuditaConfig(
api_key="key",
max_section_tokens=16000,
@@ -26,6 +52,8 @@ def _config(tmp_path, glossary_max_llm_passes=3, grammar_max_llm_passes=3):
max_retries=3,
glossary_max_llm_passes=glossary_max_llm_passes,
grammar_max_llm_passes=grammar_max_llm_passes,
grammar_validation_enabled=grammar_validation_enabled,
grammar_validation_confidence_threshold=grammar_validation_confidence_threshold,
work_dir=tmp_path / "work",
)
@@ -541,6 +569,8 @@ def test_pipeline_writes_stage_metadata_for_unresolved_retries(tmp_path):
assert metadata["grammar_max_llm_passes"] == 3
assert metadata["glossary_confidence_threshold"] == 0.8
assert metadata["grammar_confidence_threshold"] == 0.8
assert metadata["grammar_validation_enabled"] is False
assert metadata["grammar_validation_confidence_threshold"] == 0.8
assert [item["stage"] for item in metadata["stages"]] == ["glossary", "grammar"]
assert [item["pass_number"] for item in metadata["stages"][0]["passes"]] == [1, 2]
assert metadata["stages"][0]["passes"][0]["retry_segment_count"] == 1
@@ -589,6 +619,209 @@ def test_grammar_stage_runs_after_glossary_and_sees_corrected_text(tmp_path):
assert revised[0].text == "I ask Chauntea."
def test_grammar_validation_rejects_semantic_change(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "He became visible."}
]
"""
)
grammar_correction = CorrectionCandidate(
id=1,
original_text="visible",
corrected_text="invisible",
confidence=0.95,
)
validation = GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=False,
confidence=0.99,
reason="This reverses visible to invisible.",
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[grammar_correction]),
],
validation_responses=[GrammarValidationSet(validations=[validation])],
)
revised = process_transcript(
transcript,
_glossary(),
_config(tmp_path, grammar_validation_enabled=True),
llm_client=fake_client,
)
assert revised[0].text == "He became visible."
assert fake_client.validation_calls == 1
run_dirs = list((tmp_path / "work").iterdir())
assert len(run_dirs) == 1
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
skipped = diagnostics["skipped_corrections"][0]
assert skipped["stage"] == "grammar"
assert skipped["reason"] == "grammar validation rejected semantic change"
assert skipped["validation_confidence"] == 0.99
assert skipped["validation_reason"] == "This reverses visible to invisible."
metadata = json.loads((run_dirs[0] / "metadata.json").read_text(encoding="utf-8"))
grammar_pass = metadata["stages"][1]["passes"][0]
assert grammar_pass["validation_candidate_count"] == 1
assert grammar_pass["validation_approved_count"] == 0
assert grammar_pass["validation_rejected_count"] == 1
assert grammar_pass["validation_bypassed_count"] == 0
def test_grammar_validation_accepts_meaning_preserving_fix(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Keep in bind."}
]
"""
)
grammar_correction = CorrectionCandidate(
id=1,
original_text="bind",
corrected_text="mind",
confidence=0.95,
)
validation = GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=True,
confidence=0.95,
reason="This fixes the phrase keep in mind.",
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[grammar_correction]),
],
validation_responses=[GrammarValidationSet(validations=[validation])],
)
revised = process_transcript(
transcript,
_glossary(),
_config(tmp_path, grammar_validation_enabled=True),
llm_client=fake_client,
)
assert revised[0].text == "Keep in mind."
assert fake_client.validation_calls == 1
assert list((tmp_path / "work").iterdir()) == []
def test_grammar_validation_bypasses_protected_vocabulary_correction(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "The gestures arrived."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Jesters"
category: faction
summary: "The Jesters are a faction."
"""
)
grammar_correction = CorrectionCandidate(
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[grammar_correction]),
]
)
revised = process_transcript(
transcript,
glossary,
_config(tmp_path, grammar_validation_enabled=True),
llm_client=fake_client,
)
assert revised[0].text == "The Jesters arrived."
assert fake_client.validation_calls == 0
assert list((tmp_path / "work").iterdir()) == []
def test_disabled_grammar_validation_preserves_current_behavior(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "He became visible."}
]
"""
)
grammar_correction = CorrectionCandidate(
id=1,
original_text="visible",
corrected_text="invisible",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[grammar_correction]),
]
)
revised = process_transcript(
transcript,
_glossary(),
_config(tmp_path, grammar_validation_enabled=False),
llm_client=fake_client,
)
assert revised[0].text == "He became invisible."
assert fake_client.validation_calls == 0
assert list((tmp_path / "work").iterdir()) == []
def test_missing_grammar_validation_decision_fails_and_preserves_diagnostics(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "He became visible."}
]
"""
)
grammar_correction = CorrectionCandidate(
id=1,
original_text="visible",
corrected_text="invisible",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[grammar_correction]),
],
validation_responses=[GrammarValidationSet(validations=[])],
)
with pytest.raises(AuditaError):
process_transcript(
transcript,
_glossary(),
_config(tmp_path, grammar_validation_enabled=True),
llm_client=fake_client,
)
run_dirs = list((tmp_path / "work").iterdir())
assert len(run_dirs) == 1
assert (run_dirs[0] / "grammar" / "pass-0001" / "validation-prompt-0000.json").exists()
assert (run_dirs[0] / "grammar" / "pass-0001" / "validation-response-0000.json").exists()
def test_grammar_stage_cannot_reverse_glossary_protected_term(tmp_path):
transcript = parse_source_transcript_json(
"""

30
tests/test_prompts.py
View File

@@ -1,7 +1,11 @@
import json
from audita.chunking import chunk_transcript
from audita.prompts import build_glossary_correction_messages, build_grammar_correction_messages
from audita.prompts import (
build_glossary_correction_messages,
build_grammar_correction_messages,
build_grammar_validation_messages,
)
from audita.schemas import parse_glossary_yaml, parse_transcript_json
@@ -169,3 +173,27 @@ def test_grammar_prompt_uses_simplified_segment_payload():
assert "speaker" not in prompt_segments[0]
assert "start" not in prompt_segments[0]
assert "end" not in prompt_segments[0]
def test_grammar_validation_prompt_rejects_semantic_changes():
messages = build_grammar_validation_messages(
[
{
"correction_index": 0,
"id": 1,
"original_segment_text": "He became visible.",
"corrected_segment_text": "He became invisible.",
"original_text": "visible",
"corrected_text": "invisible",
}
]
)
prompt_text = "\n".join(message["content"] for message in messages)
assert "preserves meaning" in prompt_text
assert "became visible" in prompt_text
assert "became invisible" in prompt_text
assert "reverses the meaning" in prompt_text
assert "spelling and homophone fixes only when" in prompt_text
assert "correction_index" in prompt_text
assert "is_meaning_preserving" in prompt_text

165
tests/test_semantic_validation.py Normal file
View File

@@ -0,0 +1,165 @@
import pytest
from audita.errors import AuditaLLMError
from audita.semantic_validation import select_grammar_validation_candidates
from audita.semantic_validation import filter_with_grammar_validations
from audita.protection import ProtectedVocabulary
from audita.schemas import (
CorrectionCandidate,
GrammarValidationDecision,
GrammarValidationSet,
parse_glossary_yaml,
parse_transcript_json,
)
def _transcript():
return parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "he became visible and then gestures arrived"}
]
"""
)
def _vocabulary():
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Jesters"
category: faction
summary: "The Jesters are a faction."
"""
)
return ProtectedVocabulary.from_glossary(glossary)
def test_protected_vocabulary_correction_bypasses_validation():
correction = CorrectionCandidate(
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
)
candidates, bypassed_count = select_grammar_validation_candidates(
_transcript(),
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
assert candidates == []
assert bypassed_count == 1
def test_capitalization_only_correction_bypasses_validation():
correction = CorrectionCandidate(
id=1,
original_text="he",
corrected_text="He",
confidence=0.95,
)
candidates, bypassed_count = select_grammar_validation_candidates(
_transcript(),
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
assert candidates == []
assert bypassed_count == 1
def test_punctuation_only_correction_bypasses_validation():
correction = CorrectionCandidate(
id=1,
original_text="visible",
corrected_text="visible.",
confidence=0.95,
)
candidates, bypassed_count = select_grammar_validation_candidates(
_transcript(),
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
assert candidates == []
assert bypassed_count == 1
def test_meaning_sensitive_substitution_requires_validation():
correction = CorrectionCandidate(
id=1,
original_text="visible",
corrected_text="invisible",
confidence=0.95,
)
candidates, bypassed_count = select_grammar_validation_candidates(
_transcript(),
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
assert len(candidates) == 1
assert candidates[0].correction_index == 0
assert candidates[0].original_segment_text == "he became visible and then gestures arrived"
assert candidates[0].corrected_segment_text == "he became invisible and then gestures arrived"
assert bypassed_count == 0
def test_validation_rejects_duplicate_and_unknown_decisions():
correction = CorrectionCandidate(
id=1,
original_text="visible",
corrected_text="invisible",
confidence=0.95,
)
candidates, _ = select_grammar_validation_candidates(
_transcript(),
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
decision = GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=True,
confidence=0.95,
reason="Preserves meaning.",
)
with pytest.raises(AuditaLLMError):
filter_with_grammar_validations(
[correction],
candidates,
GrammarValidationSet(validations=[decision, decision]),
confidence_threshold=0.8,
)
with pytest.raises(AuditaLLMError):
filter_with_grammar_validations(
[correction],
candidates,
GrammarValidationSet(
validations=[
GrammarValidationDecision(
correction_index=99,
is_meaning_preserving=True,
confidence=0.95,
reason="Unknown.",
)
]
),
confidence_threshold=0.8,
)