eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Added new deterministic validators to catch early failures

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-25 14:14:43 -05:00
parent 619c615d21
commit a255ff28b7
9 changed files with 286 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/audita/modules/glossary.py
View File

@@ -6,8 +6,10 @@ from audita.framework.proposal_generation import generate_llm_correction_proposa
from audita.modules.prompts import build_glossary_proposal_messages
from audita.validators import (
GlossaryStageProtectedGlossaryTermsValidator,
IdenticalTextValidator,
MeaningReversalValidator,
NonEmptySegmentValidator,
OriginalTextPresentValidator,
ProposalConfidenceValidator,
SpokenFormPlausibilityValidator,
Validator,
@@ -20,6 +22,8 @@ class GlossaryModule:
def validators(self) -> Sequence[Validator]:
return [
IdenticalTextValidator("identical_text_guard"),
OriginalTextPresentValidator("original_text_present_guard"),
ProposalConfidenceValidator("proposal_confidence_guard", "glossary_confidence_threshold"),
GlossaryStageProtectedGlossaryTermsValidator("glossary_stage_protected_glossary_guard"),
NonEmptySegmentValidator("non_empty_segment_guard"),

4
src/audita/modules/grammar.py
View File

@@ -6,8 +6,10 @@ from audita.framework.proposal_generation import generate_llm_correction_proposa
from audita.modules.prompts import build_grammar_proposal_messages
from audita.validators import (
GrammarOnlyValidator,
IdenticalTextValidator,
MeaningReversalValidator,
NonEmptySegmentValidator,
OriginalTextPresentValidator,
ProposalConfidenceValidator,
ProtectedGlossaryTermsValidator,
Validator,
@@ -20,6 +22,8 @@ class GrammarModule:
def validators(self) -> Sequence[Validator]:
return [
IdenticalTextValidator("identical_text_guard"),
OriginalTextPresentValidator("original_text_present_guard"),
ProposalConfidenceValidator("proposal_confidence_guard", "grammar_confidence_threshold"),
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
NonEmptySegmentValidator("non_empty_segment_guard"),

4
src/audita/modules/homophones.py
View File

@@ -5,8 +5,10 @@ from audita.framework.models import CorrectionProposal, ModuleContext
from audita.framework.proposal_generation import generate_llm_correction_proposals
from audita.modules.prompts import build_homophones_proposal_messages
from audita.validators import (
IdenticalTextValidator,
MeaningReversalValidator,
NonEmptySegmentValidator,
OriginalTextPresentValidator,
ProposalConfidenceValidator,
ProtectedGlossaryTermsValidator,
SpokenFormPlausibilityValidator,
@@ -20,6 +22,8 @@ class HomophonesModule:
def validators(self) -> Sequence[Validator]:
return [
IdenticalTextValidator("identical_text_guard"),
OriginalTextPresentValidator("original_text_present_guard"),
ProposalConfidenceValidator("proposal_confidence_guard", "homophones_confidence_threshold"),
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
NonEmptySegmentValidator("non_empty_segment_guard"),

4
src/audita/modules/spoken_word.py
View File

@@ -5,8 +5,10 @@ from audita.framework.models import CorrectionProposal, ModuleContext
from audita.framework.proposal_generation import generate_llm_correction_proposals
from audita.modules.prompts import build_spoken_word_proposal_messages
from audita.validators import (
IdenticalTextValidator,
MeaningReversalValidator,
NonEmptySegmentValidator,
OriginalTextPresentValidator,
ProposalConfidenceValidator,
ProtectedGlossaryTermsValidator,
SpokenWordValidator,
@@ -20,6 +22,8 @@ class SpokenWordModule:
def validators(self) -> Sequence[Validator]:
return [
IdenticalTextValidator("identical_text_guard"),
OriginalTextPresentValidator("original_text_present_guard"),
ProposalConfidenceValidator("proposal_confidence_guard", "spoken_word_confidence_threshold"),
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
NonEmptySegmentValidator("non_empty_segment_guard"),

4
src/audita/validators/__init__.py
View File

@@ -2,7 +2,9 @@ from .base import ValidationContext, ValidationDecision, ValidationResult, Valid
from .deterministic import (
GlossaryStageProtectedGlossaryTermsValidator,
GrammarOnlyValidator,
IdenticalTextValidator,
NonEmptySegmentValidator,
OriginalTextPresentValidator,
ProposalConfidenceValidator,
ProtectedGlossaryTermsValidator,
)
@@ -14,6 +16,8 @@ __all__ = [
"ValidationDecision",
"ValidationResult",
"Validator",
"IdenticalTextValidator",
"OriginalTextPresentValidator",
"ProposalConfidenceValidator",
"ProtectedGlossaryTermsValidator",
"GlossaryStageProtectedGlossaryTermsValidator",

52
src/audita/validators/deterministic.py
View File

@@ -7,6 +7,58 @@ from .base import ValidationContext, ValidationDecision, ValidationResult
from .protection import ProtectedVocabulary
@dataclass(frozen=True)
class IdenticalTextValidator:
name: str
execution_kind: str = "deterministic"
def validate(self, context: ValidationContext) -> ValidationResult:
return ValidationResult(
validator_name=self.name,
execution_kind=self.execution_kind,
decisions=[
ValidationDecision(
proposal_index=proposal.proposal_index,
approved=proposal.original_text != proposal.corrected_text,
reason=(
None
if proposal.original_text != proposal.corrected_text
else "proposal original_text and corrected_text are identical"
),
)
for proposal in context.proposals
],
)
@dataclass(frozen=True)
class OriginalTextPresentValidator:
name: str
execution_kind: str = "deterministic"
def validate(self, context: ValidationContext) -> ValidationResult:
segment_text_by_id = {segment.id: segment.text for segment in context.transcript}
return ValidationResult(
validator_name=self.name,
execution_kind=self.execution_kind,
decisions=[
ValidationDecision(
proposal_index=proposal.proposal_index,
approved=(
(segment_text := segment_text_by_id.get(proposal.id)) is None
or proposal.original_text in segment_text
),
reason=(
None
if (segment_text := segment_text_by_id.get(proposal.id)) is None or proposal.original_text in segment_text
else "proposal original_text does not match segment text"
),
)
for proposal in context.proposals
],
)
@dataclass(frozen=True)
class ProposalConfidenceValidator:
name: str

97
tests/test_llm_validators.py
View File

@@ -5,7 +5,12 @@ from audita.core.config import AuditaConfig
from audita.core.errors import AuditaLLMError
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
from audita.framework.models import CorrectionProposal, ModuleRunSpec
from audita.validators import GrammarOnlyValidator, NonEmptySegmentValidator
from audita.validators import (
GrammarOnlyValidator,
IdenticalTextValidator,
NonEmptySegmentValidator,
OriginalTextPresentValidator,
)
from audita.validators.base import ValidationContext
from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
from audita.validators.prompts import (
@@ -550,6 +555,96 @@ def test_non_empty_segment_validator_allows_punctuation_only_and_unpreviewable_p
]
def test_identical_text_validator_rejects_exact_noops_only(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "hello"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="grammar",
module_key="grammar",
id=1,
original_text="hello",
corrected_text="hello",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="grammar",
module_key="grammar",
id=2,
original_text="hello",
corrected_text="Hello",
confidence=0.95,
),
]
result = IdenticalTextValidator("identical_text_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [
(0, False, "proposal original_text and corrected_text are identical"),
(1, True, None),
]
def test_original_text_present_validator_rejects_missing_spans_and_allows_present_or_unknown_segments(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello hello"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "goodbye"}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="hello",
corrected_text="hi",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="homophones",
module_key="homophones",
id=2,
original_text="rank",
corrected_text="Hrank",
confidence=0.95,
),
CorrectionProposal(
proposal_index=2,
module_instance="homophones",
module_key="homophones",
id=99,
original_text="missing",
corrected_text="present",
confidence=0.95,
),
]
result = OriginalTextPresentValidator("original_text_present_guard").validate(
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [
(0, True, None),
(1, False, "proposal original_text does not match segment text"),
(2, True, None),
]
@pytest.mark.parametrize(
("validator", "payload", "message_fragment"),
[

107
tests/test_module_proposals.py
View File

@@ -1,7 +1,7 @@
from pathlib import Path
from audita.core.chunking import chunk_transcript
from audita.core.config import AuditaConfig
from audita.core.config import AuditaConfig, ConfigOverrides
from audita.core.errors import AuditaLLMError
from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json, parse_transcript_json
from audita.framework.models import ModuleContext, ModuleRunSpec
@@ -418,8 +418,8 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_
]
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
assert result.report.skipped_corrections[0].source == "validator:proposal_confidence_guard"
assert result.report.modules[0].validators[0].rejected_count == 1
assert result.report.modules[0].validators[1].candidate_count == 0
assert result.report.modules[0].validators[2].rejected_count == 1
assert result.report.modules[0].validators[3].candidate_count == 0
def test_process_transcript_result_runs_spoken_word_module_with_full_validator_chain(tmp_path):
@@ -498,6 +498,8 @@ def test_process_transcript_result_runs_spoken_word_module_with_full_validator_c
"spoken_word:meaning_reversal_review",
]
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
@@ -558,7 +560,7 @@ def test_process_transcript_result_rejects_spoken_word_below_threshold_before_ll
assert result.transcript[0].text == "I, uh, I think we should go."
assert [call["stage_name"] for call in client.calls] == ["spoken_word:proposal"]
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
assert result.report.modules[0].validators[1].candidate_count == 0
assert result.report.modules[0].validators[3].candidate_count == 0
def test_process_transcript_result_runs_grammar_module_with_full_validator_chain(tmp_path):
@@ -626,6 +628,8 @@ def test_process_transcript_result_runs_grammar_module_with_full_validator_chain
"grammar:meaning_reversal_review",
]
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
@@ -752,7 +756,7 @@ def test_process_transcript_result_rejects_grammar_below_threshold_before_later_
assert result.transcript[0].text == "hello world"
assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"]
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
assert result.report.modules[0].validators[1].candidate_count == 0
assert result.report.modules[0].validators[3].candidate_count == 0
def test_process_transcript_result_grammar_module_still_rejects_homophone_style_proposals(tmp_path):
@@ -864,11 +868,100 @@ def test_process_transcript_result_rejects_spoken_word_whole_segment_deletion_be
assert result.report.skipped_corrections[0].source == "validator:non_empty_segment_guard"
assert result.report.skipped_corrections[0].reason == "correction would leave the segment empty"
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
"spoken_word_review",
"meaning_reversal_review",
]
assert result.report.modules[0].validators[2].rejected_count == 1
assert result.report.modules[0].validators[3].candidate_count == 0
assert result.report.modules[0].validators[4].rejected_count == 1
assert result.report.modules[0].validators[5].candidate_count == 0
def test_process_transcript_result_rejects_identical_text_before_later_validators(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
]
"""
)
config = AuditaConfig.from_sources(
env={},
overrides=ConfigOverrides(work_dir=tmp_path / "work", work_dir_retention="always"),
)
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "hello",
"corrected_text": "hello",
"confidence": 0.95,
}
]
}
]
)
result = process_transcript_result(
transcript,
_glossary(),
config,
module_keys=["grammar"],
llm_client=client,
)
assert result.transcript[0].text == "hello world"
assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"]
assert result.report.skipped_corrections[0].source == "validator:identical_text_guard"
assert result.report.skipped_corrections[0].reason == "proposal original_text and corrected_text are identical"
assert result.report.modules[0].validators[0].rejected_count == 1
assert result.report.modules[0].validators[1].candidate_count == 0
def test_process_transcript_result_rejects_missing_original_text_before_later_validators(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
]
"""
)
config = AuditaConfig.from_sources(
env={},
overrides=ConfigOverrides(work_dir=tmp_path / "work", work_dir_retention="always"),
)
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "goodbye",
"corrected_text": "farewell",
"confidence": 0.95,
}
]
}
]
)
result = process_transcript_result(
transcript,
_glossary(),
config,
module_keys=["grammar"],
llm_client=client,
)
assert result.transcript[0].text == "hello world"
assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"]
assert result.report.skipped_corrections[0].source == "validator:original_text_present_guard"
assert result.report.skipped_corrections[0].reason == "proposal original_text does not match segment text"
assert result.report.modules[0].validators[0].approved_count == 1
assert result.report.modules[0].validators[1].rejected_count == 1
assert result.report.modules[0].validators[2].candidate_count == 0

18
tests/test_new_pipeline.py
View File

@@ -129,6 +129,8 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
assert (result.run_dir / "report.json").exists()
assert (result.run_dir / "normalization" / "summary.json").exists()
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"glossary_stage_protected_glossary_guard",
"non_empty_segment_guard",
@@ -136,6 +138,8 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
"meaning_reversal_review",
]
assert [validator["name"] for validator in result.report.modules[2].to_dict()["validators"]] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"glossary_stage_protected_glossary_guard",
"non_empty_segment_guard",
@@ -143,6 +147,8 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
"meaning_reversal_review",
]
assert [validator["name"] for validator in result.report.modules[3].to_dict()["validators"]] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
@@ -150,6 +156,8 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
"meaning_reversal_review",
]
assert [validator["name"] for validator in result.report.modules[4].to_dict()["validators"]] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
@@ -190,6 +198,8 @@ def test_default_module_specs_expose_final_validator_order():
"grammar",
]
assert [validator.name for validator in specs[0].module.validators()] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"glossary_stage_protected_glossary_guard",
"non_empty_segment_guard",
@@ -197,6 +207,8 @@ def test_default_module_specs_expose_final_validator_order():
"meaning_reversal_review",
]
assert [validator.name for validator in specs[1].module.validators()] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
@@ -204,6 +216,8 @@ def test_default_module_specs_expose_final_validator_order():
"meaning_reversal_review",
]
assert [validator.name for validator in specs[2].module.validators()] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"glossary_stage_protected_glossary_guard",
"non_empty_segment_guard",
@@ -211,6 +225,8 @@ def test_default_module_specs_expose_final_validator_order():
"meaning_reversal_review",
]
assert [validator.name for validator in specs[3].module.validators()] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",
@@ -218,6 +234,8 @@ def test_default_module_specs_expose_final_validator_order():
"meaning_reversal_review",
]
assert [validator.name for validator in specs[4].module.validators()] == [
"identical_text_guard",
"original_text_present_guard",
"proposal_confidence_guard",
"protected_glossary_guard",
"non_empty_segment_guard",