Implemented a custom validator for the glossary module that allows changes from one protected glossary term to another
This commit is contained in:
@@ -5,10 +5,10 @@ from audita.framework.models import CorrectionProposal, ModuleContext
|
|||||||
from audita.framework.proposal_generation import generate_llm_correction_proposals
|
from audita.framework.proposal_generation import generate_llm_correction_proposals
|
||||||
from audita.modules.prompts import build_glossary_proposal_messages
|
from audita.modules.prompts import build_glossary_proposal_messages
|
||||||
from audita.validators import (
|
from audita.validators import (
|
||||||
|
GlossaryStageProtectedGlossaryTermsValidator,
|
||||||
MeaningReversalValidator,
|
MeaningReversalValidator,
|
||||||
NonEmptySegmentValidator,
|
NonEmptySegmentValidator,
|
||||||
ProposalConfidenceValidator,
|
ProposalConfidenceValidator,
|
||||||
ProtectedGlossaryTermsValidator,
|
|
||||||
SpokenFormPlausibilityValidator,
|
SpokenFormPlausibilityValidator,
|
||||||
Validator,
|
Validator,
|
||||||
)
|
)
|
||||||
@@ -21,7 +21,7 @@ class GlossaryModule:
|
|||||||
def validators(self) -> Sequence[Validator]:
|
def validators(self) -> Sequence[Validator]:
|
||||||
return [
|
return [
|
||||||
ProposalConfidenceValidator("proposal_confidence_guard", "glossary_confidence_threshold"),
|
ProposalConfidenceValidator("proposal_confidence_guard", "glossary_confidence_threshold"),
|
||||||
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
|
GlossaryStageProtectedGlossaryTermsValidator("glossary_stage_protected_glossary_guard"),
|
||||||
NonEmptySegmentValidator("non_empty_segment_guard"),
|
NonEmptySegmentValidator("non_empty_segment_guard"),
|
||||||
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
|
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
|
||||||
MeaningReversalValidator("meaning_reversal_review"),
|
MeaningReversalValidator("meaning_reversal_review"),
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
from .base import ValidationContext, ValidationDecision, ValidationResult, Validator
|
from .base import ValidationContext, ValidationDecision, ValidationResult, Validator
|
||||||
from .deterministic import (
|
from .deterministic import (
|
||||||
|
GlossaryStageProtectedGlossaryTermsValidator,
|
||||||
GrammarOnlyValidator,
|
GrammarOnlyValidator,
|
||||||
NonEmptySegmentValidator,
|
NonEmptySegmentValidator,
|
||||||
ProposalConfidenceValidator,
|
ProposalConfidenceValidator,
|
||||||
@@ -15,6 +16,7 @@ __all__ = [
|
|||||||
"Validator",
|
"Validator",
|
||||||
"ProposalConfidenceValidator",
|
"ProposalConfidenceValidator",
|
||||||
"ProtectedGlossaryTermsValidator",
|
"ProtectedGlossaryTermsValidator",
|
||||||
|
"GlossaryStageProtectedGlossaryTermsValidator",
|
||||||
"NonEmptySegmentValidator",
|
"NonEmptySegmentValidator",
|
||||||
"GrammarOnlyValidator",
|
"GrammarOnlyValidator",
|
||||||
"ProtectedVocabulary",
|
"ProtectedVocabulary",
|
||||||
|
|||||||
@@ -53,6 +53,30 @@ class ProtectedGlossaryTermsValidator:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class GlossaryStageProtectedGlossaryTermsValidator:
|
||||||
|
name: str
|
||||||
|
execution_kind: str = "deterministic"
|
||||||
|
|
||||||
|
def validate(self, context: ValidationContext) -> ValidationResult:
|
||||||
|
vocabulary = ProtectedVocabulary.from_glossary(context.glossary)
|
||||||
|
return ValidationResult(
|
||||||
|
validator_name=self.name,
|
||||||
|
execution_kind=self.execution_kind,
|
||||||
|
decisions=[
|
||||||
|
ValidationDecision(
|
||||||
|
proposal_index=proposal.proposal_index,
|
||||||
|
approved=(reason := vocabulary.glossary_stage_violation_reason(
|
||||||
|
proposal.original_text,
|
||||||
|
proposal.corrected_text,
|
||||||
|
)) is None,
|
||||||
|
reason=reason,
|
||||||
|
)
|
||||||
|
for proposal in context.proposals
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class NonEmptySegmentValidator:
|
class NonEmptySegmentValidator:
|
||||||
name: str
|
name: str
|
||||||
|
|||||||
@@ -38,6 +38,15 @@ class ProtectedVocabulary:
|
|||||||
return reason
|
return reason
|
||||||
return self._validate_capitalization_transitions(before_occurrences, after_occurrences)
|
return self._validate_capitalization_transitions(before_occurrences, after_occurrences)
|
||||||
|
|
||||||
|
def glossary_stage_violation_reason(self, before: str, after: str) -> Optional[str]:
|
||||||
|
before_occurrences = self._occurrences_by_identity(before)
|
||||||
|
after_occurrences = self._occurrences_by_identity(after)
|
||||||
|
|
||||||
|
reason = self._validate_glossary_stage_identity_preservation(before_occurrences, after_occurrences)
|
||||||
|
if reason is not None:
|
||||||
|
return reason
|
||||||
|
return self._validate_capitalization_transitions(before_occurrences, after_occurrences)
|
||||||
|
|
||||||
def _occurrences(self, text: str) -> List["_ProtectedOccurrence"]:
|
def _occurrences(self, text: str) -> List["_ProtectedOccurrence"]:
|
||||||
if self.pattern is None:
|
if self.pattern is None:
|
||||||
return []
|
return []
|
||||||
@@ -70,6 +79,17 @@ class ProtectedVocabulary:
|
|||||||
return "correction changes protected glossary term usage"
|
return "correction changes protected glossary term usage"
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def _validate_glossary_stage_identity_preservation(
|
||||||
|
self,
|
||||||
|
before_occurrences: Dict[int, List["_ProtectedOccurrence"]],
|
||||||
|
after_occurrences: Dict[int, List["_ProtectedOccurrence"]],
|
||||||
|
) -> Optional[str]:
|
||||||
|
before_total = sum(len(items) for items in before_occurrences.values())
|
||||||
|
after_total = sum(len(items) for items in after_occurrences.values())
|
||||||
|
if after_total < before_total:
|
||||||
|
return "correction changes protected glossary term usage"
|
||||||
|
return None
|
||||||
|
|
||||||
def _validate_capitalization_transitions(
|
def _validate_capitalization_transitions(
|
||||||
self,
|
self,
|
||||||
before_occurrences: Dict[int, List["_ProtectedOccurrence"]],
|
before_occurrences: Dict[int, List["_ProtectedOccurrence"]],
|
||||||
|
|||||||
@@ -130,7 +130,14 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
|
|||||||
assert (result.run_dir / "normalization" / "summary.json").exists()
|
assert (result.run_dir / "normalization" / "summary.json").exists()
|
||||||
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
|
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
|
||||||
"proposal_confidence_guard",
|
"proposal_confidence_guard",
|
||||||
"protected_glossary_guard",
|
"glossary_stage_protected_glossary_guard",
|
||||||
|
"non_empty_segment_guard",
|
||||||
|
"spoken_form_plausibility_review",
|
||||||
|
"meaning_reversal_review",
|
||||||
|
]
|
||||||
|
assert [validator["name"] for validator in result.report.modules[2].to_dict()["validators"]] == [
|
||||||
|
"proposal_confidence_guard",
|
||||||
|
"glossary_stage_protected_glossary_guard",
|
||||||
"non_empty_segment_guard",
|
"non_empty_segment_guard",
|
||||||
"spoken_form_plausibility_review",
|
"spoken_form_plausibility_review",
|
||||||
"meaning_reversal_review",
|
"meaning_reversal_review",
|
||||||
@@ -184,7 +191,7 @@ def test_default_module_specs_expose_final_validator_order():
|
|||||||
]
|
]
|
||||||
assert [validator.name for validator in specs[0].module.validators()] == [
|
assert [validator.name for validator in specs[0].module.validators()] == [
|
||||||
"proposal_confidence_guard",
|
"proposal_confidence_guard",
|
||||||
"protected_glossary_guard",
|
"glossary_stage_protected_glossary_guard",
|
||||||
"non_empty_segment_guard",
|
"non_empty_segment_guard",
|
||||||
"spoken_form_plausibility_review",
|
"spoken_form_plausibility_review",
|
||||||
"meaning_reversal_review",
|
"meaning_reversal_review",
|
||||||
@@ -198,7 +205,7 @@ def test_default_module_specs_expose_final_validator_order():
|
|||||||
]
|
]
|
||||||
assert [validator.name for validator in specs[2].module.validators()] == [
|
assert [validator.name for validator in specs[2].module.validators()] == [
|
||||||
"proposal_confidence_guard",
|
"proposal_confidence_guard",
|
||||||
"protected_glossary_guard",
|
"glossary_stage_protected_glossary_guard",
|
||||||
"non_empty_segment_guard",
|
"non_empty_segment_guard",
|
||||||
"spoken_form_plausibility_review",
|
"spoken_form_plausibility_review",
|
||||||
"meaning_reversal_review",
|
"meaning_reversal_review",
|
||||||
|
|||||||
@@ -1,6 +1,10 @@
|
|||||||
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
|
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
|
||||||
from audita.framework.models import CorrectionProposal, ModuleRunSpec
|
from audita.framework.models import CorrectionProposal, ModuleRunSpec
|
||||||
from audita.validators import ProtectedGlossaryTermsValidator, ProtectedVocabulary
|
from audita.validators import (
|
||||||
|
GlossaryStageProtectedGlossaryTermsValidator,
|
||||||
|
ProtectedGlossaryTermsValidator,
|
||||||
|
ProtectedVocabulary,
|
||||||
|
)
|
||||||
from audita.validators.base import ValidationContext
|
from audita.validators.base import ValidationContext
|
||||||
|
|
||||||
|
|
||||||
@@ -46,6 +50,22 @@ def test_protected_vocabulary_blocks_replacing_protected_term():
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_protected_vocabulary_glossary_stage_allows_glossary_to_glossary_changes():
|
||||||
|
vocabulary = ProtectedVocabulary.from_glossary(_glossary())
|
||||||
|
|
||||||
|
assert vocabulary.violation_reason("Hrank moves.", "Popov moves.") == "correction changes protected glossary term usage"
|
||||||
|
assert vocabulary.glossary_stage_violation_reason("Hrank moves.", "Popov moves.") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_protected_vocabulary_glossary_stage_still_blocks_glossary_to_nonglossary_changes():
|
||||||
|
vocabulary = ProtectedVocabulary.from_glossary(_glossary())
|
||||||
|
|
||||||
|
assert (
|
||||||
|
vocabulary.glossary_stage_violation_reason("Hrank moves.", "Frank moves.")
|
||||||
|
== "correction changes protected glossary term usage"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_protected_vocabulary_blocks_noncanonical_capitalization():
|
def test_protected_vocabulary_blocks_noncanonical_capitalization():
|
||||||
vocabulary = ProtectedVocabulary.from_glossary(_glossary())
|
vocabulary = ProtectedVocabulary.from_glossary(_glossary())
|
||||||
|
|
||||||
@@ -57,6 +77,10 @@ def test_protected_vocabulary_blocks_noncanonical_capitalization():
|
|||||||
vocabulary.violation_reason("gestures", "jesters")
|
vocabulary.violation_reason("gestures", "jesters")
|
||||||
== "correction changes protected glossary term capitalization"
|
== "correction changes protected glossary term capitalization"
|
||||||
)
|
)
|
||||||
|
assert (
|
||||||
|
vocabulary.glossary_stage_violation_reason("Hrank moves.", "POPOV moves.")
|
||||||
|
== "correction changes protected glossary term capitalization"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_protected_vocabulary_allows_corrections_toward_protected_terms():
|
def test_protected_vocabulary_allows_corrections_toward_protected_terms():
|
||||||
@@ -155,6 +179,54 @@ def test_protected_glossary_terms_validator_returns_proposal_indexed_decisions()
|
|||||||
assert result.decisions[1].approved is True
|
assert result.decisions[1].approved is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_glossary_stage_protected_glossary_terms_validator_allows_glossary_to_glossary_replacement():
|
||||||
|
validator = GlossaryStageProtectedGlossaryTermsValidator("glossary_stage_protected_glossary_guard")
|
||||||
|
transcript = parse_transcript_json(
|
||||||
|
"""
|
||||||
|
[
|
||||||
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hrank moves."},
|
||||||
|
{"id": 2, "speaker": "Eric", "start": 1.0, "end": 2.0, "text": "Hrank moves."}
|
||||||
|
]
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
proposals = [
|
||||||
|
CorrectionProposal(
|
||||||
|
proposal_index=0,
|
||||||
|
module_instance="glossary_1",
|
||||||
|
module_key="glossary",
|
||||||
|
id=1,
|
||||||
|
original_text="Hrank",
|
||||||
|
corrected_text="Popov",
|
||||||
|
confidence=0.9,
|
||||||
|
),
|
||||||
|
CorrectionProposal(
|
||||||
|
proposal_index=1,
|
||||||
|
module_instance="glossary_1",
|
||||||
|
module_key="glossary",
|
||||||
|
id=2,
|
||||||
|
original_text="Hrank",
|
||||||
|
corrected_text="POPOV",
|
||||||
|
confidence=0.9,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
result = validator.validate(
|
||||||
|
ValidationContext(
|
||||||
|
proposals=proposals,
|
||||||
|
transcript=transcript,
|
||||||
|
glossary=_glossary(),
|
||||||
|
config=None, # type: ignore[arg-type]
|
||||||
|
run_spec=ModuleRunSpec(instance_name="glossary_1", module_key="glossary", module=None), # type: ignore[arg-type]
|
||||||
|
run_dir=__import__("pathlib").Path("."),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert [decision.proposal_index for decision in result.decisions] == [0, 1]
|
||||||
|
assert result.decisions[0].approved is True
|
||||||
|
assert result.decisions[1].approved is False
|
||||||
|
assert result.decisions[1].reason == "correction changes protected glossary term capitalization"
|
||||||
|
|
||||||
|
|
||||||
def test_protected_glossary_terms_validator_uses_proposal_span_only():
|
def test_protected_glossary_terms_validator_uses_proposal_span_only():
|
||||||
validator = ProtectedGlossaryTermsValidator("protected_glossary_guard")
|
validator = ProtectedGlossaryTermsValidator("protected_glossary_guard")
|
||||||
glossary = parse_glossary_yaml(
|
glossary = parse_glossary_yaml(
|
||||||
|
|||||||
Reference in New Issue
Block a user