eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Implemented a second-stage validator for grammar corrections that allows homophone-related changes

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-23 11:25:35 -05:00
parent dee6ae4067
commit c4e2db75f1
13 changed files with 638 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/audita/cli.py
View File

@@ -56,6 +56,11 @@ def _build_parser() -> argparse.ArgumentParser:
type=float,
help="minimum validator confidence required to apply a validated grammar correction",
)
process.add_argument(
"--grammar-spoken-form-validation-confidence-threshold",
type=float,
help="minimum validator confidence required to apply a spoken-form rescue correction",
)
process.add_argument(
"--normalize-max-segment-gap",
type=float,
@@ -94,6 +99,9 @@ def _process(args: argparse.Namespace) -> int:
grammar_max_llm_passes=args.grammar_max_llm_passes,
grammar_validation_enabled=args.grammar_validation_enabled,
grammar_validation_confidence_threshold=args.grammar_validation_confidence_threshold,
grammar_spoken_form_validation_confidence_threshold=(
args.grammar_spoken_form_validation_confidence_threshold
),
normalize_max_segment_gap=args.normalize_max_segment_gap,
normalize_ellipsis_gap=args.normalize_ellipsis_gap,
normalize_max_segment_duration=args.normalize_max_segment_duration,

16
src/audita/config.py
View File

@@ -18,6 +18,7 @@ DEFAULT_GLOSSARY_MAX_LLM_PASSES = 3
DEFAULT_GRAMMAR_MAX_LLM_PASSES = 3
DEFAULT_GRAMMAR_VALIDATION_ENABLED = True
DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD = 0.80
DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD = 0.80
DEFAULT_NORMALIZE_MAX_SEGMENT_GAP = 5.0
DEFAULT_NORMALIZE_ELLIPSIS_GAP = 3.5
DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION = 60.0
@@ -36,6 +37,7 @@ class ConfigOverrides:
grammar_max_llm_passes: Optional[int] = None
grammar_validation_enabled: Optional[bool] = None
grammar_validation_confidence_threshold: Optional[float] = None
grammar_spoken_form_validation_confidence_threshold: Optional[float] = None
normalize_max_segment_gap: Optional[float] = None
normalize_ellipsis_gap: Optional[float] = None
normalize_max_segment_duration: Optional[float] = None
@@ -56,6 +58,9 @@ class AuditaConfig:
grammar_max_llm_passes: int = DEFAULT_GRAMMAR_MAX_LLM_PASSES
grammar_validation_enabled: bool = DEFAULT_GRAMMAR_VALIDATION_ENABLED
grammar_validation_confidence_threshold: float = DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD
grammar_spoken_form_validation_confidence_threshold: float = (
DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD
)
normalize_max_segment_gap: float = DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
normalize_ellipsis_gap: float = DEFAULT_NORMALIZE_ELLIPSIS_GAP
normalize_max_segment_duration: float = DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION
@@ -122,6 +127,12 @@ class AuditaConfig:
DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD,
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD",
)
grammar_spoken_form_validation_confidence_threshold = _select_float(
selected.grammar_spoken_form_validation_confidence_threshold,
source.get("AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD"),
DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD,
"AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD",
)
normalize_max_segment_gap = _select_float(
selected.normalize_max_segment_gap,
source.get("AUDITA_NORMALIZE_MAX_SEGMENT_GAP"),
@@ -160,6 +171,7 @@ class AuditaConfig:
grammar_max_llm_passes=grammar_max_llm_passes,
grammar_validation_enabled=grammar_validation_enabled,
grammar_validation_confidence_threshold=grammar_validation_confidence_threshold,
grammar_spoken_form_validation_confidence_threshold=grammar_spoken_form_validation_confidence_threshold,
normalize_max_segment_gap=normalize_max_segment_gap,
normalize_ellipsis_gap=normalize_ellipsis_gap,
normalize_max_segment_duration=normalize_max_segment_duration,
@@ -192,6 +204,10 @@ class AuditaConfig:
raise AuditaConfigError(
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0."
)
if not 0.0 <= self.grammar_spoken_form_validation_confidence_threshold <= 1.0:
raise AuditaConfigError(
"AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0."
)
if not math.isfinite(self.normalize_max_segment_gap):
raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_GAP must be finite.")
if not math.isfinite(self.normalize_ellipsis_gap):

22
src/audita/llm.py
View File

@@ -3,7 +3,7 @@ from typing import List
from .config import AuditaConfig
from .errors import AuditaLLMError
from .prompts import Message
from .schemas import CorrectionSet, GrammarValidationSet
from .schemas import CorrectionSet, GrammarSpokenFormValidationSet, GrammarValidationSet
class InstructorLLMClient:
@@ -52,6 +52,26 @@ class InstructorLLMClient:
"supports tool calling or structured outputs."
) from exc
def create_grammar_spoken_form_validations(
self,
messages: List[Message],
config: AuditaConfig,
) -> GrammarSpokenFormValidationSet:
model = _normalize_openrouter_model(config.model)
try:
return self._client.chat.completions.create(
model=model,
messages=messages,
response_model=GrammarSpokenFormValidationSet,
max_retries=config.max_retries,
extra_body={"provider": {"require_parameters": True}},
)
except Exception as exc:
raise AuditaLLMError(
"LLM spoken-form grammar validation request failed. Confirm the configured OpenRouter model "
"supports tool calling or structured outputs."
) from exc
def _normalize_openrouter_model(model: str) -> str:
prefix = "openrouter/"

15
src/audita/passes.py
View File

@@ -5,7 +5,13 @@ from typing import List, Protocol
from .chunking import TranscriptSection
from .config import AuditaConfig
from .prompts import build_glossary_correction_messages, build_grammar_correction_messages
from .schemas import CorrectionCandidate, CorrectionSet, Glossary, GrammarValidationSet
from .schemas import (
CorrectionCandidate,
CorrectionSet,
Glossary,
GrammarSpokenFormValidationSet,
GrammarValidationSet,
)
class LLMClient(Protocol):
@@ -15,6 +21,13 @@ class LLMClient(Protocol):
def create_grammar_validations(self, messages: List[dict], config: AuditaConfig) -> GrammarValidationSet:
...
def create_grammar_spoken_form_validations(
self,
messages: List[dict],
config: AuditaConfig,
) -> GrammarSpokenFormValidationSet:
...
class CorrectionPass(Protocol):
def run(

55
src/audita/pipeline.py
View File

@@ -12,10 +12,12 @@ from .corrections import CorrectionGuard, ReplacementMode, SkippedCorrection, ap
from .errors import AuditaError
from .normalization import NormalizationResult, normalize_transcript
from .passes import CorrectionPass, GlossaryCorrectionPass, GrammarCorrectionPass, LLMClient
from .prompts import build_grammar_validation_messages
from .prompts import build_grammar_spoken_form_validation_messages, build_grammar_validation_messages
from .protection import ProtectedVocabulary
from .semantic_validation import (
filter_with_grammar_validations,
filter_with_meaning_preserving_validations,
filter_with_spoken_form_validations,
keep_corrections_with_indexes,
select_grammar_validation_candidates,
)
from .schemas import CorrectionCandidate, Glossary, SourceTranscriptSegment, TranscriptSegment, parse_transcript_json
@@ -266,6 +268,9 @@ def _validate_grammar_corrections(
"validation_approved_count": 0,
"validation_rejected_count": 0,
"validation_bypassed_count": 0,
"spoken_form_validation_candidate_count": 0,
"spoken_form_validation_approved_count": 0,
"spoken_form_validation_rejected_count": 0,
}
if stage.name != "grammar":
return corrections, [], validation_summary
@@ -297,16 +302,47 @@ def _validate_grammar_corrections(
response_path = pass_dir / "validation-response-0000.json"
response_path.write_text(response.model_dump_json(indent=2) + "\n", encoding="utf-8")
result = filter_with_grammar_validations(
corrections,
meaning_result = filter_with_meaning_preserving_validations(
candidates,
response,
config.grammar_validation_confidence_threshold,
)
validation_summary["validation_approved_count"] = result.approved_count
validation_summary["validation_rejected_count"] = result.rejected_count
validation_summary["validation_bypassed_count"] = result.bypassed_count
return result.corrections, result.skipped, validation_summary
validation_summary["validation_approved_count"] = meaning_result.approved_count
validation_summary["validation_rejected_count"] = meaning_result.rejected_count
validation_summary["spoken_form_validation_candidate_count"] = len(meaning_result.rescue_candidates)
candidate_indexes = {candidate.correction_index for candidate in candidates}
allowed_indexes = set(range(len(corrections))) - candidate_indexes
allowed_indexes.update(meaning_result.approved_correction_indexes)
if not meaning_result.rescue_candidates:
return keep_corrections_with_indexes(corrections, allowed_indexes), [], validation_summary
spoken_form_payload = [candidate.to_prompt_payload() for candidate in meaning_result.rescue_candidates]
spoken_form_messages = build_grammar_spoken_form_validation_messages(spoken_form_payload)
spoken_form_prompt_path = pass_dir / "spoken-form-validation-prompt-0000.json"
spoken_form_prompt_path.write_text(
json.dumps(spoken_form_messages, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
spoken_form_response = llm_client.create_grammar_spoken_form_validations(spoken_form_messages, config)
spoken_form_response_path = pass_dir / "spoken-form-validation-response-0000.json"
spoken_form_response_path.write_text(spoken_form_response.model_dump_json(indent=2) + "\n", encoding="utf-8")
spoken_form_result = filter_with_spoken_form_validations(
meaning_result.rescue_candidates,
spoken_form_response,
config.grammar_spoken_form_validation_confidence_threshold,
)
validation_summary["spoken_form_validation_approved_count"] = spoken_form_result.approved_count
validation_summary["spoken_form_validation_rejected_count"] = spoken_form_result.rejected_count
allowed_indexes.update(spoken_form_result.approved_correction_indexes)
return (
keep_corrections_with_indexes(corrections, allowed_indexes),
spoken_form_result.skipped,
validation_summary,
)
def _create_run_dir(work_dir: Path) -> Path:
@@ -331,6 +367,9 @@ def _write_run_metadata(
"grammar_confidence_threshold": config.grammar_confidence_threshold,
"grammar_validation_enabled": config.grammar_validation_enabled,
"grammar_validation_confidence_threshold": config.grammar_validation_confidence_threshold,
"grammar_spoken_form_validation_confidence_threshold": (
config.grammar_spoken_form_validation_confidence_threshold
),
"max_retries": config.max_retries,
"glossary_max_llm_passes": config.glossary_max_llm_passes,
"grammar_max_llm_passes": config.grammar_max_llm_passes,

29
src/audita/prompts.py
View File

@@ -116,7 +116,8 @@ def build_grammar_validation_messages(validation_payload: List[dict]) -> List[Me
system = (
"You are Audita, a conservative semantic validation assistant. "
"Evaluate whether each proposed grammar correction preserves the same spoken content and meaning. "
"Do not judge whether the correction is more polished; judge only whether it changes meaning."
"Do not judge whether the correction is more polished, and do not try to rescue likely homophone or transcription fixes. "
"Judge only whether the corrected text preserves the same written meaning."
)
user = (
"Review these proposed grammar corrections and decide whether each correction preserves meaning.\n\n"
@@ -125,10 +126,34 @@ def build_grammar_validation_messages(validation_payload: List[dict]) -> List[Me
"- Reject corrections that add or remove negation, reverse meaning, introduce antonyms, change names, change quantities, change actions, change who did what, or otherwise substantively alter the speaker's meaning.\n"
"- Reject corrections like changing \"became visible\" to \"became invisible\" because that reverses the meaning.\n"
"- Allow capitalization and punctuation changes when they preserve meaning.\n"
"- Allow spelling and homophone fixes only when the corrected segment preserves the same spoken content.\n"
"- If a correction changes meaning because the original transcript may have used the wrong homophone or a phonetic misspelling, reject it here; that question is handled in a separate spoken-form validation step.\n"
"- Do not use domain knowledge to second-guess protected glossary terms; glossary-protected corrections are excluded from this validation step.\n"
"- Each returned validation must contain only correction_index, is_meaning_preserving, confidence, and reason.\n"
"- confidence must be between 0.0 and 1.0.\n\n"
f"Corrections to validate:\n{payload_json}"
)
return [{"role": "system", "content": system}, {"role": "user", "content": user}]
def build_grammar_spoken_form_validation_messages(validation_payload: List[dict]) -> List[Message]:
payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2)
system = (
"You are Audita, a conservative spoken-form validation assistant. "
"Evaluate whether each rejected grammar correction is still a likely homophone, spoken-form, or transcription fix "
"supported by the local segment context. "
"Approve only corrections that plausibly recover the intended spoken words from a mistaken transcript rendering."
)
user = (
"Review these rejected grammar corrections and decide whether each one is a likely spoken-form correction.\n\n"
"Rules:\n"
"- Return one validation decision for every correction_index in the input.\n"
"- Approve a correction only when the original transcript text is plausibly a mistaken homophone, phonetic rendering, or transcription error, and the corrected text better matches the likely spoken words in context.\n"
"- Allow examples like changing \"dam\" to \"damn\" when the surrounding phrase strongly supports the intended spoken phrase.\n"
"- Reject examples like changing \"became visible\" to \"became invisible\" because that is a semantic reversal, not a likely spoken-form correction.\n"
"- Reject paraphrases, stylistic rewrites, content additions or removals, and any meaning-changing edit that is not clearly explained by a transcription or spoken-form mistake.\n"
"- Do not use domain knowledge to second-guess protected glossary terms; glossary-protected corrections are excluded from this validation step.\n"
"- Each returned validation must contain only correction_index, is_likely_spoken_form_correction, confidence, and reason.\n"
"- confidence must be between 0.0 and 1.0.\n\n"
f"Corrections to validate:\n{payload_json}"
)
return [{"role": "system", "content": system}, {"role": "user", "content": user}]

46
src/audita/schemas.py
View File

@@ -217,6 +217,52 @@ class GrammarValidationSet(BaseModel):
validations: List[GrammarValidationDecision] = Field(default_factory=list)
class GrammarSpokenFormValidationDecision(BaseModel):
model_config = ConfigDict(extra="forbid")
correction_index: int = Field(ge=0)
is_likely_spoken_form_correction: bool
confidence: float = Field(ge=0.0, le=1.0)
reason: StrictStr
@field_validator("correction_index", mode="before")
@classmethod
def require_integer_index(cls, value: Any) -> int:
if isinstance(value, bool) or not isinstance(value, int):
raise ValueError("must be an integer")
return value
@field_validator("is_likely_spoken_form_correction", mode="before")
@classmethod
def require_boolean(cls, value: Any) -> bool:
if not isinstance(value, bool):
raise ValueError("must be a boolean")
return value
@field_validator("confidence", mode="before")
@classmethod
def require_number(cls, value: Any) -> float:
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise ValueError("must be a JSON number")
number = float(value)
if not math.isfinite(number):
raise ValueError("must be finite")
return number
@field_validator("reason")
@classmethod
def require_non_empty_reason(cls, value: str) -> str:
if not value.strip():
raise ValueError("must not be empty")
return value
class GrammarSpokenFormValidationSet(BaseModel):
model_config = ConfigDict(extra="forbid")
validations: List[GrammarSpokenFormValidationDecision] = Field(default_factory=list)
_TRANSCRIPT_ADAPTER = TypeAdapter(List[TranscriptSegment])
_SOURCE_TRANSCRIPT_ADAPTER = TypeAdapter(List[SourceTranscriptSegment])

136
src/audita/semantic_validation.py
View File

@@ -1,11 +1,16 @@
import string
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple
from typing import Any, Dict, Iterable, List, Optional, Tuple
from .corrections import ReplacementMode, SkippedCorrection
from .errors import AuditaLLMError
from .protection import ProtectedVocabulary
from .schemas import CorrectionCandidate, GrammarValidationSet, TranscriptSegment
from .schemas import (
CorrectionCandidate,
GrammarSpokenFormValidationSet,
GrammarValidationSet,
TranscriptSegment,
)
@dataclass(frozen=True)
@@ -27,13 +32,19 @@ class GrammarValidationCandidate:
@dataclass(frozen=True)
class GrammarValidationFilterResult:
corrections: List[CorrectionCandidate]
skipped: List[SkippedCorrection]
candidate_count: int
class GrammarMeaningValidationResult:
approved_correction_indexes: List[int]
rescue_candidates: List[GrammarValidationCandidate]
approved_count: int
rejected_count: int
@dataclass(frozen=True)
class GrammarSpokenFormValidationResult:
approved_correction_indexes: List[int]
skipped: List[SkippedCorrection]
approved_count: int
rejected_count: int
bypassed_count: int
def select_grammar_validation_candidates(
@@ -74,30 +85,58 @@ def select_grammar_validation_candidates(
return candidates, bypassed_count
def filter_with_grammar_validations(
corrections: List[CorrectionCandidate],
def filter_with_meaning_preserving_validations(
candidates: List[GrammarValidationCandidate],
validation_set: GrammarValidationSet,
confidence_threshold: float,
) -> GrammarValidationFilterResult:
candidate_indexes = {candidate.correction_index for candidate in candidates}
decisions_by_index = {}
for decision in validation_set.validations:
if decision.correction_index in decisions_by_index:
raise AuditaLLMError("LLM grammar validation response included duplicate correction_index values.")
if decision.correction_index not in candidate_indexes:
raise AuditaLLMError("LLM grammar validation response included an unknown correction_index.")
decisions_by_index[decision.correction_index] = decision
) -> GrammarMeaningValidationResult:
decisions_by_index = _index_validation_decisions(
validation_set.validations,
candidates,
duplicate_error="LLM grammar validation response included duplicate correction_index values.",
unknown_error="LLM grammar validation response included an unknown correction_index.",
missing_error="LLM grammar validation response omitted correction_index values.",
)
missing_indexes = sorted(candidate_indexes - set(decisions_by_index))
if missing_indexes:
raise AuditaLLMError("LLM grammar validation response omitted correction_index values.")
rejected_by_index: Dict[int, SkippedCorrection] = {}
approved_correction_indexes: List[int] = []
rescue_candidates: List[GrammarValidationCandidate] = []
for candidate in candidates:
decision = decisions_by_index[candidate.correction_index]
if not decision.is_meaning_preserving or decision.confidence < confidence_threshold:
rejected_by_index[candidate.correction_index] = SkippedCorrection(
if decision.is_meaning_preserving and decision.confidence >= confidence_threshold:
approved_correction_indexes.append(candidate.correction_index)
else:
rescue_candidates.append(candidate)
return GrammarMeaningValidationResult(
approved_correction_indexes=approved_correction_indexes,
rescue_candidates=rescue_candidates,
approved_count=len(approved_correction_indexes),
rejected_count=len(rescue_candidates),
)
def filter_with_spoken_form_validations(
candidates: List[GrammarValidationCandidate],
validation_set: GrammarSpokenFormValidationSet,
confidence_threshold: float,
) -> GrammarSpokenFormValidationResult:
decisions_by_index = _index_validation_decisions(
validation_set.validations,
candidates,
duplicate_error="LLM spoken-form validation response included duplicate correction_index values.",
unknown_error="LLM spoken-form validation response included an unknown correction_index.",
missing_error="LLM spoken-form validation response omitted correction_index values.",
)
approved_correction_indexes: List[int] = []
skipped: List[SkippedCorrection] = []
for candidate in candidates:
decision = decisions_by_index[candidate.correction_index]
if decision.is_likely_spoken_form_correction and decision.confidence >= confidence_threshold:
approved_correction_indexes.append(candidate.correction_index)
continue
skipped.append(
SkippedCorrection(
id=candidate.correction.id,
reason="grammar validation rejected semantic change",
original_text=candidate.correction.original_text,
@@ -107,21 +146,24 @@ def filter_with_grammar_validations(
validation_confidence=decision.confidence,
validation_reason=decision.reason,
)
)
filtered_corrections = [
correction for index, correction in enumerate(corrections) if index not in rejected_by_index
]
approved_count = len(candidates) - len(rejected_by_index)
return GrammarValidationFilterResult(
corrections=filtered_corrections,
skipped=[rejected_by_index[index] for index in sorted(rejected_by_index)],
candidate_count=len(candidates),
approved_count=approved_count,
rejected_count=len(rejected_by_index),
bypassed_count=len(corrections) - len(candidates),
return GrammarSpokenFormValidationResult(
approved_correction_indexes=approved_correction_indexes,
skipped=skipped,
approved_count=len(approved_correction_indexes),
rejected_count=len(skipped),
)
def keep_corrections_with_indexes(
corrections: List[CorrectionCandidate],
allowed_indexes: Iterable[int],
) -> List[CorrectionCandidate]:
allowed_index_set = set(allowed_indexes)
return [correction for index, correction in enumerate(corrections) if index in allowed_index_set]
def is_capitalization_or_punctuation_only(original_text: str, corrected_text: str) -> bool:
return _semantic_key(original_text) == _semantic_key(corrected_text)
@@ -152,6 +194,28 @@ def _preview_corrected_segment_text(
return segment.text.replace(correction.original_text, correction.corrected_text)
def _index_validation_decisions(
decisions: Iterable[Any],
candidates: List[GrammarValidationCandidate],
duplicate_error: str,
unknown_error: str,
missing_error: str,
) -> Dict[int, Any]:
candidate_indexes = {candidate.correction_index for candidate in candidates}
decisions_by_index: Dict[int, Any] = {}
for decision in decisions:
if decision.correction_index in decisions_by_index:
raise AuditaLLMError(duplicate_error)
if decision.correction_index not in candidate_indexes:
raise AuditaLLMError(unknown_error)
decisions_by_index[decision.correction_index] = decision
missing_indexes = sorted(candidate_indexes - set(decisions_by_index))
if missing_indexes:
raise AuditaLLMError(missing_error)
return decisions_by_index
_PUNCTUATION = set(string.punctuation) | {"—", "–", "…", "“", "”", "‘", "’"}

1
tests/test_cli.py
View File

@@ -23,6 +23,7 @@ def test_process_help_includes_glossary_pass_flag(capsys):
assert "--grammar-confidence-threshold" in output
assert "--grammar-validation-enabled" in output
assert "--grammar-validation-confidence-threshold" in output
assert "--grammar-spoken-form-validation-confidence-threshold" in output
assert "--normalize-max-segment-gap" in output
assert "--normalize-ellipsis-gap" in output
assert "--normalize-max-segment-duration" in output

18
tests/test_config.py
View File

@@ -8,6 +8,7 @@ from audita.config import (
DEFAULT_GLOSSARY_MAX_LLM_PASSES,
DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD,
DEFAULT_GRAMMAR_MAX_LLM_PASSES,
DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD,
DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD,
DEFAULT_GRAMMAR_VALIDATION_ENABLED,
DEFAULT_MAX_RETRIES,
@@ -36,6 +37,11 @@ def test_config_uses_defaults_with_api_key():
assert config.grammar_validation_enabled is True
assert config.grammar_validation_confidence_threshold == DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD
assert config.grammar_validation_confidence_threshold == 0.8
assert (
config.grammar_spoken_form_validation_confidence_threshold
== DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD
)
assert config.grammar_spoken_form_validation_confidence_threshold == 0.8
assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
assert config.normalize_max_segment_gap == 5.0
assert config.normalize_ellipsis_gap == DEFAULT_NORMALIZE_ELLIPSIS_GAP
@@ -58,6 +64,7 @@ def test_config_env_overrides_defaults():
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "4",
"AUDITA_GRAMMAR_VALIDATION_ENABLED": "false",
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "0.91",
"AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD": "0.87",
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "4.5",
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "1.5",
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0",
@@ -74,6 +81,7 @@ def test_config_env_overrides_defaults():
assert config.grammar_max_llm_passes == 4
assert config.grammar_validation_enabled is False
assert config.grammar_validation_confidence_threshold == 0.91
assert config.grammar_spoken_form_validation_confidence_threshold == 0.87
assert config.normalize_max_segment_gap == 4.5
assert config.normalize_ellipsis_gap == 1.5
assert config.normalize_max_segment_duration == 45.0
@@ -91,6 +99,7 @@ def test_config_cli_overrides_env():
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "6",
"AUDITA_GRAMMAR_VALIDATION_ENABLED": "false",
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "0.91",
"AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD": "0.87",
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "4.5",
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "1.5",
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0",
@@ -106,6 +115,7 @@ def test_config_cli_overrides_env():
grammar_max_llm_passes=3,
grammar_validation_enabled=True,
grammar_validation_confidence_threshold=0.75,
grammar_spoken_form_validation_confidence_threshold=0.72,
normalize_max_segment_gap=3.0,
normalize_ellipsis_gap=1.0,
normalize_max_segment_duration=30.0,
@@ -122,6 +132,7 @@ def test_config_cli_overrides_env():
assert config.grammar_max_llm_passes == 3
assert config.grammar_validation_enabled is True
assert config.grammar_validation_confidence_threshold == 0.75
assert config.grammar_spoken_form_validation_confidence_threshold == 0.72
assert config.normalize_max_segment_gap == 3.0
assert config.normalize_ellipsis_gap == 1.0
assert config.normalize_max_segment_duration == 30.0
@@ -168,6 +179,13 @@ def test_config_rejects_invalid_stage_thresholds():
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "1.1"}
)
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={
"OPENROUTER_API_KEY": "key",
"AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD": "-0.1",
}
)
def test_config_rejects_invalid_grammar_validation_enabled():

135
tests/test_pipeline.py
View File

@@ -7,6 +7,8 @@ from audita.errors import AuditaError
from audita.pipeline import process_transcript
from audita.schemas import (
CorrectionCandidate,
GrammarSpokenFormValidationDecision,
GrammarSpokenFormValidationSet,
CorrectionSet,
GrammarValidationDecision,
GrammarValidationSet,
@@ -16,13 +18,16 @@ from audita.schemas import (
class FakeLLMClient:
def __init__(self, responses, validation_responses=None):
def __init__(self, responses, validation_responses=None, spoken_form_validation_responses=None):
self.responses = list(responses)
self.validation_responses = list(validation_responses or [])
self.spoken_form_validation_responses = list(spoken_form_validation_responses or [])
self.calls = 0
self.validation_calls = 0
self.spoken_form_validation_calls = 0
self.messages = []
self.validation_messages = []
self.spoken_form_validation_messages = []
def create_corrections(self, messages, config):
self.calls += 1
@@ -36,6 +41,13 @@ class FakeLLMClient:
raise AssertionError("Unexpected grammar validation request.")
return self.validation_responses.pop(0)
def create_grammar_spoken_form_validations(self, messages, config):
self.spoken_form_validation_calls += 1
self.spoken_form_validation_messages.append(messages)
if not self.spoken_form_validation_responses:
raise AssertionError("Unexpected spoken-form validation request.")
return self.spoken_form_validation_responses.pop(0)
def _config(
tmp_path,
@@ -43,6 +55,7 @@ def _config(
grammar_max_llm_passes=3,
grammar_validation_enabled=False,
grammar_validation_confidence_threshold=0.8,
grammar_spoken_form_validation_confidence_threshold=0.8,
):
return AuditaConfig(
api_key="key",
@@ -54,6 +67,7 @@ def _config(
grammar_max_llm_passes=grammar_max_llm_passes,
grammar_validation_enabled=grammar_validation_enabled,
grammar_validation_confidence_threshold=grammar_validation_confidence_threshold,
grammar_spoken_form_validation_confidence_threshold=grammar_spoken_form_validation_confidence_threshold,
work_dir=tmp_path / "work",
)
@@ -698,6 +712,7 @@ def test_pipeline_writes_stage_metadata_for_unresolved_retries(tmp_path):
assert metadata["grammar_confidence_threshold"] == 0.8
assert metadata["grammar_validation_enabled"] is False
assert metadata["grammar_validation_confidence_threshold"] == 0.8
assert metadata["grammar_spoken_form_validation_confidence_threshold"] == 0.8
assert [item["stage"] for item in metadata["stages"]] == ["glossary", "grammar"]
assert [item["pass_number"] for item in metadata["stages"][0]["passes"]] == [1, 2]
assert metadata["stages"][0]["passes"][0]["retry_segment_count"] == 1
@@ -766,12 +781,21 @@ def test_grammar_validation_rejects_semantic_change(tmp_path):
confidence=0.99,
reason="This reverses visible to invisible.",
)
spoken_form_validation = GrammarSpokenFormValidationDecision(
correction_index=0,
is_likely_spoken_form_correction=False,
confidence=0.98,
reason="This is a semantic reversal, not a likely spoken-form correction.",
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[grammar_correction]),
],
validation_responses=[GrammarValidationSet(validations=[validation])],
spoken_form_validation_responses=[
GrammarSpokenFormValidationSet(validations=[spoken_form_validation])
],
)
revised = process_transcript(
@@ -783,20 +807,24 @@ def test_grammar_validation_rejects_semantic_change(tmp_path):
assert revised[0].text == "He became visible."
assert fake_client.validation_calls == 1
assert fake_client.spoken_form_validation_calls == 1
run_dirs = list((tmp_path / "work").iterdir())
assert len(run_dirs) == 1
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
skipped = diagnostics["skipped_corrections"][0]
assert skipped["stage"] == "grammar"
assert skipped["reason"] == "grammar validation rejected semantic change"
assert skipped["validation_confidence"] == 0.99
assert skipped["validation_reason"] == "This reverses visible to invisible."
assert skipped["validation_confidence"] == 0.98
assert skipped["validation_reason"] == "This is a semantic reversal, not a likely spoken-form correction."
metadata = json.loads((run_dirs[0] / "metadata.json").read_text(encoding="utf-8"))
grammar_pass = metadata["stages"][1]["passes"][0]
assert grammar_pass["validation_candidate_count"] == 1
assert grammar_pass["validation_approved_count"] == 0
assert grammar_pass["validation_rejected_count"] == 1
assert grammar_pass["validation_bypassed_count"] == 0
assert grammar_pass["spoken_form_validation_candidate_count"] == 1
assert grammar_pass["spoken_form_validation_approved_count"] == 0
assert grammar_pass["spoken_form_validation_rejected_count"] == 1
def test_grammar_validation_accepts_meaning_preserving_fix(tmp_path):
@@ -836,6 +864,57 @@ def test_grammar_validation_accepts_meaning_preserving_fix(tmp_path):
assert revised[0].text == "Keep in mind."
assert fake_client.validation_calls == 1
assert fake_client.spoken_form_validation_calls == 0
assert list((tmp_path / "work").iterdir()) == []
def test_grammar_validation_rescues_likely_spoken_form_fix(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."}
]
"""
)
grammar_correction = CorrectionCandidate(
id=1,
original_text="dam",
corrected_text="damn",
confidence=0.95,
)
meaning_validation = GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=False,
confidence=0.98,
reason="Written meaning changes from a barrier to a curse word.",
)
spoken_form_validation = GrammarSpokenFormValidationDecision(
correction_index=0,
is_likely_spoken_form_correction=True,
confidence=0.97,
reason="The phrase strongly suggests the intended spoken word was the expletive.",
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[grammar_correction]),
],
validation_responses=[GrammarValidationSet(validations=[meaning_validation])],
spoken_form_validation_responses=[
GrammarSpokenFormValidationSet(validations=[spoken_form_validation])
],
)
revised = process_transcript(
transcript,
_glossary(),
_config(tmp_path, grammar_validation_enabled=True),
llm_client=fake_client,
)
assert revised[0].text == "ChatGPT still can't really do that with a damn."
assert fake_client.validation_calls == 1
assert fake_client.spoken_form_validation_calls == 1
assert list((tmp_path / "work").iterdir()) == []
@@ -877,6 +956,7 @@ def test_grammar_validation_bypasses_protected_vocabulary_correction(tmp_path):
assert revised[0].text == "The Jesters arrived."
assert fake_client.validation_calls == 0
assert fake_client.spoken_form_validation_calls == 0
assert list((tmp_path / "work").iterdir()) == []
@@ -910,6 +990,7 @@ def test_disabled_grammar_validation_preserves_current_behavior(tmp_path):
assert revised[0].text == "He became invisible."
assert fake_client.validation_calls == 0
assert fake_client.spoken_form_validation_calls == 0
assert list((tmp_path / "work").iterdir()) == []
@@ -949,6 +1030,54 @@ def test_missing_grammar_validation_decision_fails_and_preserves_diagnostics(tmp
assert (run_dirs[0] / "grammar" / "pass-0001" / "validation-response-0000.json").exists()
def test_missing_spoken_form_validation_decision_fails_and_preserves_diagnostics(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."}
]
"""
)
grammar_correction = CorrectionCandidate(
id=1,
original_text="dam",
corrected_text="damn",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[grammar_correction]),
],
validation_responses=[
GrammarValidationSet(
validations=[
GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=False,
confidence=0.99,
reason="Written meaning changes.",
)
]
)
],
spoken_form_validation_responses=[GrammarSpokenFormValidationSet(validations=[])],
)
with pytest.raises(AuditaError):
process_transcript(
transcript,
_glossary(),
_config(tmp_path, grammar_validation_enabled=True),
llm_client=fake_client,
)
run_dirs = list((tmp_path / "work").iterdir())
assert len(run_dirs) == 1
assert (run_dirs[0] / "grammar" / "pass-0001" / "spoken-form-validation-prompt-0000.json").exists()
assert (run_dirs[0] / "grammar" / "pass-0001" / "spoken-form-validation-response-0000.json").exists()
def test_grammar_stage_cannot_reverse_glossary_protected_term(tmp_path):
transcript = parse_source_transcript_json(
"""

25
tests/test_prompts.py
View File

@@ -4,6 +4,7 @@ from audita.chunking import chunk_transcript
from audita.prompts import (
build_glossary_correction_messages,
build_grammar_correction_messages,
build_grammar_spoken_form_validation_messages,
build_grammar_validation_messages,
)
from audita.schemas import parse_glossary_yaml, parse_transcript_json
@@ -194,6 +195,28 @@ def test_grammar_validation_prompt_rejects_semantic_changes():
assert "became visible" in prompt_text
assert "became invisible" in prompt_text
assert "reverses the meaning" in prompt_text
assert "spelling and homophone fixes only when" in prompt_text
assert "do not try to rescue likely homophone or transcription fixes" in prompt_text
assert "handled in a separate spoken-form validation step" in prompt_text
assert "correction_index" in prompt_text
assert "is_meaning_preserving" in prompt_text
def test_grammar_spoken_form_validation_prompt_allows_homophone_rescue():
messages = build_grammar_spoken_form_validation_messages(
[
{
"correction_index": 0,
"id": 1,
"original_segment_text": "ChatGPT still can't really do that with a dam.",
"corrected_segment_text": "ChatGPT still can't really do that with a damn.",
"original_text": "dam",
"corrected_text": "damn",
}
]
)
prompt_text = "\n".join(message["content"] for message in messages)
assert "likely homophone, spoken-form, or transcription fix" in prompt_text
assert '"dam" to "damn"' in prompt_text
assert '"became visible" to "became invisible"' in prompt_text
assert "is_likely_spoken_form_correction" in prompt_text

197
tests/test_semantic_validation.py
View File

@@ -1,16 +1,22 @@
import pytest
from audita.errors import AuditaLLMError
from audita.semantic_validation import select_grammar_validation_candidates
from audita.semantic_validation import filter_with_grammar_validations
from audita.protection import ProtectedVocabulary
from audita.schemas import (
CorrectionCandidate,
GrammarSpokenFormValidationDecision,
GrammarSpokenFormValidationSet,
GrammarValidationDecision,
GrammarValidationSet,
parse_glossary_yaml,
parse_transcript_json,
)
from audita.semantic_validation import (
filter_with_meaning_preserving_validations,
filter_with_spoken_form_validations,
keep_corrections_with_indexes,
select_grammar_validation_candidates,
)
def _transcript():
@@ -118,7 +124,106 @@ def test_meaning_sensitive_substitution_requires_validation():
assert bypassed_count == 0
def test_validation_rejects_duplicate_and_unknown_decisions():
def test_meaning_preserving_validation_approves_without_rescue():
correction = CorrectionCandidate(
id=1,
original_text="bind",
corrected_text="mind",
confidence=0.95,
)
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Keep in bind."}
]
"""
)
candidates, _ = select_grammar_validation_candidates(
transcript,
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
result = filter_with_meaning_preserving_validations(
candidates,
GrammarValidationSet(
validations=[
GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=True,
confidence=0.95,
reason="This preserves the intended meaning.",
)
]
),
confidence_threshold=0.8,
)
assert result.approved_correction_indexes == [0]
assert result.rescue_candidates == []
assert result.approved_count == 1
assert result.rejected_count == 0
def test_spoken_form_validation_can_rescue_homophone_fix():
correction = CorrectionCandidate(
id=1,
original_text="dam",
corrected_text="damn",
confidence=0.95,
)
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."}
]
"""
)
candidates, _ = select_grammar_validation_candidates(
transcript,
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
meaning_result = filter_with_meaning_preserving_validations(
candidates,
GrammarValidationSet(
validations=[
GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=False,
confidence=0.99,
reason="Written meaning changes from a barrier to a curse word.",
)
]
),
confidence_threshold=0.8,
)
spoken_form_result = filter_with_spoken_form_validations(
meaning_result.rescue_candidates,
GrammarSpokenFormValidationSet(
validations=[
GrammarSpokenFormValidationDecision(
correction_index=0,
is_likely_spoken_form_correction=True,
confidence=0.95,
reason="The surrounding phrase strongly supports the intended spoken phrase with a curse word.",
)
]
),
confidence_threshold=0.8,
)
kept = keep_corrections_with_indexes([correction], spoken_form_result.approved_correction_indexes)
assert meaning_result.approved_correction_indexes == []
assert kept == [correction]
assert spoken_form_result.skipped == []
def test_spoken_form_validation_rejects_non_homophone_semantic_change():
correction = CorrectionCandidate(
id=1,
original_text="visible",
@@ -132,30 +237,90 @@ def test_validation_rejects_duplicate_and_unknown_decisions():
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
decision = GrammarValidationDecision(
meaning_result = filter_with_meaning_preserving_validations(
candidates,
GrammarValidationSet(
validations=[
GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=False,
confidence=0.99,
reason="This reverses visible to invisible.",
)
]
),
confidence_threshold=0.8,
)
spoken_form_result = filter_with_spoken_form_validations(
meaning_result.rescue_candidates,
GrammarSpokenFormValidationSet(
validations=[
GrammarSpokenFormValidationDecision(
correction_index=0,
is_likely_spoken_form_correction=False,
confidence=0.99,
reason="This is a semantic reversal, not a likely spoken-form transcription error.",
)
]
),
confidence_threshold=0.8,
)
assert spoken_form_result.approved_correction_indexes == []
assert spoken_form_result.rejected_count == 1
assert spoken_form_result.skipped[0].reason == "grammar validation rejected semantic change"
assert spoken_form_result.skipped[0].validation_reason == (
"This is a semantic reversal, not a likely spoken-form transcription error."
)
def test_validation_rejects_duplicate_unknown_and_missing_decisions():
correction = CorrectionCandidate(
id=1,
original_text="visible",
corrected_text="invisible",
confidence=0.95,
)
candidates, _ = select_grammar_validation_candidates(
_transcript(),
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
meaning_decision = GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=True,
confidence=0.95,
reason="Preserves meaning.",
)
spoken_form_decision = GrammarSpokenFormValidationDecision(
correction_index=0,
is_likely_spoken_form_correction=True,
confidence=0.95,
reason="Likely spoken-form correction.",
)
with pytest.raises(AuditaLLMError):
filter_with_grammar_validations(
[correction],
filter_with_meaning_preserving_validations(
candidates,
GrammarValidationSet(validations=[decision, decision]),
GrammarValidationSet(validations=[meaning_decision, meaning_decision]),
confidence_threshold=0.8,
)
with pytest.raises(AuditaLLMError):
filter_with_grammar_validations(
[correction],
filter_with_meaning_preserving_validations(
candidates,
GrammarValidationSet(
GrammarValidationSet(validations=[]),
confidence_threshold=0.8,
)
with pytest.raises(AuditaLLMError):
filter_with_spoken_form_validations(
candidates,
GrammarSpokenFormValidationSet(
validations=[
GrammarValidationDecision(
GrammarSpokenFormValidationDecision(
correction_index=99,
is_meaning_preserving=True,
is_likely_spoken_form_correction=True,
confidence=0.95,
reason="Unknown.",
)
@@ -163,3 +328,9 @@ def test_validation_rejects_duplicate_and_unknown_decisions():
),
confidence_threshold=0.8,
)
with pytest.raises(AuditaLLMError):
filter_with_spoken_form_validations(
candidates,
GrammarSpokenFormValidationSet(validations=[spoken_form_decision, spoken_form_decision]),
confidence_threshold=0.8,
)