diff --git a/src/audita/cli.py b/src/audita/cli.py index d0d4e8a..469c0e2 100644 --- a/src/audita/cli.py +++ b/src/audita/cli.py @@ -56,6 +56,11 @@ def _build_parser() -> argparse.ArgumentParser: type=float, help="minimum validator confidence required to apply a validated grammar correction", ) + process.add_argument( + "--grammar-spoken-form-validation-confidence-threshold", + type=float, + help="minimum validator confidence required to apply a spoken-form rescue correction", + ) process.add_argument( "--normalize-max-segment-gap", type=float, @@ -94,6 +99,9 @@ def _process(args: argparse.Namespace) -> int: grammar_max_llm_passes=args.grammar_max_llm_passes, grammar_validation_enabled=args.grammar_validation_enabled, grammar_validation_confidence_threshold=args.grammar_validation_confidence_threshold, + grammar_spoken_form_validation_confidence_threshold=( + args.grammar_spoken_form_validation_confidence_threshold + ), normalize_max_segment_gap=args.normalize_max_segment_gap, normalize_ellipsis_gap=args.normalize_ellipsis_gap, normalize_max_segment_duration=args.normalize_max_segment_duration, diff --git a/src/audita/config.py b/src/audita/config.py index c92889f..9e9ba34 100644 --- a/src/audita/config.py +++ b/src/audita/config.py @@ -18,6 +18,7 @@ DEFAULT_GLOSSARY_MAX_LLM_PASSES = 3 DEFAULT_GRAMMAR_MAX_LLM_PASSES = 3 DEFAULT_GRAMMAR_VALIDATION_ENABLED = True DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD = 0.80 +DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD = 0.80 DEFAULT_NORMALIZE_MAX_SEGMENT_GAP = 5.0 DEFAULT_NORMALIZE_ELLIPSIS_GAP = 3.5 DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION = 60.0 @@ -36,6 +37,7 @@ class ConfigOverrides: grammar_max_llm_passes: Optional[int] = None grammar_validation_enabled: Optional[bool] = None grammar_validation_confidence_threshold: Optional[float] = None + grammar_spoken_form_validation_confidence_threshold: Optional[float] = None normalize_max_segment_gap: Optional[float] = None normalize_ellipsis_gap: Optional[float] = None normalize_max_segment_duration: Optional[float] = None @@ -56,6 +58,9 @@ class AuditaConfig: grammar_max_llm_passes: int = DEFAULT_GRAMMAR_MAX_LLM_PASSES grammar_validation_enabled: bool = DEFAULT_GRAMMAR_VALIDATION_ENABLED grammar_validation_confidence_threshold: float = DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD + grammar_spoken_form_validation_confidence_threshold: float = ( + DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD + ) normalize_max_segment_gap: float = DEFAULT_NORMALIZE_MAX_SEGMENT_GAP normalize_ellipsis_gap: float = DEFAULT_NORMALIZE_ELLIPSIS_GAP normalize_max_segment_duration: float = DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION @@ -122,6 +127,12 @@ class AuditaConfig: DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD, "AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD", ) + grammar_spoken_form_validation_confidence_threshold = _select_float( + selected.grammar_spoken_form_validation_confidence_threshold, + source.get("AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD"), + DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD, + "AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD", + ) normalize_max_segment_gap = _select_float( selected.normalize_max_segment_gap, source.get("AUDITA_NORMALIZE_MAX_SEGMENT_GAP"), @@ -160,6 +171,7 @@ class AuditaConfig: grammar_max_llm_passes=grammar_max_llm_passes, grammar_validation_enabled=grammar_validation_enabled, grammar_validation_confidence_threshold=grammar_validation_confidence_threshold, + grammar_spoken_form_validation_confidence_threshold=grammar_spoken_form_validation_confidence_threshold, normalize_max_segment_gap=normalize_max_segment_gap, normalize_ellipsis_gap=normalize_ellipsis_gap, normalize_max_segment_duration=normalize_max_segment_duration, @@ -192,6 +204,10 @@ class AuditaConfig: raise AuditaConfigError( "AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0." ) + if not 0.0 <= self.grammar_spoken_form_validation_confidence_threshold <= 1.0: + raise AuditaConfigError( + "AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0." + ) if not math.isfinite(self.normalize_max_segment_gap): raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_GAP must be finite.") if not math.isfinite(self.normalize_ellipsis_gap): diff --git a/src/audita/llm.py b/src/audita/llm.py index 55f6b21..3f8fd2a 100644 --- a/src/audita/llm.py +++ b/src/audita/llm.py @@ -3,7 +3,7 @@ from typing import List from .config import AuditaConfig from .errors import AuditaLLMError from .prompts import Message -from .schemas import CorrectionSet, GrammarValidationSet +from .schemas import CorrectionSet, GrammarSpokenFormValidationSet, GrammarValidationSet class InstructorLLMClient: @@ -52,6 +52,26 @@ class InstructorLLMClient: "supports tool calling or structured outputs." ) from exc + def create_grammar_spoken_form_validations( + self, + messages: List[Message], + config: AuditaConfig, + ) -> GrammarSpokenFormValidationSet: + model = _normalize_openrouter_model(config.model) + try: + return self._client.chat.completions.create( + model=model, + messages=messages, + response_model=GrammarSpokenFormValidationSet, + max_retries=config.max_retries, + extra_body={"provider": {"require_parameters": True}}, + ) + except Exception as exc: + raise AuditaLLMError( + "LLM spoken-form grammar validation request failed. Confirm the configured OpenRouter model " + "supports tool calling or structured outputs." + ) from exc + def _normalize_openrouter_model(model: str) -> str: prefix = "openrouter/" diff --git a/src/audita/passes.py b/src/audita/passes.py index 3bd830f..270a40f 100644 --- a/src/audita/passes.py +++ b/src/audita/passes.py @@ -5,7 +5,13 @@ from typing import List, Protocol from .chunking import TranscriptSection from .config import AuditaConfig from .prompts import build_glossary_correction_messages, build_grammar_correction_messages -from .schemas import CorrectionCandidate, CorrectionSet, Glossary, GrammarValidationSet +from .schemas import ( + CorrectionCandidate, + CorrectionSet, + Glossary, + GrammarSpokenFormValidationSet, + GrammarValidationSet, +) class LLMClient(Protocol): @@ -15,6 +21,13 @@ class LLMClient(Protocol): def create_grammar_validations(self, messages: List[dict], config: AuditaConfig) -> GrammarValidationSet: ... + def create_grammar_spoken_form_validations( + self, + messages: List[dict], + config: AuditaConfig, + ) -> GrammarSpokenFormValidationSet: + ... + class CorrectionPass(Protocol): def run( diff --git a/src/audita/pipeline.py b/src/audita/pipeline.py index 45365fa..88e9ac9 100644 --- a/src/audita/pipeline.py +++ b/src/audita/pipeline.py @@ -12,10 +12,12 @@ from .corrections import CorrectionGuard, ReplacementMode, SkippedCorrection, ap from .errors import AuditaError from .normalization import NormalizationResult, normalize_transcript from .passes import CorrectionPass, GlossaryCorrectionPass, GrammarCorrectionPass, LLMClient -from .prompts import build_grammar_validation_messages +from .prompts import build_grammar_spoken_form_validation_messages, build_grammar_validation_messages from .protection import ProtectedVocabulary from .semantic_validation import ( - filter_with_grammar_validations, + filter_with_meaning_preserving_validations, + filter_with_spoken_form_validations, + keep_corrections_with_indexes, select_grammar_validation_candidates, ) from .schemas import CorrectionCandidate, Glossary, SourceTranscriptSegment, TranscriptSegment, parse_transcript_json @@ -266,6 +268,9 @@ def _validate_grammar_corrections( "validation_approved_count": 0, "validation_rejected_count": 0, "validation_bypassed_count": 0, + "spoken_form_validation_candidate_count": 0, + "spoken_form_validation_approved_count": 0, + "spoken_form_validation_rejected_count": 0, } if stage.name != "grammar": return corrections, [], validation_summary @@ -297,16 +302,47 @@ def _validate_grammar_corrections( response_path = pass_dir / "validation-response-0000.json" response_path.write_text(response.model_dump_json(indent=2) + "\n", encoding="utf-8") - result = filter_with_grammar_validations( - corrections, + meaning_result = filter_with_meaning_preserving_validations( candidates, response, config.grammar_validation_confidence_threshold, ) - validation_summary["validation_approved_count"] = result.approved_count - validation_summary["validation_rejected_count"] = result.rejected_count - validation_summary["validation_bypassed_count"] = result.bypassed_count - return result.corrections, result.skipped, validation_summary + validation_summary["validation_approved_count"] = meaning_result.approved_count + validation_summary["validation_rejected_count"] = meaning_result.rejected_count + validation_summary["spoken_form_validation_candidate_count"] = len(meaning_result.rescue_candidates) + + candidate_indexes = {candidate.correction_index for candidate in candidates} + allowed_indexes = set(range(len(corrections))) - candidate_indexes + allowed_indexes.update(meaning_result.approved_correction_indexes) + + if not meaning_result.rescue_candidates: + return keep_corrections_with_indexes(corrections, allowed_indexes), [], validation_summary + + spoken_form_payload = [candidate.to_prompt_payload() for candidate in meaning_result.rescue_candidates] + spoken_form_messages = build_grammar_spoken_form_validation_messages(spoken_form_payload) + spoken_form_prompt_path = pass_dir / "spoken-form-validation-prompt-0000.json" + spoken_form_prompt_path.write_text( + json.dumps(spoken_form_messages, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + spoken_form_response = llm_client.create_grammar_spoken_form_validations(spoken_form_messages, config) + spoken_form_response_path = pass_dir / "spoken-form-validation-response-0000.json" + spoken_form_response_path.write_text(spoken_form_response.model_dump_json(indent=2) + "\n", encoding="utf-8") + + spoken_form_result = filter_with_spoken_form_validations( + meaning_result.rescue_candidates, + spoken_form_response, + config.grammar_spoken_form_validation_confidence_threshold, + ) + validation_summary["spoken_form_validation_approved_count"] = spoken_form_result.approved_count + validation_summary["spoken_form_validation_rejected_count"] = spoken_form_result.rejected_count + allowed_indexes.update(spoken_form_result.approved_correction_indexes) + return ( + keep_corrections_with_indexes(corrections, allowed_indexes), + spoken_form_result.skipped, + validation_summary, + ) def _create_run_dir(work_dir: Path) -> Path: @@ -331,6 +367,9 @@ def _write_run_metadata( "grammar_confidence_threshold": config.grammar_confidence_threshold, "grammar_validation_enabled": config.grammar_validation_enabled, "grammar_validation_confidence_threshold": config.grammar_validation_confidence_threshold, + "grammar_spoken_form_validation_confidence_threshold": ( + config.grammar_spoken_form_validation_confidence_threshold + ), "max_retries": config.max_retries, "glossary_max_llm_passes": config.glossary_max_llm_passes, "grammar_max_llm_passes": config.grammar_max_llm_passes, diff --git a/src/audita/prompts.py b/src/audita/prompts.py index 4dfd8cd..14d372d 100644 --- a/src/audita/prompts.py +++ b/src/audita/prompts.py @@ -116,7 +116,8 @@ def build_grammar_validation_messages(validation_payload: List[dict]) -> List[Me system = ( "You are Audita, a conservative semantic validation assistant. " "Evaluate whether each proposed grammar correction preserves the same spoken content and meaning. " - "Do not judge whether the correction is more polished; judge only whether it changes meaning." + "Do not judge whether the correction is more polished, and do not try to rescue likely homophone or transcription fixes. " + "Judge only whether the corrected text preserves the same written meaning." ) user = ( "Review these proposed grammar corrections and decide whether each correction preserves meaning.\n\n" @@ -125,10 +126,34 @@ def build_grammar_validation_messages(validation_payload: List[dict]) -> List[Me "- Reject corrections that add or remove negation, reverse meaning, introduce antonyms, change names, change quantities, change actions, change who did what, or otherwise substantively alter the speaker's meaning.\n" "- Reject corrections like changing \"became visible\" to \"became invisible\" because that reverses the meaning.\n" "- Allow capitalization and punctuation changes when they preserve meaning.\n" - "- Allow spelling and homophone fixes only when the corrected segment preserves the same spoken content.\n" + "- If a correction changes meaning because the original transcript may have used the wrong homophone or a phonetic misspelling, reject it here; that question is handled in a separate spoken-form validation step.\n" "- Do not use domain knowledge to second-guess protected glossary terms; glossary-protected corrections are excluded from this validation step.\n" "- Each returned validation must contain only correction_index, is_meaning_preserving, confidence, and reason.\n" "- confidence must be between 0.0 and 1.0.\n\n" f"Corrections to validate:\n{payload_json}" ) return [{"role": "system", "content": system}, {"role": "user", "content": user}] + + +def build_grammar_spoken_form_validation_messages(validation_payload: List[dict]) -> List[Message]: + payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2) + system = ( + "You are Audita, a conservative spoken-form validation assistant. " + "Evaluate whether each rejected grammar correction is still a likely homophone, spoken-form, or transcription fix " + "supported by the local segment context. " + "Approve only corrections that plausibly recover the intended spoken words from a mistaken transcript rendering." + ) + user = ( + "Review these rejected grammar corrections and decide whether each one is a likely spoken-form correction.\n\n" + "Rules:\n" + "- Return one validation decision for every correction_index in the input.\n" + "- Approve a correction only when the original transcript text is plausibly a mistaken homophone, phonetic rendering, or transcription error, and the corrected text better matches the likely spoken words in context.\n" + "- Allow examples like changing \"dam\" to \"damn\" when the surrounding phrase strongly supports the intended spoken phrase.\n" + "- Reject examples like changing \"became visible\" to \"became invisible\" because that is a semantic reversal, not a likely spoken-form correction.\n" + "- Reject paraphrases, stylistic rewrites, content additions or removals, and any meaning-changing edit that is not clearly explained by a transcription or spoken-form mistake.\n" + "- Do not use domain knowledge to second-guess protected glossary terms; glossary-protected corrections are excluded from this validation step.\n" + "- Each returned validation must contain only correction_index, is_likely_spoken_form_correction, confidence, and reason.\n" + "- confidence must be between 0.0 and 1.0.\n\n" + f"Corrections to validate:\n{payload_json}" + ) + return [{"role": "system", "content": system}, {"role": "user", "content": user}] diff --git a/src/audita/schemas.py b/src/audita/schemas.py index e94ba50..6116749 100644 --- a/src/audita/schemas.py +++ b/src/audita/schemas.py @@ -217,6 +217,52 @@ class GrammarValidationSet(BaseModel): validations: List[GrammarValidationDecision] = Field(default_factory=list) +class GrammarSpokenFormValidationDecision(BaseModel): + model_config = ConfigDict(extra="forbid") + + correction_index: int = Field(ge=0) + is_likely_spoken_form_correction: bool + confidence: float = Field(ge=0.0, le=1.0) + reason: StrictStr + + @field_validator("correction_index", mode="before") + @classmethod + def require_integer_index(cls, value: Any) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError("must be an integer") + return value + + @field_validator("is_likely_spoken_form_correction", mode="before") + @classmethod + def require_boolean(cls, value: Any) -> bool: + if not isinstance(value, bool): + raise ValueError("must be a boolean") + return value + + @field_validator("confidence", mode="before") + @classmethod + def require_number(cls, value: Any) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError("must be a JSON number") + number = float(value) + if not math.isfinite(number): + raise ValueError("must be finite") + return number + + @field_validator("reason") + @classmethod + def require_non_empty_reason(cls, value: str) -> str: + if not value.strip(): + raise ValueError("must not be empty") + return value + + +class GrammarSpokenFormValidationSet(BaseModel): + model_config = ConfigDict(extra="forbid") + + validations: List[GrammarSpokenFormValidationDecision] = Field(default_factory=list) + + _TRANSCRIPT_ADAPTER = TypeAdapter(List[TranscriptSegment]) _SOURCE_TRANSCRIPT_ADAPTER = TypeAdapter(List[SourceTranscriptSegment]) diff --git a/src/audita/semantic_validation.py b/src/audita/semantic_validation.py index 9d5273a..fe15ced 100644 --- a/src/audita/semantic_validation.py +++ b/src/audita/semantic_validation.py @@ -1,11 +1,16 @@ import string from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple +from typing import Any, Dict, Iterable, List, Optional, Tuple from .corrections import ReplacementMode, SkippedCorrection from .errors import AuditaLLMError from .protection import ProtectedVocabulary -from .schemas import CorrectionCandidate, GrammarValidationSet, TranscriptSegment +from .schemas import ( + CorrectionCandidate, + GrammarSpokenFormValidationSet, + GrammarValidationSet, + TranscriptSegment, +) @dataclass(frozen=True) @@ -27,13 +32,19 @@ class GrammarValidationCandidate: @dataclass(frozen=True) -class GrammarValidationFilterResult: - corrections: List[CorrectionCandidate] - skipped: List[SkippedCorrection] - candidate_count: int +class GrammarMeaningValidationResult: + approved_correction_indexes: List[int] + rescue_candidates: List[GrammarValidationCandidate] + approved_count: int + rejected_count: int + + +@dataclass(frozen=True) +class GrammarSpokenFormValidationResult: + approved_correction_indexes: List[int] + skipped: List[SkippedCorrection] approved_count: int rejected_count: int - bypassed_count: int def select_grammar_validation_candidates( @@ -74,30 +85,58 @@ def select_grammar_validation_candidates( return candidates, bypassed_count -def filter_with_grammar_validations( - corrections: List[CorrectionCandidate], +def filter_with_meaning_preserving_validations( candidates: List[GrammarValidationCandidate], validation_set: GrammarValidationSet, confidence_threshold: float, -) -> GrammarValidationFilterResult: - candidate_indexes = {candidate.correction_index for candidate in candidates} - decisions_by_index = {} - for decision in validation_set.validations: - if decision.correction_index in decisions_by_index: - raise AuditaLLMError("LLM grammar validation response included duplicate correction_index values.") - if decision.correction_index not in candidate_indexes: - raise AuditaLLMError("LLM grammar validation response included an unknown correction_index.") - decisions_by_index[decision.correction_index] = decision +) -> GrammarMeaningValidationResult: + decisions_by_index = _index_validation_decisions( + validation_set.validations, + candidates, + duplicate_error="LLM grammar validation response included duplicate correction_index values.", + unknown_error="LLM grammar validation response included an unknown correction_index.", + missing_error="LLM grammar validation response omitted correction_index values.", + ) - missing_indexes = sorted(candidate_indexes - set(decisions_by_index)) - if missing_indexes: - raise AuditaLLMError("LLM grammar validation response omitted correction_index values.") - - rejected_by_index: Dict[int, SkippedCorrection] = {} + approved_correction_indexes: List[int] = [] + rescue_candidates: List[GrammarValidationCandidate] = [] for candidate in candidates: decision = decisions_by_index[candidate.correction_index] - if not decision.is_meaning_preserving or decision.confidence < confidence_threshold: - rejected_by_index[candidate.correction_index] = SkippedCorrection( + if decision.is_meaning_preserving and decision.confidence >= confidence_threshold: + approved_correction_indexes.append(candidate.correction_index) + else: + rescue_candidates.append(candidate) + + return GrammarMeaningValidationResult( + approved_correction_indexes=approved_correction_indexes, + rescue_candidates=rescue_candidates, + approved_count=len(approved_correction_indexes), + rejected_count=len(rescue_candidates), + ) + + +def filter_with_spoken_form_validations( + candidates: List[GrammarValidationCandidate], + validation_set: GrammarSpokenFormValidationSet, + confidence_threshold: float, +) -> GrammarSpokenFormValidationResult: + decisions_by_index = _index_validation_decisions( + validation_set.validations, + candidates, + duplicate_error="LLM spoken-form validation response included duplicate correction_index values.", + unknown_error="LLM spoken-form validation response included an unknown correction_index.", + missing_error="LLM spoken-form validation response omitted correction_index values.", + ) + + approved_correction_indexes: List[int] = [] + skipped: List[SkippedCorrection] = [] + for candidate in candidates: + decision = decisions_by_index[candidate.correction_index] + if decision.is_likely_spoken_form_correction and decision.confidence >= confidence_threshold: + approved_correction_indexes.append(candidate.correction_index) + continue + skipped.append( + SkippedCorrection( id=candidate.correction.id, reason="grammar validation rejected semantic change", original_text=candidate.correction.original_text, @@ -107,21 +146,24 @@ def filter_with_grammar_validations( validation_confidence=decision.confidence, validation_reason=decision.reason, ) + ) - filtered_corrections = [ - correction for index, correction in enumerate(corrections) if index not in rejected_by_index - ] - approved_count = len(candidates) - len(rejected_by_index) - return GrammarValidationFilterResult( - corrections=filtered_corrections, - skipped=[rejected_by_index[index] for index in sorted(rejected_by_index)], - candidate_count=len(candidates), - approved_count=approved_count, - rejected_count=len(rejected_by_index), - bypassed_count=len(corrections) - len(candidates), + return GrammarSpokenFormValidationResult( + approved_correction_indexes=approved_correction_indexes, + skipped=skipped, + approved_count=len(approved_correction_indexes), + rejected_count=len(skipped), ) +def keep_corrections_with_indexes( + corrections: List[CorrectionCandidate], + allowed_indexes: Iterable[int], +) -> List[CorrectionCandidate]: + allowed_index_set = set(allowed_indexes) + return [correction for index, correction in enumerate(corrections) if index in allowed_index_set] + + def is_capitalization_or_punctuation_only(original_text: str, corrected_text: str) -> bool: return _semantic_key(original_text) == _semantic_key(corrected_text) @@ -152,6 +194,28 @@ def _preview_corrected_segment_text( return segment.text.replace(correction.original_text, correction.corrected_text) +def _index_validation_decisions( + decisions: Iterable[Any], + candidates: List[GrammarValidationCandidate], + duplicate_error: str, + unknown_error: str, + missing_error: str, +) -> Dict[int, Any]: + candidate_indexes = {candidate.correction_index for candidate in candidates} + decisions_by_index: Dict[int, Any] = {} + for decision in decisions: + if decision.correction_index in decisions_by_index: + raise AuditaLLMError(duplicate_error) + if decision.correction_index not in candidate_indexes: + raise AuditaLLMError(unknown_error) + decisions_by_index[decision.correction_index] = decision + + missing_indexes = sorted(candidate_indexes - set(decisions_by_index)) + if missing_indexes: + raise AuditaLLMError(missing_error) + return decisions_by_index + + _PUNCTUATION = set(string.punctuation) | {"—", "–", "…", "“", "”", "‘", "’"} diff --git a/tests/test_cli.py b/tests/test_cli.py index 8b40fec..ab53515 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -23,6 +23,7 @@ def test_process_help_includes_glossary_pass_flag(capsys): assert "--grammar-confidence-threshold" in output assert "--grammar-validation-enabled" in output assert "--grammar-validation-confidence-threshold" in output + assert "--grammar-spoken-form-validation-confidence-threshold" in output assert "--normalize-max-segment-gap" in output assert "--normalize-ellipsis-gap" in output assert "--normalize-max-segment-duration" in output diff --git a/tests/test_config.py b/tests/test_config.py index bfb5921..988739b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -8,6 +8,7 @@ from audita.config import ( DEFAULT_GLOSSARY_MAX_LLM_PASSES, DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD, DEFAULT_GRAMMAR_MAX_LLM_PASSES, + DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD, DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD, DEFAULT_GRAMMAR_VALIDATION_ENABLED, DEFAULT_MAX_RETRIES, @@ -36,6 +37,11 @@ def test_config_uses_defaults_with_api_key(): assert config.grammar_validation_enabled is True assert config.grammar_validation_confidence_threshold == DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD assert config.grammar_validation_confidence_threshold == 0.8 + assert ( + config.grammar_spoken_form_validation_confidence_threshold + == DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD + ) + assert config.grammar_spoken_form_validation_confidence_threshold == 0.8 assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP assert config.normalize_max_segment_gap == 5.0 assert config.normalize_ellipsis_gap == DEFAULT_NORMALIZE_ELLIPSIS_GAP @@ -58,6 +64,7 @@ def test_config_env_overrides_defaults(): "AUDITA_GRAMMAR_MAX_LLM_PASSES": "4", "AUDITA_GRAMMAR_VALIDATION_ENABLED": "false", "AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "0.91", + "AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD": "0.87", "AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "4.5", "AUDITA_NORMALIZE_ELLIPSIS_GAP": "1.5", "AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0", @@ -74,6 +81,7 @@ def test_config_env_overrides_defaults(): assert config.grammar_max_llm_passes == 4 assert config.grammar_validation_enabled is False assert config.grammar_validation_confidence_threshold == 0.91 + assert config.grammar_spoken_form_validation_confidence_threshold == 0.87 assert config.normalize_max_segment_gap == 4.5 assert config.normalize_ellipsis_gap == 1.5 assert config.normalize_max_segment_duration == 45.0 @@ -91,6 +99,7 @@ def test_config_cli_overrides_env(): "AUDITA_GRAMMAR_MAX_LLM_PASSES": "6", "AUDITA_GRAMMAR_VALIDATION_ENABLED": "false", "AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "0.91", + "AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD": "0.87", "AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "4.5", "AUDITA_NORMALIZE_ELLIPSIS_GAP": "1.5", "AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0", @@ -106,6 +115,7 @@ def test_config_cli_overrides_env(): grammar_max_llm_passes=3, grammar_validation_enabled=True, grammar_validation_confidence_threshold=0.75, + grammar_spoken_form_validation_confidence_threshold=0.72, normalize_max_segment_gap=3.0, normalize_ellipsis_gap=1.0, normalize_max_segment_duration=30.0, @@ -122,6 +132,7 @@ def test_config_cli_overrides_env(): assert config.grammar_max_llm_passes == 3 assert config.grammar_validation_enabled is True assert config.grammar_validation_confidence_threshold == 0.75 + assert config.grammar_spoken_form_validation_confidence_threshold == 0.72 assert config.normalize_max_segment_gap == 3.0 assert config.normalize_ellipsis_gap == 1.0 assert config.normalize_max_segment_duration == 30.0 @@ -168,6 +179,13 @@ def test_config_rejects_invalid_stage_thresholds(): AuditaConfig.from_sources( env={"OPENROUTER_API_KEY": "key", "AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "1.1"} ) + with pytest.raises(AuditaConfigError): + AuditaConfig.from_sources( + env={ + "OPENROUTER_API_KEY": "key", + "AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD": "-0.1", + } + ) def test_config_rejects_invalid_grammar_validation_enabled(): diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index fb2b0ef..acddacd 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -7,6 +7,8 @@ from audita.errors import AuditaError from audita.pipeline import process_transcript from audita.schemas import ( CorrectionCandidate, + GrammarSpokenFormValidationDecision, + GrammarSpokenFormValidationSet, CorrectionSet, GrammarValidationDecision, GrammarValidationSet, @@ -16,13 +18,16 @@ from audita.schemas import ( class FakeLLMClient: - def __init__(self, responses, validation_responses=None): + def __init__(self, responses, validation_responses=None, spoken_form_validation_responses=None): self.responses = list(responses) self.validation_responses = list(validation_responses or []) + self.spoken_form_validation_responses = list(spoken_form_validation_responses or []) self.calls = 0 self.validation_calls = 0 + self.spoken_form_validation_calls = 0 self.messages = [] self.validation_messages = [] + self.spoken_form_validation_messages = [] def create_corrections(self, messages, config): self.calls += 1 @@ -36,6 +41,13 @@ class FakeLLMClient: raise AssertionError("Unexpected grammar validation request.") return self.validation_responses.pop(0) + def create_grammar_spoken_form_validations(self, messages, config): + self.spoken_form_validation_calls += 1 + self.spoken_form_validation_messages.append(messages) + if not self.spoken_form_validation_responses: + raise AssertionError("Unexpected spoken-form validation request.") + return self.spoken_form_validation_responses.pop(0) + def _config( tmp_path, @@ -43,6 +55,7 @@ def _config( grammar_max_llm_passes=3, grammar_validation_enabled=False, grammar_validation_confidence_threshold=0.8, + grammar_spoken_form_validation_confidence_threshold=0.8, ): return AuditaConfig( api_key="key", @@ -54,6 +67,7 @@ def _config( grammar_max_llm_passes=grammar_max_llm_passes, grammar_validation_enabled=grammar_validation_enabled, grammar_validation_confidence_threshold=grammar_validation_confidence_threshold, + grammar_spoken_form_validation_confidence_threshold=grammar_spoken_form_validation_confidence_threshold, work_dir=tmp_path / "work", ) @@ -698,6 +712,7 @@ def test_pipeline_writes_stage_metadata_for_unresolved_retries(tmp_path): assert metadata["grammar_confidence_threshold"] == 0.8 assert metadata["grammar_validation_enabled"] is False assert metadata["grammar_validation_confidence_threshold"] == 0.8 + assert metadata["grammar_spoken_form_validation_confidence_threshold"] == 0.8 assert [item["stage"] for item in metadata["stages"]] == ["glossary", "grammar"] assert [item["pass_number"] for item in metadata["stages"][0]["passes"]] == [1, 2] assert metadata["stages"][0]["passes"][0]["retry_segment_count"] == 1 @@ -766,12 +781,21 @@ def test_grammar_validation_rejects_semantic_change(tmp_path): confidence=0.99, reason="This reverses visible to invisible.", ) + spoken_form_validation = GrammarSpokenFormValidationDecision( + correction_index=0, + is_likely_spoken_form_correction=False, + confidence=0.98, + reason="This is a semantic reversal, not a likely spoken-form correction.", + ) fake_client = FakeLLMClient( [ CorrectionSet(corrections=[]), CorrectionSet(corrections=[grammar_correction]), ], validation_responses=[GrammarValidationSet(validations=[validation])], + spoken_form_validation_responses=[ + GrammarSpokenFormValidationSet(validations=[spoken_form_validation]) + ], ) revised = process_transcript( @@ -783,20 +807,24 @@ def test_grammar_validation_rejects_semantic_change(tmp_path): assert revised[0].text == "He became visible." assert fake_client.validation_calls == 1 + assert fake_client.spoken_form_validation_calls == 1 run_dirs = list((tmp_path / "work").iterdir()) assert len(run_dirs) == 1 diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8")) skipped = diagnostics["skipped_corrections"][0] assert skipped["stage"] == "grammar" assert skipped["reason"] == "grammar validation rejected semantic change" - assert skipped["validation_confidence"] == 0.99 - assert skipped["validation_reason"] == "This reverses visible to invisible." + assert skipped["validation_confidence"] == 0.98 + assert skipped["validation_reason"] == "This is a semantic reversal, not a likely spoken-form correction." metadata = json.loads((run_dirs[0] / "metadata.json").read_text(encoding="utf-8")) grammar_pass = metadata["stages"][1]["passes"][0] assert grammar_pass["validation_candidate_count"] == 1 assert grammar_pass["validation_approved_count"] == 0 assert grammar_pass["validation_rejected_count"] == 1 assert grammar_pass["validation_bypassed_count"] == 0 + assert grammar_pass["spoken_form_validation_candidate_count"] == 1 + assert grammar_pass["spoken_form_validation_approved_count"] == 0 + assert grammar_pass["spoken_form_validation_rejected_count"] == 1 def test_grammar_validation_accepts_meaning_preserving_fix(tmp_path): @@ -836,6 +864,57 @@ def test_grammar_validation_accepts_meaning_preserving_fix(tmp_path): assert revised[0].text == "Keep in mind." assert fake_client.validation_calls == 1 + assert fake_client.spoken_form_validation_calls == 0 + assert list((tmp_path / "work").iterdir()) == [] + + +def test_grammar_validation_rescues_likely_spoken_form_fix(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."} + ] + """ + ) + grammar_correction = CorrectionCandidate( + id=1, + original_text="dam", + corrected_text="damn", + confidence=0.95, + ) + meaning_validation = GrammarValidationDecision( + correction_index=0, + is_meaning_preserving=False, + confidence=0.98, + reason="Written meaning changes from a barrier to a curse word.", + ) + spoken_form_validation = GrammarSpokenFormValidationDecision( + correction_index=0, + is_likely_spoken_form_correction=True, + confidence=0.97, + reason="The phrase strongly suggests the intended spoken word was the expletive.", + ) + fake_client = FakeLLMClient( + [ + CorrectionSet(corrections=[]), + CorrectionSet(corrections=[grammar_correction]), + ], + validation_responses=[GrammarValidationSet(validations=[meaning_validation])], + spoken_form_validation_responses=[ + GrammarSpokenFormValidationSet(validations=[spoken_form_validation]) + ], + ) + + revised = process_transcript( + transcript, + _glossary(), + _config(tmp_path, grammar_validation_enabled=True), + llm_client=fake_client, + ) + + assert revised[0].text == "ChatGPT still can't really do that with a damn." + assert fake_client.validation_calls == 1 + assert fake_client.spoken_form_validation_calls == 1 assert list((tmp_path / "work").iterdir()) == [] @@ -877,6 +956,7 @@ def test_grammar_validation_bypasses_protected_vocabulary_correction(tmp_path): assert revised[0].text == "The Jesters arrived." assert fake_client.validation_calls == 0 + assert fake_client.spoken_form_validation_calls == 0 assert list((tmp_path / "work").iterdir()) == [] @@ -910,6 +990,7 @@ def test_disabled_grammar_validation_preserves_current_behavior(tmp_path): assert revised[0].text == "He became invisible." assert fake_client.validation_calls == 0 + assert fake_client.spoken_form_validation_calls == 0 assert list((tmp_path / "work").iterdir()) == [] @@ -949,6 +1030,54 @@ def test_missing_grammar_validation_decision_fails_and_preserves_diagnostics(tmp assert (run_dirs[0] / "grammar" / "pass-0001" / "validation-response-0000.json").exists() +def test_missing_spoken_form_validation_decision_fails_and_preserves_diagnostics(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."} + ] + """ + ) + grammar_correction = CorrectionCandidate( + id=1, + original_text="dam", + corrected_text="damn", + confidence=0.95, + ) + fake_client = FakeLLMClient( + [ + CorrectionSet(corrections=[]), + CorrectionSet(corrections=[grammar_correction]), + ], + validation_responses=[ + GrammarValidationSet( + validations=[ + GrammarValidationDecision( + correction_index=0, + is_meaning_preserving=False, + confidence=0.99, + reason="Written meaning changes.", + ) + ] + ) + ], + spoken_form_validation_responses=[GrammarSpokenFormValidationSet(validations=[])], + ) + + with pytest.raises(AuditaError): + process_transcript( + transcript, + _glossary(), + _config(tmp_path, grammar_validation_enabled=True), + llm_client=fake_client, + ) + + run_dirs = list((tmp_path / "work").iterdir()) + assert len(run_dirs) == 1 + assert (run_dirs[0] / "grammar" / "pass-0001" / "spoken-form-validation-prompt-0000.json").exists() + assert (run_dirs[0] / "grammar" / "pass-0001" / "spoken-form-validation-response-0000.json").exists() + + def test_grammar_stage_cannot_reverse_glossary_protected_term(tmp_path): transcript = parse_source_transcript_json( """ diff --git a/tests/test_prompts.py b/tests/test_prompts.py index 33aeb13..2b3be7d 100644 --- a/tests/test_prompts.py +++ b/tests/test_prompts.py @@ -4,6 +4,7 @@ from audita.chunking import chunk_transcript from audita.prompts import ( build_glossary_correction_messages, build_grammar_correction_messages, + build_grammar_spoken_form_validation_messages, build_grammar_validation_messages, ) from audita.schemas import parse_glossary_yaml, parse_transcript_json @@ -194,6 +195,28 @@ def test_grammar_validation_prompt_rejects_semantic_changes(): assert "became visible" in prompt_text assert "became invisible" in prompt_text assert "reverses the meaning" in prompt_text - assert "spelling and homophone fixes only when" in prompt_text + assert "do not try to rescue likely homophone or transcription fixes" in prompt_text + assert "handled in a separate spoken-form validation step" in prompt_text assert "correction_index" in prompt_text assert "is_meaning_preserving" in prompt_text + + +def test_grammar_spoken_form_validation_prompt_allows_homophone_rescue(): + messages = build_grammar_spoken_form_validation_messages( + [ + { + "correction_index": 0, + "id": 1, + "original_segment_text": "ChatGPT still can't really do that with a dam.", + "corrected_segment_text": "ChatGPT still can't really do that with a damn.", + "original_text": "dam", + "corrected_text": "damn", + } + ] + ) + prompt_text = "\n".join(message["content"] for message in messages) + + assert "likely homophone, spoken-form, or transcription fix" in prompt_text + assert '"dam" to "damn"' in prompt_text + assert '"became visible" to "became invisible"' in prompt_text + assert "is_likely_spoken_form_correction" in prompt_text diff --git a/tests/test_semantic_validation.py b/tests/test_semantic_validation.py index 0a21f64..a652406 100644 --- a/tests/test_semantic_validation.py +++ b/tests/test_semantic_validation.py @@ -1,16 +1,22 @@ import pytest from audita.errors import AuditaLLMError -from audita.semantic_validation import select_grammar_validation_candidates -from audita.semantic_validation import filter_with_grammar_validations from audita.protection import ProtectedVocabulary from audita.schemas import ( CorrectionCandidate, + GrammarSpokenFormValidationDecision, + GrammarSpokenFormValidationSet, GrammarValidationDecision, GrammarValidationSet, parse_glossary_yaml, parse_transcript_json, ) +from audita.semantic_validation import ( + filter_with_meaning_preserving_validations, + filter_with_spoken_form_validations, + keep_corrections_with_indexes, + select_grammar_validation_candidates, +) def _transcript(): @@ -118,7 +124,106 @@ def test_meaning_sensitive_substitution_requires_validation(): assert bypassed_count == 0 -def test_validation_rejects_duplicate_and_unknown_decisions(): +def test_meaning_preserving_validation_approves_without_rescue(): + correction = CorrectionCandidate( + id=1, + original_text="bind", + corrected_text="mind", + confidence=0.95, + ) + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Keep in bind."} + ] + """ + ) + candidates, _ = select_grammar_validation_candidates( + transcript, + [correction], + confidence_threshold=0.8, + replacement_mode="require_unique", + protected_vocabulary=_vocabulary(), + ) + + result = filter_with_meaning_preserving_validations( + candidates, + GrammarValidationSet( + validations=[ + GrammarValidationDecision( + correction_index=0, + is_meaning_preserving=True, + confidence=0.95, + reason="This preserves the intended meaning.", + ) + ] + ), + confidence_threshold=0.8, + ) + + assert result.approved_correction_indexes == [0] + assert result.rescue_candidates == [] + assert result.approved_count == 1 + assert result.rejected_count == 0 + + +def test_spoken_form_validation_can_rescue_homophone_fix(): + correction = CorrectionCandidate( + id=1, + original_text="dam", + corrected_text="damn", + confidence=0.95, + ) + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."} + ] + """ + ) + candidates, _ = select_grammar_validation_candidates( + transcript, + [correction], + confidence_threshold=0.8, + replacement_mode="require_unique", + protected_vocabulary=_vocabulary(), + ) + meaning_result = filter_with_meaning_preserving_validations( + candidates, + GrammarValidationSet( + validations=[ + GrammarValidationDecision( + correction_index=0, + is_meaning_preserving=False, + confidence=0.99, + reason="Written meaning changes from a barrier to a curse word.", + ) + ] + ), + confidence_threshold=0.8, + ) + spoken_form_result = filter_with_spoken_form_validations( + meaning_result.rescue_candidates, + GrammarSpokenFormValidationSet( + validations=[ + GrammarSpokenFormValidationDecision( + correction_index=0, + is_likely_spoken_form_correction=True, + confidence=0.95, + reason="The surrounding phrase strongly supports the intended spoken phrase with a curse word.", + ) + ] + ), + confidence_threshold=0.8, + ) + + kept = keep_corrections_with_indexes([correction], spoken_form_result.approved_correction_indexes) + assert meaning_result.approved_correction_indexes == [] + assert kept == [correction] + assert spoken_form_result.skipped == [] + + +def test_spoken_form_validation_rejects_non_homophone_semantic_change(): correction = CorrectionCandidate( id=1, original_text="visible", @@ -132,30 +237,90 @@ def test_validation_rejects_duplicate_and_unknown_decisions(): replacement_mode="require_unique", protected_vocabulary=_vocabulary(), ) - decision = GrammarValidationDecision( + meaning_result = filter_with_meaning_preserving_validations( + candidates, + GrammarValidationSet( + validations=[ + GrammarValidationDecision( + correction_index=0, + is_meaning_preserving=False, + confidence=0.99, + reason="This reverses visible to invisible.", + ) + ] + ), + confidence_threshold=0.8, + ) + spoken_form_result = filter_with_spoken_form_validations( + meaning_result.rescue_candidates, + GrammarSpokenFormValidationSet( + validations=[ + GrammarSpokenFormValidationDecision( + correction_index=0, + is_likely_spoken_form_correction=False, + confidence=0.99, + reason="This is a semantic reversal, not a likely spoken-form transcription error.", + ) + ] + ), + confidence_threshold=0.8, + ) + + assert spoken_form_result.approved_correction_indexes == [] + assert spoken_form_result.rejected_count == 1 + assert spoken_form_result.skipped[0].reason == "grammar validation rejected semantic change" + assert spoken_form_result.skipped[0].validation_reason == ( + "This is a semantic reversal, not a likely spoken-form transcription error." + ) + + +def test_validation_rejects_duplicate_unknown_and_missing_decisions(): + correction = CorrectionCandidate( + id=1, + original_text="visible", + corrected_text="invisible", + confidence=0.95, + ) + candidates, _ = select_grammar_validation_candidates( + _transcript(), + [correction], + confidence_threshold=0.8, + replacement_mode="require_unique", + protected_vocabulary=_vocabulary(), + ) + meaning_decision = GrammarValidationDecision( correction_index=0, is_meaning_preserving=True, confidence=0.95, reason="Preserves meaning.", ) + spoken_form_decision = GrammarSpokenFormValidationDecision( + correction_index=0, + is_likely_spoken_form_correction=True, + confidence=0.95, + reason="Likely spoken-form correction.", + ) with pytest.raises(AuditaLLMError): - filter_with_grammar_validations( - [correction], + filter_with_meaning_preserving_validations( candidates, - GrammarValidationSet(validations=[decision, decision]), + GrammarValidationSet(validations=[meaning_decision, meaning_decision]), confidence_threshold=0.8, ) - with pytest.raises(AuditaLLMError): - filter_with_grammar_validations( - [correction], + filter_with_meaning_preserving_validations( candidates, - GrammarValidationSet( + GrammarValidationSet(validations=[]), + confidence_threshold=0.8, + ) + with pytest.raises(AuditaLLMError): + filter_with_spoken_form_validations( + candidates, + GrammarSpokenFormValidationSet( validations=[ - GrammarValidationDecision( + GrammarSpokenFormValidationDecision( correction_index=99, - is_meaning_preserving=True, + is_likely_spoken_form_correction=True, confidence=0.95, reason="Unknown.", ) @@ -163,3 +328,9 @@ def test_validation_rejects_duplicate_and_unknown_decisions(): ), confidence_threshold=0.8, ) + with pytest.raises(AuditaLLMError): + filter_with_spoken_form_validations( + candidates, + GrammarSpokenFormValidationSet(validations=[spoken_form_decision, spoken_form_decision]), + confidence_threshold=0.8, + )