diff --git a/README.md b/README.md index 89e1ad4..e2bbc8e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Audita is a framework-first transcript correction application. The public `audit - deterministic transcript normalization - token-batched module orchestration -- concrete `glossary`, `homophones`, and `spoken_word` modules built on reusable proposal / validator contracts +- concrete `glossary`, `homophones`, `spoken_word`, and `grammar` modules built on reusable proposal / validator contracts - structured run reporting and work-dir diagnostics The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`. @@ -42,10 +42,12 @@ Resolved run instance names are auto-numbered for repeats, so the default report 4. `spoken_word` 5. `grammar` -The default module sequence is partially implemented today: +The default module sequence is fully implemented today: -- `glossary`, `homophones`, the second `glossary` pass, and `spoken_word` run real LLM-backed proposal and validation stages -- `grammar` remains a stub and currently proposes no corrections +- `glossary` proposes glossary-supported acoustic corrections +- `homophones` proposes conservative homophone and mistranscription corrections +- `spoken_word` proposes conservative dysfluency cleanup +- `grammar` proposes punctuation, capitalization, and spacing cleanup only To run a custom module sequence, pass `--modules`: @@ -77,7 +79,7 @@ audita process transcript.json --glossary glossary.yaml --output corrected.json Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr. `--report-json` writes a separate machine-readable run report and never mixes report data into stdout. -Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Normal runs now require `OPENROUTER_API_KEY`, because the `glossary`, `homophones`, and `spoken_word` modules make real LLM calls. +Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Normal runs now require `OPENROUTER_API_KEY`, because the `glossary`, `homophones`, `spoken_word`, and `grammar` modules make real LLM calls. | Environment variable | CLI flag | Default | Purpose | | --- | --- | --- | --- | @@ -87,6 +89,7 @@ Useful configuration can be supplied by CLI flag or environment variable. CLI fl | `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses | | `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `6144` | Maximum estimated tokens per transcript batch | | `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation | +| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | `--grammar-confidence-threshold` | `0.8` | Minimum confidence required for grammar proposals to survive validation | | `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation | | `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | `--spoken-word-confidence-threshold` | `0.8` | Minimum confidence required for spoken-word proposals to survive validation | | `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging | diff --git a/src/audita/cli.py b/src/audita/cli.py index 3048f7f..9a065cf 100644 --- a/src/audita/cli.py +++ b/src/audita/cli.py @@ -40,6 +40,11 @@ def _build_parser() -> argparse.ArgumentParser: type=float, help="minimum confidence required for glossary proposals to survive validation", ) + process.add_argument( + "--grammar-confidence-threshold", + type=float, + help="minimum confidence required for grammar proposals to survive validation", + ) process.add_argument( "--homophones-confidence-threshold", type=float, @@ -89,6 +94,7 @@ def _process(args: argparse.Namespace) -> int: max_retries=args.max_retries, max_section_tokens=args.max_section_tokens, glossary_confidence_threshold=args.glossary_confidence_threshold, + grammar_confidence_threshold=args.grammar_confidence_threshold, homophones_confidence_threshold=args.homophones_confidence_threshold, spoken_word_confidence_threshold=args.spoken_word_confidence_threshold, normalize_max_segment_gap=args.normalize_max_segment_gap, diff --git a/src/audita/core/config.py b/src/audita/core/config.py index 6a66db4..5b16b86 100644 --- a/src/audita/core/config.py +++ b/src/audita/core/config.py @@ -14,6 +14,7 @@ DEFAULT_BASE_URL = "https://openrouter.ai/api/v1" DEFAULT_MAX_RETRIES = 3 DEFAULT_MAX_SECTION_TOKENS = 6144 DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD = 0.80 +DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD = 0.80 DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD = 0.80 DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD = 0.80 DEFAULT_WORK_DIR = "/tmp/audita" @@ -32,6 +33,7 @@ class ConfigOverrides: max_retries: Optional[int] = None max_section_tokens: Optional[int] = None glossary_confidence_threshold: Optional[float] = None + grammar_confidence_threshold: Optional[float] = None homophones_confidence_threshold: Optional[float] = None spoken_word_confidence_threshold: Optional[float] = None normalize_max_segment_gap: Optional[float] = None @@ -51,6 +53,7 @@ class AuditaConfig: max_retries: int = DEFAULT_MAX_RETRIES max_section_tokens: int = DEFAULT_MAX_SECTION_TOKENS glossary_confidence_threshold: float = DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD + grammar_confidence_threshold: float = DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD homophones_confidence_threshold: float = DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD spoken_word_confidence_threshold: float = DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD normalize_max_segment_gap: float = DEFAULT_NORMALIZE_MAX_SEGMENT_GAP @@ -96,6 +99,12 @@ class AuditaConfig: DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD, "AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD", ), + grammar_confidence_threshold=_select_float( + selected.grammar_confidence_threshold, + source.get("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD"), + DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD, + "AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD", + ), homophones_confidence_threshold=_select_float( selected.homophones_confidence_threshold, source.get("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD"), @@ -156,6 +165,8 @@ class AuditaConfig: raise AuditaConfigError("AUDITA_MAX_SECTION_TOKENS must be greater than zero.") if not 0.0 <= self.glossary_confidence_threshold <= 1.0: raise AuditaConfigError("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.") + if not 0.0 <= self.grammar_confidence_threshold <= 1.0: + raise AuditaConfigError("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.") if not 0.0 <= self.homophones_confidence_threshold <= 1.0: raise AuditaConfigError("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.") if not 0.0 <= self.spoken_word_confidence_threshold <= 1.0: @@ -190,6 +201,7 @@ class AuditaConfig: "max_retries": self.max_retries, "max_section_tokens": self.max_section_tokens, "glossary_confidence_threshold": self.glossary_confidence_threshold, + "grammar_confidence_threshold": self.grammar_confidence_threshold, "homophones_confidence_threshold": self.homophones_confidence_threshold, "spoken_word_confidence_threshold": self.spoken_word_confidence_threshold, "normalize_max_segment_gap": self.normalize_max_segment_gap, diff --git a/src/audita/modules/grammar.py b/src/audita/modules/grammar.py index 8cf896e..e3f1be8 100644 --- a/src/audita/modules/grammar.py +++ b/src/audita/modules/grammar.py @@ -2,7 +2,15 @@ from typing import Sequence from audita.core.chunking import TranscriptSection from audita.framework.models import CorrectionProposal, ModuleContext -from audita.validators import ProtectedGlossaryTermsValidator, Validator +from audita.framework.proposal_generation import generate_llm_correction_proposals +from audita.modules.prompts import build_grammar_proposal_messages +from audita.validators import ( + GrammarOnlyValidator, + MeaningReversalValidator, + ProposalConfidenceValidator, + ProtectedGlossaryTermsValidator, + Validator, +) class GrammarModule: @@ -10,11 +18,20 @@ class GrammarModule: replacement_policy = "require_unique" def validators(self) -> Sequence[Validator]: - return [ProtectedGlossaryTermsValidator("protected_glossary_guard")] + return [ + ProposalConfidenceValidator("proposal_confidence_guard", "grammar_confidence_threshold"), + ProtectedGlossaryTermsValidator("protected_glossary_guard"), + GrammarOnlyValidator("grammar_only_guard"), + MeaningReversalValidator("meaning_reversal_review"), + ] def propose( self, transcript_section: TranscriptSection, context: ModuleContext, ) -> Sequence[CorrectionProposal]: - return [] + return generate_llm_correction_proposals( + section=transcript_section, + context=context, + prompt_builder=build_grammar_proposal_messages, + ) diff --git a/src/audita/modules/prompts.py b/src/audita/modules/prompts.py index 1530b05..a729555 100644 --- a/src/audita/modules/prompts.py +++ b/src/audita/modules/prompts.py @@ -122,3 +122,38 @@ def build_spoken_word_proposal_messages(section: TranscriptSection, glossary: Gl f"Transcript section:\n{section_json}" ) return [{"role": "system", "content": system}, {"role": "user", "content": user}] + + +def build_grammar_proposal_messages(section: TranscriptSection, glossary: Glossary) -> List[Message]: + glossary_json = json.dumps(glossary.model_dump(mode="json", exclude_none=True), ensure_ascii=False, indent=2) + section_json = json.dumps(section.prompt_payload(), ensure_ascii=False, indent=2) + + system = ( + "You are Audita, a conservative grammar cleanup assistant. " + "Identify only punctuation, capitalization, and spacing cleanup that preserves the same underlying words. " + "Do not change content, substitute words, or rewrite the speaker's phrasing." + ) + user = ( + "Review this transcript section and return only grammar cleanup corrections that should be applied.\n\n" + "Rules:\n" + "- Allowed changes are punctuation, capitalization, and spacing cleanup only.\n" + "- You may add, remove, or adjust commas, periods, quotation marks, apostrophes, dashes, ellipses, spacing, and capitalization when the underlying words stay the same.\n" + "- Do not make word substitutions, spelling fixes, homophone fixes, filler cleanup, repetition cleanup, paraphrases, or other semantic rewrites.\n" + "- Do not change one written word into a different written word, except for capitalization changes to the same letters.\n" + "- Treat glossary names and aliases as protected spellings and context.\n" + "- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.\n" + "- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n" + "- Use the exact id from the input segment.\n" + "- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n" + "- Choose an original_text span that appears exactly once in the current segment text.\n" + "- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n" + "- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n" + "- Do not return corrections where original_text and corrected_text are identical.\n" + "- Do not return speaker, start, or end fields.\n" + "- Return only changed segments; do not return entries for unchanged segments.\n" + "- confidence must be between 0.0 and 1.0.\n" + "- If no corrections are needed, return an empty corrections list.\n\n" + f"Protected glossary/context:\n{glossary_json}\n\n" + f"Transcript section:\n{section_json}" + ) + return [{"role": "system", "content": system}, {"role": "user", "content": user}] diff --git a/src/audita/validators/__init__.py b/src/audita/validators/__init__.py index 19dc727..e62a3c9 100644 --- a/src/audita/validators/__init__.py +++ b/src/audita/validators/__init__.py @@ -1,5 +1,5 @@ from .base import ValidationContext, ValidationDecision, ValidationResult, Validator -from .deterministic import ProposalConfidenceValidator, ProtectedGlossaryTermsValidator +from .deterministic import GrammarOnlyValidator, ProposalConfidenceValidator, ProtectedGlossaryTermsValidator from .llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator from .protection import ProtectedVocabulary @@ -10,6 +10,7 @@ __all__ = [ "Validator", "ProposalConfidenceValidator", "ProtectedGlossaryTermsValidator", + "GrammarOnlyValidator", "ProtectedVocabulary", "SpokenFormPlausibilityValidator", "SpokenWordValidator", diff --git a/src/audita/validators/deterministic.py b/src/audita/validators/deterministic.py index 07a9fc2..74a515a 100644 --- a/src/audita/validators/deterministic.py +++ b/src/audita/validators/deterministic.py @@ -1,3 +1,4 @@ +import string from dataclasses import dataclass from .base import ValidationContext, ValidationDecision, ValidationResult @@ -48,3 +49,38 @@ class ProtectedGlossaryTermsValidator: for proposal in context.proposals ], ) + + +_GRAMMAR_PUNCTUATION = set(string.punctuation) | {"—", "–", "…", "“", "”", "‘", "’"} + + +@dataclass(frozen=True) +class GrammarOnlyValidator: + name: str + execution_kind: str = "deterministic" + + def validate(self, context: ValidationContext) -> ValidationResult: + return ValidationResult( + validator_name=self.name, + execution_kind=self.execution_kind, + decisions=[ + ValidationDecision( + proposal_index=proposal.proposal_index, + approved=_grammar_semantic_key(proposal.original_text) == _grammar_semantic_key(proposal.corrected_text), + reason=( + None + if _grammar_semantic_key(proposal.original_text) == _grammar_semantic_key(proposal.corrected_text) + else "correction is not limited to punctuation, capitalization, and spacing" + ), + ) + for proposal in context.proposals + ], + ) + + +def _grammar_semantic_key(text: str) -> str: + return "".join( + character.casefold() + for character in text + if not character.isspace() and character not in _GRAMMAR_PUNCTUATION + ) diff --git a/tests/test_llm_validators.py b/tests/test_llm_validators.py index f30eab0..8f2efeb 100644 --- a/tests/test_llm_validators.py +++ b/tests/test_llm_validators.py @@ -5,6 +5,7 @@ from audita.core.config import AuditaConfig from audita.core.errors import AuditaLLMError from audita.core.schemas import parse_glossary_yaml, parse_transcript_json from audita.framework.models import CorrectionProposal, ModuleRunSpec +from audita.validators import GrammarOnlyValidator from audita.validators.base import ValidationContext from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator from audita.validators.prompts import ( @@ -309,6 +310,118 @@ def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)] +def test_grammar_only_validator_allows_formatting_only_changes(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello there"}, + {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "cant we go"}, + {"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "Hello,world"} + ] + """ + ) + proposals = [ + CorrectionProposal( + proposal_index=0, + module_instance="grammar", + module_key="grammar", + id=1, + original_text="hello there", + corrected_text="Hello there.", + confidence=0.95, + ), + CorrectionProposal( + proposal_index=1, + module_instance="grammar", + module_key="grammar", + id=2, + original_text="cant", + corrected_text="can't", + confidence=0.95, + ), + CorrectionProposal( + proposal_index=2, + module_instance="grammar", + module_key="grammar", + id=3, + original_text="Hello,world", + corrected_text="Hello, world", + confidence=0.95, + ), + ] + + result = GrammarOnlyValidator("grammar_only_guard").validate( + _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path) + ) + + assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [ + (0, True), + (1, True), + (2, True), + ] + + +def test_grammar_only_validator_rejects_word_level_changes(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "their plan"}, + {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "dam"}, + {"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "uh"}, + {"id": 4, "speaker": "A", "start": 3.0, "end": 4.0, "text": "I I agree"} + ] + """ + ) + proposals = [ + CorrectionProposal( + proposal_index=0, + module_instance="grammar", + module_key="grammar", + id=1, + original_text="their", + corrected_text="there", + confidence=0.95, + ), + CorrectionProposal( + proposal_index=1, + module_instance="grammar", + module_key="grammar", + id=2, + original_text="dam", + corrected_text="damn", + confidence=0.95, + ), + CorrectionProposal( + proposal_index=2, + module_instance="grammar", + module_key="grammar", + id=3, + original_text="uh", + corrected_text="", + confidence=0.95, + ), + CorrectionProposal( + proposal_index=3, + module_instance="grammar", + module_key="grammar", + id=4, + original_text="I I", + corrected_text="I", + confidence=0.95, + ), + ] + + result = GrammarOnlyValidator("grammar_only_guard").validate( + _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path) + ) + + assert [decision.approved for decision in result.decisions] == [False, False, False, False] + assert all( + decision.reason == "correction is not limited to punctuation, capitalization, and spacing" + for decision in result.decisions + ) + + @pytest.mark.parametrize( ("validator", "payload", "message_fragment"), [ diff --git a/tests/test_module_proposals.py b/tests/test_module_proposals.py index 3ceccf6..3544f80 100644 --- a/tests/test_module_proposals.py +++ b/tests/test_module_proposals.py @@ -5,9 +5,14 @@ from audita.core.config import AuditaConfig from audita.core.errors import AuditaLLMError from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json, parse_transcript_json from audita.framework.models import ModuleContext, ModuleRunSpec +from audita.modules.grammar import GrammarModule from audita.modules.glossary import GlossaryModule from audita.modules.homophones import HomophonesModule -from audita.modules.prompts import build_homophones_proposal_messages, build_spoken_word_proposal_messages +from audita.modules.prompts import ( + build_grammar_proposal_messages, + build_homophones_proposal_messages, + build_spoken_word_proposal_messages, +) from audita.modules.spoken_word import SpokenWordModule from audita.pipeline import process_transcript_result @@ -175,6 +180,70 @@ def test_spoken_word_prompt_is_explicitly_scoped_to_dysfluency_cleanup(): assert '"id": 1' in messages[1]["content"] +def test_grammar_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} + ] + """ + ) + section = chunk_transcript(transcript, max_section_tokens=1000)[0] + module = GrammarModule() + client = FakeStructuredLLMClient( + [ + { + "corrections": [ + { + "id": 1, + "original_text": "hello world", + "corrected_text": "Hello world.", + "confidence": 0.95, + } + ] + } + ] + ) + context = ModuleContext( + run_spec=ModuleRunSpec(instance_name="grammar", module_key="grammar", module=module), + glossary=_glossary(), + config=AuditaConfig.from_sources(env={}), + run_dir=tmp_path, + llm_client=client, + ) + + proposals = list(module.propose(section, context)) + + assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [ + (1, "hello world", "Hello world.", 0.95) + ] + assert (tmp_path / "prompt-0000.json").exists() + assert (tmp_path / "corrections-0000.json").exists() + prompt_text = client.calls[0]["messages"][1]["content"] + assert "punctuation, capitalization, and spacing" in prompt_text + assert "exact text span" in prompt_text + assert "word substitutions" in prompt_text + + +def test_grammar_prompt_is_explicitly_scoped_to_formatting_cleanup(): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} + ] + """ + ) + section = chunk_transcript(transcript, max_section_tokens=1000)[0] + + messages = build_grammar_proposal_messages(section, _glossary()) + combined = messages[0]["content"] + messages[1]["content"] + + assert "punctuation, capitalization, and spacing" in combined + assert "word substitutions" in combined + assert "homophone fixes" in combined + assert '"id": 1' in messages[1]["content"] + + def test_process_transcript_result_uses_injected_fake_client_and_applies_sequential_module_updates(tmp_path): transcript = parse_source_transcript_json( """ @@ -194,7 +263,9 @@ def test_process_transcript_result_uses_injected_fake_client_and_applies_sequent max_retries=config.max_retries, max_section_tokens=config.max_section_tokens, glossary_confidence_threshold=config.glossary_confidence_threshold, + grammar_confidence_threshold=config.grammar_confidence_threshold, homophones_confidence_threshold=config.homophones_confidence_threshold, + spoken_word_confidence_threshold=config.spoken_word_confidence_threshold, normalize_max_segment_gap=config.normalize_max_segment_gap, normalize_ellipsis_gap=config.normalize_ellipsis_gap, normalize_max_segment_duration=config.normalize_max_segment_duration, @@ -266,6 +337,7 @@ def test_process_transcript_result_uses_injected_fake_client_and_applies_sequent }, {"corrections": []}, {"corrections": []}, + {"corrections": []}, ] ) @@ -281,6 +353,7 @@ def test_process_transcript_result_uses_injected_fake_client_and_applies_sequent "homophones:meaning_reversal_review", "glossary_2:proposal", "spoken_word:proposal", + "grammar:proposal", ] assert "There were Jesters at the dam." in client.calls[3]["messages"][1]["content"] @@ -301,7 +374,9 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_ max_retries=base_config.max_retries, max_section_tokens=base_config.max_section_tokens, glossary_confidence_threshold=0.96, + grammar_confidence_threshold=base_config.grammar_confidence_threshold, homophones_confidence_threshold=base_config.homophones_confidence_threshold, + spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, normalize_max_segment_gap=base_config.normalize_max_segment_gap, normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, normalize_max_segment_duration=base_config.normalize_max_segment_duration, @@ -324,6 +399,7 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_ {"corrections": []}, {"corrections": []}, {"corrections": []}, + {"corrections": []}, ] ) @@ -335,6 +411,7 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_ "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", + "grammar:proposal", ] assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold" assert result.report.skipped_corrections[0].source == "validator:proposal_confidence_guard" @@ -358,6 +435,7 @@ def test_process_transcript_result_runs_spoken_word_module_with_full_validator_c max_retries=base_config.max_retries, max_section_tokens=base_config.max_section_tokens, glossary_confidence_threshold=base_config.glossary_confidence_threshold, + grammar_confidence_threshold=base_config.grammar_confidence_threshold, homophones_confidence_threshold=base_config.homophones_confidence_threshold, spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, normalize_max_segment_gap=base_config.normalize_max_segment_gap, @@ -440,6 +518,7 @@ def test_process_transcript_result_rejects_spoken_word_below_threshold_before_ll max_retries=base_config.max_retries, max_section_tokens=base_config.max_section_tokens, glossary_confidence_threshold=base_config.glossary_confidence_threshold, + grammar_confidence_threshold=base_config.grammar_confidence_threshold, homophones_confidence_threshold=base_config.homophones_confidence_threshold, spoken_word_confidence_threshold=0.96, normalize_max_segment_gap=base_config.normalize_max_segment_gap, @@ -476,3 +555,130 @@ def test_process_transcript_result_rejects_spoken_word_below_threshold_before_ll assert [call["stage_name"] for call in client.calls] == ["spoken_word:proposal"] assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold" assert result.report.modules[0].validators[1].candidate_count == 0 + + +def test_process_transcript_result_runs_grammar_module_with_full_validator_chain(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} + ] + """ + ) + base_config = AuditaConfig.from_sources(env={}) + config = AuditaConfig( + api_key=base_config.api_key, + model=base_config.model, + base_url=base_config.base_url, + max_retries=base_config.max_retries, + max_section_tokens=base_config.max_section_tokens, + glossary_confidence_threshold=base_config.glossary_confidence_threshold, + grammar_confidence_threshold=base_config.grammar_confidence_threshold, + homophones_confidence_threshold=base_config.homophones_confidence_threshold, + spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, + normalize_max_segment_gap=base_config.normalize_max_segment_gap, + normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, + normalize_max_segment_duration=base_config.normalize_max_segment_duration, + normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, + work_dir=tmp_path / "work", + work_dir_retention="always", + ) + client = FakeStructuredLLMClient( + [ + { + "corrections": [ + { + "id": 1, + "original_text": "hello world", + "corrected_text": "Hello world.", + "confidence": 0.95, + } + ] + }, + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.99, + "reason": "Does not reverse the segment meaning.", + } + ] + }, + ] + ) + + result = process_transcript_result( + transcript, + _glossary(), + config, + module_keys=["grammar"], + llm_client=client, + ) + + assert result.transcript[0].text == "Hello world." + assert [call["stage_name"] for call in client.calls] == [ + "grammar:proposal", + "grammar:meaning_reversal_review", + ] + assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ + "proposal_confidence_guard", + "protected_glossary_guard", + "grammar_only_guard", + "meaning_reversal_review", + ] + + +def test_process_transcript_result_rejects_grammar_below_threshold_before_later_validators(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} + ] + """ + ) + base_config = AuditaConfig.from_sources(env={}) + config = AuditaConfig( + api_key=base_config.api_key, + model=base_config.model, + base_url=base_config.base_url, + max_retries=base_config.max_retries, + max_section_tokens=base_config.max_section_tokens, + glossary_confidence_threshold=base_config.glossary_confidence_threshold, + grammar_confidence_threshold=0.96, + homophones_confidence_threshold=base_config.homophones_confidence_threshold, + spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, + normalize_max_segment_gap=base_config.normalize_max_segment_gap, + normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, + normalize_max_segment_duration=base_config.normalize_max_segment_duration, + normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, + work_dir=tmp_path / "work", + work_dir_retention="always", + ) + client = FakeStructuredLLMClient( + [ + { + "corrections": [ + { + "id": 1, + "original_text": "hello world", + "corrected_text": "Hello world.", + "confidence": 0.95, + } + ] + } + ] + ) + + result = process_transcript_result( + transcript, + _glossary(), + config, + module_keys=["grammar"], + llm_client=client, + ) + + assert result.transcript[0].text == "hello world" + assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"] + assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold" + assert result.report.modules[0].validators[1].candidate_count == 0 diff --git a/tests/test_new_cli.py b/tests/test_new_cli.py index 3a8422e..4d8824a 100644 --- a/tests/test_new_cli.py +++ b/tests/test_new_cli.py @@ -26,6 +26,7 @@ def test_process_help_exposes_framework_flags(capsys): assert "--max-retries" in output assert "--max-section-tokens" in output assert "--glossary-confidence-threshold" in output + assert "--grammar-confidence-threshold" in output assert "--homophones-confidence-threshold" in output assert "--spoken-word-confidence-threshold" in output assert "--work-dir-retention" in output diff --git a/tests/test_new_config.py b/tests/test_new_config.py index 1f3ee30..2d88e23 100644 --- a/tests/test_new_config.py +++ b/tests/test_new_config.py @@ -4,6 +4,7 @@ from audita.core.config import ( AuditaConfig, ConfigOverrides, DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD, + DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD, DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD, DEFAULT_NORMALIZE_MAX_SEGMENT_GAP, DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD, @@ -19,6 +20,7 @@ def test_default_config_allows_missing_api_key(): assert config.api_key is None assert config.module_keys == DEFAULT_MODULE_KEYS assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD + assert config.grammar_confidence_threshold == DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD assert config.homophones_confidence_threshold == DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD assert config.spoken_word_confidence_threshold == DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP @@ -58,17 +60,20 @@ def test_threshold_overrides_take_precedence(): config = AuditaConfig.from_sources( env={ "AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.6", + "AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.65", "AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.7", "AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.75", }, overrides=ConfigOverrides( glossary_confidence_threshold=0.85, + grammar_confidence_threshold=0.88, homophones_confidence_threshold=0.9, spoken_word_confidence_threshold=0.95, ), ) assert config.glossary_confidence_threshold == 0.85 + assert config.grammar_confidence_threshold == 0.88 assert config.homophones_confidence_threshold == 0.9 assert config.spoken_word_confidence_threshold == 0.95 @@ -90,6 +95,7 @@ def test_invalid_module_sequences_are_rejected(value): "env_name", [ "AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD", + "AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD", "AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD", "AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD", ], diff --git a/tests/test_new_pipeline.py b/tests/test_new_pipeline.py index 51df649..57c3bdb 100644 --- a/tests/test_new_pipeline.py +++ b/tests/test_new_pipeline.py @@ -61,6 +61,7 @@ def test_process_transcript_runs_noop_framework(tmp_path): {"corrections": []}, {"corrections": []}, {"corrections": []}, + {"corrections": []}, ] ) revised = process_transcript( @@ -78,6 +79,7 @@ def test_process_transcript_runs_noop_framework(tmp_path): "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", + "grammar:proposal", ] @@ -93,7 +95,9 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy( max_retries=config.max_retries, max_section_tokens=config.max_section_tokens, glossary_confidence_threshold=config.glossary_confidence_threshold, + grammar_confidence_threshold=config.grammar_confidence_threshold, homophones_confidence_threshold=config.homophones_confidence_threshold, + spoken_word_confidence_threshold=config.spoken_word_confidence_threshold, normalize_max_segment_gap=config.normalize_max_segment_gap, normalize_ellipsis_gap=config.normalize_ellipsis_gap, normalize_max_segment_duration=config.normalize_max_segment_duration, @@ -108,6 +112,7 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy( {"corrections": []}, {"corrections": []}, {"corrections": []}, + {"corrections": []}, ] ) result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client) @@ -135,6 +140,12 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy( "spoken_word_review", "meaning_reversal_review", ] + assert [validator["name"] for validator in result.report.modules[4].to_dict()["validators"]] == [ + "proposal_confidence_guard", + "protected_glossary_guard", + "grammar_only_guard", + "meaning_reversal_review", + ] def test_external_report_can_be_written(tmp_path): @@ -145,6 +156,7 @@ def test_external_report_can_be_written(tmp_path): {"corrections": []}, {"corrections": []}, {"corrections": []}, + {"corrections": []}, ] ) result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client) @@ -191,7 +203,12 @@ def test_default_module_specs_expose_final_validator_order(): "spoken_word_review", "meaning_reversal_review", ] - assert [validator.name for validator in specs[4].module.validators()] == ["protected_glossary_guard"] + assert [validator.name for validator in specs[4].module.validators()] == [ + "proposal_confidence_guard", + "protected_glossary_guard", + "grammar_only_guard", + "meaning_reversal_review", + ] def test_process_transcript_result_missing_api_key_writes_failed_report(tmp_path): @@ -206,7 +223,9 @@ def test_process_transcript_result_missing_api_key_writes_failed_report(tmp_path max_retries=config.max_retries, max_section_tokens=config.max_section_tokens, glossary_confidence_threshold=config.glossary_confidence_threshold, + grammar_confidence_threshold=config.grammar_confidence_threshold, homophones_confidence_threshold=config.homophones_confidence_threshold, + spoken_word_confidence_threshold=config.spoken_word_confidence_threshold, normalize_max_segment_gap=config.normalize_max_segment_gap, normalize_ellipsis_gap=config.normalize_ellipsis_gap, normalize_max_segment_duration=config.normalize_max_segment_duration, @@ -257,7 +276,9 @@ def test_process_transcript_result_preserves_partial_progress_when_later_module_ max_retries=config.max_retries, max_section_tokens=config.max_section_tokens, glossary_confidence_threshold=config.glossary_confidence_threshold, + grammar_confidence_threshold=config.grammar_confidence_threshold, homophones_confidence_threshold=config.homophones_confidence_threshold, + spoken_word_confidence_threshold=config.spoken_word_confidence_threshold, normalize_max_segment_gap=config.normalize_max_segment_gap, normalize_ellipsis_gap=config.normalize_ellipsis_gap, normalize_max_segment_duration=config.normalize_max_segment_duration, @@ -330,7 +351,9 @@ def test_process_transcript_result_preserves_partial_skips_and_validator_diagnos max_retries=config.max_retries, max_section_tokens=config.max_section_tokens, glossary_confidence_threshold=0.8, + grammar_confidence_threshold=config.grammar_confidence_threshold, homophones_confidence_threshold=config.homophones_confidence_threshold, + spoken_word_confidence_threshold=config.spoken_word_confidence_threshold, normalize_max_segment_gap=config.normalize_max_segment_gap, normalize_ellipsis_gap=config.normalize_ellipsis_gap, normalize_max_segment_duration=config.normalize_max_segment_duration, @@ -404,7 +427,7 @@ def test_process_transcript_result_supports_grammar_only_module_override(tmp_pat _glossary(), config, module_keys=["grammar"], - llm_client=FakeStructuredLLMClient([]), + llm_client=FakeStructuredLLMClient([{"corrections": []}]), ) assert [segment.id for segment in result.transcript] == [1, 2]