From 52d29f7228640522bc036a360ebecb671df1c47b Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 25 Apr 2026 08:12:36 -0500 Subject: [PATCH] Implemented the spoken_word LLM review module --- README.md | 9 +- src/audita/cli.py | 6 + src/audita/core/config.py | 12 ++ src/audita/modules/prompts.py | 37 ++++++ src/audita/modules/spoken_word.py | 25 +++- src/audita/validators/__init__.py | 3 +- src/audita/validators/llm.py | 8 +- src/audita/validators/prompts.py | 24 ++++ tests/test_llm_validators.py | 142 +++++++++++++++++++- tests/test_module_proposals.py | 207 +++++++++++++++++++++++++++++- tests/test_new_cli.py | 1 + tests/test_new_config.py | 6 + tests/test_new_pipeline.py | 17 ++- 13 files changed, 484 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 9cb8766..89e1ad4 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Audita is a framework-first transcript correction application. The public `audit - deterministic transcript normalization - token-batched module orchestration -- concrete `glossary` and `homophones` modules built on reusable proposal / validator contracts +- concrete `glossary`, `homophones`, and `spoken_word` modules built on reusable proposal / validator contracts - structured run reporting and work-dir diagnostics The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`. @@ -44,8 +44,8 @@ Resolved run instance names are auto-numbered for repeats, so the default report The default module sequence is partially implemented today: -- `glossary`, `homophones`, and the second `glossary` pass run real LLM-backed proposal and validation stages -- `spoken_word` and `grammar` remain stubs and currently propose no corrections +- `glossary`, `homophones`, the second `glossary` pass, and `spoken_word` run real LLM-backed proposal and validation stages +- `grammar` remains a stub and currently proposes no corrections To run a custom module sequence, pass `--modules`: @@ -77,7 +77,7 @@ audita process transcript.json --glossary glossary.yaml --output corrected.json Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr. `--report-json` writes a separate machine-readable run report and never mixes report data into stdout. -Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Normal runs now require `OPENROUTER_API_KEY`, because the `glossary` and `homophones` modules make real LLM calls. +Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Normal runs now require `OPENROUTER_API_KEY`, because the `glossary`, `homophones`, and `spoken_word` modules make real LLM calls. | Environment variable | CLI flag | Default | Purpose | | --- | --- | --- | --- | @@ -88,6 +88,7 @@ Useful configuration can be supplied by CLI flag or environment variable. CLI fl | `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `6144` | Maximum estimated tokens per transcript batch | | `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation | | `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation | +| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | `--spoken-word-confidence-threshold` | `0.8` | Minimum confidence required for spoken-word proposals to survive validation | | `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging | | `AUDITA_NORMALIZE_ELLIPSIS_GAP` | `--normalize-ellipsis-gap` | `3.5` | Same-speaker gaps above this value are joined with ` ... ` | | `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | `--normalize-max-segment-duration` | `60.0` | Maximum merged segment duration | diff --git a/src/audita/cli.py b/src/audita/cli.py index a1f2aff..3048f7f 100644 --- a/src/audita/cli.py +++ b/src/audita/cli.py @@ -45,6 +45,11 @@ def _build_parser() -> argparse.ArgumentParser: type=float, help="minimum confidence required for homophone proposals to survive validation", ) + process.add_argument( + "--spoken-word-confidence-threshold", + type=float, + help="minimum confidence required for spoken-word proposals to survive validation", + ) process.add_argument( "--normalize-max-segment-gap", type=float, @@ -85,6 +90,7 @@ def _process(args: argparse.Namespace) -> int: max_section_tokens=args.max_section_tokens, glossary_confidence_threshold=args.glossary_confidence_threshold, homophones_confidence_threshold=args.homophones_confidence_threshold, + spoken_word_confidence_threshold=args.spoken_word_confidence_threshold, normalize_max_segment_gap=args.normalize_max_segment_gap, normalize_ellipsis_gap=args.normalize_ellipsis_gap, normalize_max_segment_duration=args.normalize_max_segment_duration, diff --git a/src/audita/core/config.py b/src/audita/core/config.py index f4bb463..6a66db4 100644 --- a/src/audita/core/config.py +++ b/src/audita/core/config.py @@ -15,6 +15,7 @@ DEFAULT_MAX_RETRIES = 3 DEFAULT_MAX_SECTION_TOKENS = 6144 DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD = 0.80 DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD = 0.80 +DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD = 0.80 DEFAULT_WORK_DIR = "/tmp/audita" DEFAULT_WORK_DIR_RETENTION = "auto" DEFAULT_NORMALIZE_MAX_SEGMENT_GAP = 4.0 @@ -32,6 +33,7 @@ class ConfigOverrides: max_section_tokens: Optional[int] = None glossary_confidence_threshold: Optional[float] = None homophones_confidence_threshold: Optional[float] = None + spoken_word_confidence_threshold: Optional[float] = None normalize_max_segment_gap: Optional[float] = None normalize_ellipsis_gap: Optional[float] = None normalize_max_segment_duration: Optional[float] = None @@ -50,6 +52,7 @@ class AuditaConfig: max_section_tokens: int = DEFAULT_MAX_SECTION_TOKENS glossary_confidence_threshold: float = DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD homophones_confidence_threshold: float = DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD + spoken_word_confidence_threshold: float = DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD normalize_max_segment_gap: float = DEFAULT_NORMALIZE_MAX_SEGMENT_GAP normalize_ellipsis_gap: float = DEFAULT_NORMALIZE_ELLIPSIS_GAP normalize_max_segment_duration: float = DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION @@ -99,6 +102,12 @@ class AuditaConfig: DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD, "AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD", ), + spoken_word_confidence_threshold=_select_float( + selected.spoken_word_confidence_threshold, + source.get("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD"), + DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD, + "AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD", + ), normalize_max_segment_gap=_select_float( selected.normalize_max_segment_gap, source.get("AUDITA_NORMALIZE_MAX_SEGMENT_GAP"), @@ -149,6 +158,8 @@ class AuditaConfig: raise AuditaConfigError("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.") if not 0.0 <= self.homophones_confidence_threshold <= 1.0: raise AuditaConfigError("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.") + if not 0.0 <= self.spoken_word_confidence_threshold <= 1.0: + raise AuditaConfigError("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.") if not math.isfinite(self.normalize_max_segment_gap): raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_GAP must be finite.") if not math.isfinite(self.normalize_ellipsis_gap): @@ -180,6 +191,7 @@ class AuditaConfig: "max_section_tokens": self.max_section_tokens, "glossary_confidence_threshold": self.glossary_confidence_threshold, "homophones_confidence_threshold": self.homophones_confidence_threshold, + "spoken_word_confidence_threshold": self.spoken_word_confidence_threshold, "normalize_max_segment_gap": self.normalize_max_segment_gap, "normalize_ellipsis_gap": self.normalize_ellipsis_gap, "normalize_max_segment_duration": self.normalize_max_segment_duration, diff --git a/src/audita/modules/prompts.py b/src/audita/modules/prompts.py index 88ef908..1530b05 100644 --- a/src/audita/modules/prompts.py +++ b/src/audita/modules/prompts.py @@ -85,3 +85,40 @@ def build_homophones_proposal_messages(section: TranscriptSection, glossary: Glo f"Transcript section:\n{section_json}" ) return [{"role": "system", "content": system}, {"role": "user", "content": user}] + + +def build_spoken_word_proposal_messages(section: TranscriptSection, glossary: Glossary) -> List[Message]: + glossary_json = json.dumps(glossary.model_dump(mode="json", exclude_none=True), ensure_ascii=False, indent=2) + section_json = json.dumps(section.prompt_payload(), ensure_ascii=False, indent=2) + + system = ( + "You are Audita, a conservative spoken-word cleanup assistant. " + "Identify only low-risk cleanup of repeated words or short phrases, filler words, hesitation artifacts, " + "and similar dysfluencies that commonly appear in spoken English transcripts. " + "Preserve substantive meaning, named entities, and transcript content." + ) + user = ( + "Review this transcript section and return only spoken-word cleanup corrections that should be applied.\n\n" + "Rules:\n" + "- Approve only conservative cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar spoken dysfluencies.\n" + "- You may collapse adjacent repetition such as \"I I think\" to \"I think\" or remove filler spans such as \"you know\" or \"uh\" when local context supports that cleanup.\n" + "- You may include low-risk punctuation, spacing, or capitalization cleanup when it is part of removing a dysfluency, such as removing ellipses or hesitation punctuation that no longer belongs after the cleanup.\n" + "- Do not paraphrase, summarize, reorder ideas, replace content with different wording, or make substantive semantic edits.\n" + "- Do not change clear content words just because a different phrasing reads better.\n" + "- Treat glossary names and aliases as protected spellings and context.\n" + "- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.\n" + "- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n" + "- Use the exact id from the input segment.\n" + "- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n" + "- Choose an original_text span that appears exactly once in the current segment text.\n" + "- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n" + "- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n" + "- Do not return corrections where original_text and corrected_text are identical.\n" + "- Do not return speaker, start, or end fields.\n" + "- Return only changed segments; do not return entries for unchanged segments.\n" + "- confidence must be between 0.0 and 1.0.\n" + "- If no corrections are needed, return an empty corrections list.\n\n" + f"Protected glossary/context:\n{glossary_json}\n\n" + f"Transcript section:\n{section_json}" + ) + return [{"role": "system", "content": system}, {"role": "user", "content": user}] diff --git a/src/audita/modules/spoken_word.py b/src/audita/modules/spoken_word.py index a771794..94db328 100644 --- a/src/audita/modules/spoken_word.py +++ b/src/audita/modules/spoken_word.py @@ -2,19 +2,36 @@ from typing import Sequence from audita.core.chunking import TranscriptSection from audita.framework.models import CorrectionProposal, ModuleContext -from audita.validators import ProtectedGlossaryTermsValidator, Validator +from audita.framework.proposal_generation import generate_llm_correction_proposals +from audita.modules.prompts import build_spoken_word_proposal_messages +from audita.validators import ( + MeaningReversalValidator, + ProposalConfidenceValidator, + ProtectedGlossaryTermsValidator, + SpokenWordValidator, + Validator, +) class SpokenWordModule: module_key = "spoken_word" - replacement_policy = "replace_all" + replacement_policy = "require_unique" def validators(self) -> Sequence[Validator]: - return [ProtectedGlossaryTermsValidator("protected_glossary_guard")] + return [ + ProposalConfidenceValidator("proposal_confidence_guard", "spoken_word_confidence_threshold"), + ProtectedGlossaryTermsValidator("protected_glossary_guard"), + SpokenWordValidator("spoken_word_review"), + MeaningReversalValidator("meaning_reversal_review"), + ] def propose( self, transcript_section: TranscriptSection, context: ModuleContext, ) -> Sequence[CorrectionProposal]: - return [] + return generate_llm_correction_proposals( + section=transcript_section, + context=context, + prompt_builder=build_spoken_word_proposal_messages, + ) diff --git a/src/audita/validators/__init__.py b/src/audita/validators/__init__.py index ef316b1..19dc727 100644 --- a/src/audita/validators/__init__.py +++ b/src/audita/validators/__init__.py @@ -1,6 +1,6 @@ from .base import ValidationContext, ValidationDecision, ValidationResult, Validator from .deterministic import ProposalConfidenceValidator, ProtectedGlossaryTermsValidator -from .llm import MeaningReversalValidator, SpokenFormPlausibilityValidator +from .llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator from .protection import ProtectedVocabulary __all__ = [ @@ -12,5 +12,6 @@ __all__ = [ "ProtectedGlossaryTermsValidator", "ProtectedVocabulary", "SpokenFormPlausibilityValidator", + "SpokenWordValidator", "MeaningReversalValidator", ] diff --git a/src/audita/validators/llm.py b/src/audita/validators/llm.py index 62ef88d..e9e10c7 100644 --- a/src/audita/validators/llm.py +++ b/src/audita/validators/llm.py @@ -12,7 +12,7 @@ from audita.core.errors import AuditaLLMError from audita.framework.proposals import ProposalPreview, ProposalPreviewError, preview_proposal from .base import ValidationContext, ValidationDecision, ValidationResult -from .prompts import build_meaning_reversal_messages, build_spoken_form_plausibility_messages +from .prompts import build_meaning_reversal_messages, build_spoken_form_plausibility_messages, build_spoken_word_messages class _LLMValidationDecisionModel(BaseModel): @@ -141,5 +141,11 @@ class MeaningReversalValidator(_BaseLLMValidator): prompt_builder: PromptBuilder = build_meaning_reversal_messages +@dataclass(frozen=True) +class SpokenWordValidator(_BaseLLMValidator): + name: str = "spoken_word_review" + prompt_builder: PromptBuilder = build_spoken_word_messages + + def _write_json(path: Path, payload: dict) -> None: path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") diff --git a/src/audita/validators/prompts.py b/src/audita/validators/prompts.py index 5480b0f..417ffac 100644 --- a/src/audita/validators/prompts.py +++ b/src/audita/validators/prompts.py @@ -54,3 +54,27 @@ def build_meaning_reversal_messages(validation_payload: List[dict]) -> List[Mess f"Corrections to validate:\n{payload_json}" ) return [{"role": "system", "content": system}, {"role": "user", "content": user}] + + +def build_spoken_word_messages(validation_payload: List[dict]) -> List[Message]: + payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2) + system = ( + "You are Audita, a conservative spoken-word cleanup validation assistant. " + "Evaluate whether each proposed correction is a reasonable cleanup of repeated words, repeated short phrases, " + "filler words, hesitation artifacts, or similar spoken dysfluencies. " + "Approve only low-risk cleanup that preserves the segment's substantive meaning." + ) + user = ( + "Review these proposed transcript corrections and decide whether each one is a valid spoken-word cleanup.\n\n" + "Rules:\n" + "- Return one validation decision for every correction_index in the input.\n" + "- Approve cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies.\n" + "- Approve minor punctuation, spacing, or capitalization cleanup only when it is plausibly part of removing a dysfluency.\n" + "- Reject free-standing stylistic polishing, readability edits, paraphrases, and general rewriting.\n" + "- Reject edits that materially change the segment's substantive meaning, even if they are not literal antonyms.\n" + "- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n" + "- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" + "- confidence must be between 0.0 and 1.0.\n\n" + f"Corrections to validate:\n{payload_json}" + ) + return [{"role": "system", "content": system}, {"role": "user", "content": user}] diff --git a/tests/test_llm_validators.py b/tests/test_llm_validators.py index c702bbf..f30eab0 100644 --- a/tests/test_llm_validators.py +++ b/tests/test_llm_validators.py @@ -6,10 +6,11 @@ from audita.core.errors import AuditaLLMError from audita.core.schemas import parse_glossary_yaml, parse_transcript_json from audita.framework.models import CorrectionProposal, ModuleRunSpec from audita.validators.base import ValidationContext -from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator +from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator from audita.validators.prompts import ( build_meaning_reversal_messages, build_spoken_form_plausibility_messages, + build_spoken_word_messages, ) import audita.validators.llm as llm_module @@ -204,6 +205,110 @@ def test_meaning_reversal_validator_rejects_reversal_and_approves_nonreversal(tm assert "The figure became invisible in the doorway." in prompt_text +def test_spoken_word_validator_approves_cleanup_and_rejects_rewrite(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."}, + {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "We should maybe proceed carefully."} + ] + """ + ) + proposals = [ + CorrectionProposal( + proposal_index=0, + module_instance="spoken_word", + module_key="spoken_word", + id=1, + original_text="I, uh, I think", + corrected_text="I think", + confidence=0.95, + ), + CorrectionProposal( + proposal_index=1, + module_instance="spoken_word", + module_key="spoken_word", + id=2, + original_text="maybe proceed carefully", + corrected_text="go now", + confidence=0.95, + ), + ] + client = FakeStructuredLLMClient( + [ + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.97, + "reason": "Reasonable dysfluency cleanup that preserves meaning.", + }, + { + "correction_index": 1, + "approved": False, + "confidence": 0.99, + "reason": "This changes the substance of the segment rather than cleaning a dysfluency.", + }, + ] + } + ] + ) + + result = SpokenWordValidator("spoken_word_review").validate( + _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) + ) + + assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [ + (0, True), + (1, False), + ] + prompt_text = client.calls[0]["messages"][1]["content"] + assert "I think we should go." in prompt_text + assert "go now" in prompt_text + + +def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Well ... I think we should go."} + ] + """ + ) + proposals = [ + CorrectionProposal( + proposal_index=0, + module_instance="spoken_word", + module_key="spoken_word", + id=1, + original_text="Well ... ", + corrected_text="", + confidence=0.95, + ) + ] + client = FakeStructuredLLMClient( + [ + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.95, + "reason": "Removes a hesitation artifact without changing substantive meaning.", + } + ] + } + ] + ) + + result = SpokenWordValidator("spoken_word_review").validate( + _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path) + ) + + assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)] + + @pytest.mark.parametrize( ("validator", "payload", "message_fragment"), [ @@ -246,6 +351,20 @@ def test_meaning_reversal_validator_rejects_reversal_and_approves_nonreversal(tm }, "unknown correction_index", ), + ( + SpokenWordValidator("spoken_word_review"), + { + "validations": [ + { + "correction_index": 99, + "approved": True, + "confidence": 0.9, + "reason": "unknown", + } + ] + }, + "unknown correction_index", + ), ], ) def test_llm_validators_reject_bad_correction_indexes(tmp_path, validator, payload, message_fragment): @@ -356,6 +475,27 @@ def test_meaning_reversal_prompt_emphasizes_antonyms_and_segment_context(): assert "original_segment_text" in messages[1]["content"] +def test_spoken_word_prompt_emphasizes_dysfluency_cleanup(): + messages = build_spoken_word_messages( + [ + { + "correction_index": 0, + "id": 1, + "original_segment_text": "I, uh, I think we should go.", + "corrected_segment_text": "I think we should go.", + "original_text": "I, uh, I think", + "corrected_text": "I think", + } + ] + ) + + combined = messages[0]["content"] + messages[1]["content"] + assert "dysfluencies" in combined + assert "punctuation" in combined + assert "substantive meaning" in combined + assert "original_segment_text" in messages[1]["content"] + + def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path): transcript = parse_transcript_json( """ diff --git a/tests/test_module_proposals.py b/tests/test_module_proposals.py index 673e908..3ceccf6 100644 --- a/tests/test_module_proposals.py +++ b/tests/test_module_proposals.py @@ -7,7 +7,8 @@ from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_jso from audita.framework.models import ModuleContext, ModuleRunSpec from audita.modules.glossary import GlossaryModule from audita.modules.homophones import HomophonesModule -from audita.modules.prompts import build_homophones_proposal_messages +from audita.modules.prompts import build_homophones_proposal_messages, build_spoken_word_proposal_messages +from audita.modules.spoken_word import SpokenWordModule from audita.pipeline import process_transcript_result @@ -110,6 +111,70 @@ def test_homophones_prompt_is_explicitly_scoped_to_spoken_form_corrections(): assert '"id": 1' in messages[1]["content"] +def test_spoken_word_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."} + ] + """ + ) + section = chunk_transcript(transcript, max_section_tokens=1000)[0] + module = SpokenWordModule() + client = FakeStructuredLLMClient( + [ + { + "corrections": [ + { + "id": 1, + "original_text": "I, uh, I think", + "corrected_text": "I think", + "confidence": 0.95, + } + ] + } + ] + ) + context = ModuleContext( + run_spec=ModuleRunSpec(instance_name="spoken_word", module_key="spoken_word", module=module), + glossary=_glossary(), + config=AuditaConfig.from_sources(env={}), + run_dir=tmp_path, + llm_client=client, + ) + + proposals = list(module.propose(section, context)) + + assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [ + (1, "I, uh, I think", "I think", 0.95) + ] + assert (tmp_path / "prompt-0000.json").exists() + assert (tmp_path / "corrections-0000.json").exists() + prompt_text = client.calls[0]["messages"][1]["content"] + assert "spoken-word cleanup" in prompt_text + assert "exact text span" in prompt_text + assert "uh" in prompt_text + + +def test_spoken_word_prompt_is_explicitly_scoped_to_dysfluency_cleanup(): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Well ... I think we should go."} + ] + """ + ) + section = chunk_transcript(transcript, max_section_tokens=1000)[0] + + messages = build_spoken_word_proposal_messages(section, _glossary()) + combined = messages[0]["content"] + messages[1]["content"] + + assert "dysfluencies" in combined + assert "punctuation" in combined + assert "paraphrase" in combined + assert '"id": 1' in messages[1]["content"] + + def test_process_transcript_result_uses_injected_fake_client_and_applies_sequential_module_updates(tmp_path): transcript = parse_source_transcript_json( """ @@ -200,6 +265,7 @@ def test_process_transcript_result_uses_injected_fake_client_and_applies_sequent ] }, {"corrections": []}, + {"corrections": []}, ] ) @@ -214,6 +280,7 @@ def test_process_transcript_result_uses_injected_fake_client_and_applies_sequent "homophones:spoken_form_plausibility_review", "homophones:meaning_reversal_review", "glossary_2:proposal", + "spoken_word:proposal", ] assert "There were Jesters at the dam." in client.calls[3]["messages"][1]["content"] @@ -256,6 +323,7 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_ }, {"corrections": []}, {"corrections": []}, + {"corrections": []}, ] ) @@ -266,8 +334,145 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_ "glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", + "spoken_word:proposal", ] assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold" assert result.report.skipped_corrections[0].source == "validator:proposal_confidence_guard" assert result.report.modules[0].validators[0].rejected_count == 1 assert result.report.modules[0].validators[1].candidate_count == 0 + + +def test_process_transcript_result_runs_spoken_word_module_with_full_validator_chain(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."} + ] + """ + ) + base_config = AuditaConfig.from_sources(env={}) + config = AuditaConfig( + api_key=base_config.api_key, + model=base_config.model, + base_url=base_config.base_url, + max_retries=base_config.max_retries, + max_section_tokens=base_config.max_section_tokens, + glossary_confidence_threshold=base_config.glossary_confidence_threshold, + homophones_confidence_threshold=base_config.homophones_confidence_threshold, + spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, + normalize_max_segment_gap=base_config.normalize_max_segment_gap, + normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, + normalize_max_segment_duration=base_config.normalize_max_segment_duration, + normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, + work_dir=tmp_path / "work", + work_dir_retention="always", + ) + client = FakeStructuredLLMClient( + [ + { + "corrections": [ + { + "id": 1, + "original_text": "I, uh, I think", + "corrected_text": "I think", + "confidence": 0.95, + } + ] + }, + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.97, + "reason": "Reasonable dysfluency cleanup that preserves meaning.", + } + ] + }, + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.99, + "reason": "Does not reverse the segment meaning.", + } + ] + }, + ] + ) + + result = process_transcript_result( + transcript, + _glossary(), + config, + module_keys=["spoken_word"], + llm_client=client, + ) + + assert result.transcript[0].text == "I think we should go." + assert [call["stage_name"] for call in client.calls] == [ + "spoken_word:proposal", + "spoken_word:spoken_word_review", + "spoken_word:meaning_reversal_review", + ] + assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ + "proposal_confidence_guard", + "protected_glossary_guard", + "spoken_word_review", + "meaning_reversal_review", + ] + + +def test_process_transcript_result_rejects_spoken_word_below_threshold_before_llm_validators(tmp_path): + transcript = parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."} + ] + """ + ) + base_config = AuditaConfig.from_sources(env={}) + config = AuditaConfig( + api_key=base_config.api_key, + model=base_config.model, + base_url=base_config.base_url, + max_retries=base_config.max_retries, + max_section_tokens=base_config.max_section_tokens, + glossary_confidence_threshold=base_config.glossary_confidence_threshold, + homophones_confidence_threshold=base_config.homophones_confidence_threshold, + spoken_word_confidence_threshold=0.96, + normalize_max_segment_gap=base_config.normalize_max_segment_gap, + normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, + normalize_max_segment_duration=base_config.normalize_max_segment_duration, + normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, + work_dir=tmp_path / "work", + work_dir_retention="always", + ) + client = FakeStructuredLLMClient( + [ + { + "corrections": [ + { + "id": 1, + "original_text": "I, uh, I think", + "corrected_text": "I think", + "confidence": 0.95, + } + ] + } + ] + ) + + result = process_transcript_result( + transcript, + _glossary(), + config, + module_keys=["spoken_word"], + llm_client=client, + ) + + assert result.transcript[0].text == "I, uh, I think we should go." + assert [call["stage_name"] for call in client.calls] == ["spoken_word:proposal"] + assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold" + assert result.report.modules[0].validators[1].candidate_count == 0 diff --git a/tests/test_new_cli.py b/tests/test_new_cli.py index da37f90..3a8422e 100644 --- a/tests/test_new_cli.py +++ b/tests/test_new_cli.py @@ -27,6 +27,7 @@ def test_process_help_exposes_framework_flags(capsys): assert "--max-section-tokens" in output assert "--glossary-confidence-threshold" in output assert "--homophones-confidence-threshold" in output + assert "--spoken-word-confidence-threshold" in output assert "--work-dir-retention" in output assert "--normalize-max-segment-gap" in output assert "--grammar-validation-enabled" not in output diff --git a/tests/test_new_config.py b/tests/test_new_config.py index ffec401..1f3ee30 100644 --- a/tests/test_new_config.py +++ b/tests/test_new_config.py @@ -6,6 +6,7 @@ from audita.core.config import ( DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD, DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD, DEFAULT_NORMALIZE_MAX_SEGMENT_GAP, + DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD, DEFAULT_WORK_DIR_RETENTION, ) from audita.core.errors import AuditaConfigError @@ -19,6 +20,7 @@ def test_default_config_allows_missing_api_key(): assert config.module_keys == DEFAULT_MODULE_KEYS assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD assert config.homophones_confidence_threshold == DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD + assert config.spoken_word_confidence_threshold == DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP assert config.work_dir_retention == DEFAULT_WORK_DIR_RETENTION @@ -57,15 +59,18 @@ def test_threshold_overrides_take_precedence(): env={ "AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.6", "AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.7", + "AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.75", }, overrides=ConfigOverrides( glossary_confidence_threshold=0.85, homophones_confidence_threshold=0.9, + spoken_word_confidence_threshold=0.95, ), ) assert config.glossary_confidence_threshold == 0.85 assert config.homophones_confidence_threshold == 0.9 + assert config.spoken_word_confidence_threshold == 0.95 @pytest.mark.parametrize( @@ -86,6 +91,7 @@ def test_invalid_module_sequences_are_rejected(value): [ "AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD", "AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD", + "AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD", ], ) def test_invalid_thresholds_are_rejected(env_name): diff --git a/tests/test_new_pipeline.py b/tests/test_new_pipeline.py index a85516e..51df649 100644 --- a/tests/test_new_pipeline.py +++ b/tests/test_new_pipeline.py @@ -60,6 +60,7 @@ def test_process_transcript_runs_noop_framework(tmp_path): {"corrections": []}, {"corrections": []}, {"corrections": []}, + {"corrections": []}, ] ) revised = process_transcript( @@ -76,6 +77,7 @@ def test_process_transcript_runs_noop_framework(tmp_path): "glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", + "spoken_word:proposal", ] @@ -105,6 +107,7 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy( {"corrections": []}, {"corrections": []}, {"corrections": []}, + {"corrections": []}, ] ) result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client) @@ -126,6 +129,12 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy( "spoken_form_plausibility_review", "meaning_reversal_review", ] + assert [validator["name"] for validator in result.report.modules[3].to_dict()["validators"]] == [ + "proposal_confidence_guard", + "protected_glossary_guard", + "spoken_word_review", + "meaning_reversal_review", + ] def test_external_report_can_be_written(tmp_path): @@ -135,6 +144,7 @@ def test_external_report_can_be_written(tmp_path): {"corrections": []}, {"corrections": []}, {"corrections": []}, + {"corrections": []}, ] ) result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client) @@ -175,7 +185,12 @@ def test_default_module_specs_expose_final_validator_order(): "spoken_form_plausibility_review", "meaning_reversal_review", ] - assert [validator.name for validator in specs[3].module.validators()] == ["protected_glossary_guard"] + assert [validator.name for validator in specs[3].module.validators()] == [ + "proposal_confidence_guard", + "protected_glossary_guard", + "spoken_word_review", + "meaning_reversal_review", + ] assert [validator.name for validator in specs[4].module.validators()] == ["protected_glossary_guard"]