import threading from pathlib import Path from audita.core.chunking import chunk_transcript from audita.core.config import AuditaConfig, ConfigOverrides from audita.core.errors import AuditaLLMError from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json, parse_transcript_json from audita.framework.models import ModuleContext, ModuleRunSpec from audita.modules.grammar import GrammarModule from audita.modules.glossary import GlossaryModule from audita.modules.homophones import HomophonesModule from audita.modules.prompts import ( build_grammar_proposal_messages, build_homophones_proposal_messages, build_spoken_word_proposal_messages, ) from audita.modules.spoken_word import SpokenWordModule from audita.pipeline import process_transcript_result class FakeStructuredLLMClient: def __init__(self, responses): self._responses = responses self._lock = threading.Lock() self.calls = [] def run_structured(self, *, stage_name, messages, response_model, config): with self._lock: self.calls.append( { "stage_name": stage_name, "messages": list(messages), "response_model": response_model, "config": config, } ) payload = _pop_llm_response(self._responses, stage_name) return response_model.model_validate(payload) def _pop_llm_response(responses, stage_name): if isinstance(responses, dict): if stage_name not in responses: raise AuditaLLMError(f"FakeStructuredLLMClient received unexpected stage_name: {stage_name}") payloads = responses[stage_name] if isinstance(payloads, list): if not payloads: raise AuditaLLMError(f"FakeStructuredLLMClient received too many calls for stage_name: {stage_name}") return payloads.pop(0) payload = payloads del responses[stage_name] return payload if not responses: raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.") return responses.pop(0) def _glossary(): return parse_glossary_yaml( """ glossary: - name: "Jesters" aliases: - "Jester" category: faction summary: "A faction." - name: "Hrank" category: pc summary: "A player character." """ ) def test_glossary_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} ] """ ) section = chunk_transcript(transcript, max_section_tokens=1000)[0] module = GlossaryModule() client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95, } ] } ] ) context = ModuleContext( run_spec=ModuleRunSpec(instance_name="glossary_primary", module_key="glossary", module=module), glossary=_glossary(), config=AuditaConfig.from_sources(env={}), run_dir=tmp_path, llm_client=client, ) proposals = list(module.propose(section, context)) assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [ (1, "gestures", "Jesters", 0.95) ] assert (tmp_path / "prompt-0000.json").exists() assert (tmp_path / "corrections-0000.json").exists() prompt_text = client.calls[0]["messages"][1]["content"] assert "Glossary:" in prompt_text assert "exact text span" in prompt_text assert "gestures" in prompt_text def test_module_propose_uses_primary_llm_config(tmp_path): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} ] """ ) section = chunk_transcript(transcript, max_section_tokens=1000)[0] module = GlossaryModule() client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95, } ] } ] ) config = AuditaConfig.from_sources( env={"OPENROUTER_API_KEY": "primary-key"}, overrides=ConfigOverrides( model="primary-model", base_url="http://localhost:8000/v1", max_retries=7, llm_timeout_seconds=120, validation_llm_api_key="validation-key", validation_model="validation-model", validation_base_url="http://localhost:9000/v1", validation_max_retries=2, validation_llm_timeout_seconds=240, validation_llm_concurrency=3, ), ) context = ModuleContext( run_spec=ModuleRunSpec(instance_name="glossary_primary", module_key="glossary", module=module), glossary=_glossary(), config=config, run_dir=tmp_path, llm_client=client, ) module.propose(section, context) proposal_config = client.calls[0]["config"] assert proposal_config.api_key == "primary-key" assert proposal_config.model == "primary-model" assert proposal_config.base_url == "http://localhost:8000/v1" assert proposal_config.max_retries == 7 assert proposal_config.llm_timeout_seconds == 120 def test_homophones_prompt_is_explicitly_scoped_to_spoken_form_corrections(): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't do that with a dam."} ] """ ) section = chunk_transcript(transcript, max_section_tokens=1000)[0] messages = build_homophones_proposal_messages(section, _glossary()) combined = messages[0]["content"] + messages[1]["content"] assert "homophone" in combined assert "mistranscription" in combined assert "Do not add or remove punctuation" in combined assert "visible" in combined assert '"id": 1' in messages[1]["content"] def test_spoken_word_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."} ] """ ) section = chunk_transcript(transcript, max_section_tokens=1000)[0] module = SpokenWordModule() client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "I, uh, I think", "corrected_text": "I think", "confidence": 0.95, } ] } ] ) context = ModuleContext( run_spec=ModuleRunSpec(instance_name="spoken_word", module_key="spoken_word", module=module), glossary=_glossary(), config=AuditaConfig.from_sources(env={}), run_dir=tmp_path, llm_client=client, ) proposals = list(module.propose(section, context)) assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [ (1, "I, uh, I think", "I think", 0.95) ] assert (tmp_path / "prompt-0000.json").exists() assert (tmp_path / "corrections-0000.json").exists() prompt_text = client.calls[0]["messages"][1]["content"] assert "spoken-word cleanup" in prompt_text assert "exact text span" in prompt_text assert "uh" in prompt_text def test_spoken_word_prompt_is_explicitly_scoped_to_dysfluency_cleanup(): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Well ... I think we should go."} ] """ ) section = chunk_transcript(transcript, max_section_tokens=1000)[0] messages = build_spoken_word_proposal_messages(section, _glossary()) combined = messages[0]["content"] + messages[1]["content"] assert "dysfluencies" in combined assert "intentional emphasis" in combined assert "Help! Help! Help!" in combined assert "punctuation" in combined assert "paraphrase" in combined assert '"id": 1' in messages[1]["content"] def test_grammar_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} ] """ ) section = chunk_transcript(transcript, max_section_tokens=1000)[0] module = GrammarModule() client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "hello world", "corrected_text": "Hello world.", "confidence": 0.95, } ] } ] ) context = ModuleContext( run_spec=ModuleRunSpec(instance_name="grammar", module_key="grammar", module=module), glossary=_glossary(), config=AuditaConfig.from_sources(env={}), run_dir=tmp_path, llm_client=client, ) proposals = list(module.propose(section, context)) assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [ (1, "hello world", "Hello world.", 0.95) ] assert (tmp_path / "prompt-0000.json").exists() assert (tmp_path / "corrections-0000.json").exists() prompt_text = client.calls[0]["messages"][1]["content"] assert "punctuation, capitalization, spacing, and article cleanup" in prompt_text assert "exact text span" in prompt_text assert "word substitutions" in prompt_text assert "later review stage" in prompt_text def test_grammar_prompt_is_explicitly_scoped_to_formatting_cleanup(): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} ] """ ) section = chunk_transcript(transcript, max_section_tokens=1000)[0] messages = build_grammar_proposal_messages(section, _glossary()) combined = messages[0]["content"] + messages[1]["content"] assert "punctuation, capitalization, spacing, and article cleanup" in combined assert "word substitutions" in combined assert "homophone fixes" in combined assert "later review stage" in combined assert "mistranscription" in combined assert '"id": 1' in messages[1]["content"] def test_process_transcript_result_uses_injected_fake_client_and_applies_sequential_module_updates(tmp_path): transcript = parse_source_transcript_json( """ [ {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."} ] """ ) config = AuditaConfig.from_sources( env={}, overrides=None, ) config = AuditaConfig( api_key=config.api_key, model=config.model, base_url=config.base_url, max_retries=config.max_retries, max_section_tokens=config.max_section_tokens, glossary_confidence_threshold=config.glossary_confidence_threshold, grammar_confidence_threshold=config.grammar_confidence_threshold, homophones_confidence_threshold=config.homophones_confidence_threshold, spoken_word_confidence_threshold=config.spoken_word_confidence_threshold, normalize_max_segment_gap=config.normalize_max_segment_gap, normalize_ellipsis_gap=config.normalize_ellipsis_gap, normalize_max_segment_duration=config.normalize_max_segment_duration, normalize_max_segment_tokens=config.normalize_max_segment_tokens, work_dir=tmp_path / "work", work_dir_retention="always", ) client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95, } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.97, "reason": "Likely spoken-form correction in context.", } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.99, "reason": "Does not reverse the segment meaning.", } ] }, { "corrections": [ { "id": 1, "original_text": "dam", "corrected_text": "damn", "confidence": 0.92, } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.98, "reason": "Likely spoken-form correction in context.", } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.99, "reason": "Does not reverse the segment meaning.", } ] }, {"corrections": []}, {"corrections": []}, {"corrections": []}, ] ) result = process_transcript_result(transcript, _glossary(), config, llm_client=client) assert result.transcript[0].text == "There were Jesters at the damn." assert [call["stage_name"] for call in client.calls] == [ "glossary_1:proposal", "glossary_1:spoken_form_plausibility_review", "glossary_1:meaning_reversal_review", "homophones:proposal", "homophones:spoken_form_plausibility_review", "homophones:meaning_reversal_review", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal", ] assert "There were Jesters at the dam." in client.calls[3]["messages"][1]["content"] def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_validators(tmp_path): transcript = parse_source_transcript_json( """ [ {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} ] """ ) base_config = AuditaConfig.from_sources(env={}) config = AuditaConfig( api_key=base_config.api_key, model=base_config.model, base_url=base_config.base_url, max_retries=base_config.max_retries, max_section_tokens=base_config.max_section_tokens, glossary_confidence_threshold=0.96, grammar_confidence_threshold=base_config.grammar_confidence_threshold, homophones_confidence_threshold=base_config.homophones_confidence_threshold, spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, normalize_max_segment_gap=base_config.normalize_max_segment_gap, normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, normalize_max_segment_duration=base_config.normalize_max_segment_duration, normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, work_dir=tmp_path / "work", work_dir_retention="always", ) client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95, } ] }, {"corrections": []}, {"corrections": []}, {"corrections": []}, {"corrections": []}, ] ) result = process_transcript_result(transcript, _glossary(), config, llm_client=client) assert result.transcript[0].text == "There were gestures at the temple." assert [call["stage_name"] for call in client.calls] == [ "glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", "grammar:proposal", ] assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold" assert result.report.skipped_corrections[0].source == "validator:proposal_confidence_guard" assert result.report.modules[0].validators[2].rejected_count == 1 assert result.report.modules[0].validators[3].candidate_count == 0 def test_process_transcript_result_runs_spoken_word_module_with_full_validator_chain(tmp_path): transcript = parse_source_transcript_json( """ [ {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."} ] """ ) base_config = AuditaConfig.from_sources(env={}) config = AuditaConfig( api_key=base_config.api_key, model=base_config.model, base_url=base_config.base_url, max_retries=base_config.max_retries, max_section_tokens=base_config.max_section_tokens, glossary_confidence_threshold=base_config.glossary_confidence_threshold, grammar_confidence_threshold=base_config.grammar_confidence_threshold, homophones_confidence_threshold=base_config.homophones_confidence_threshold, spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, normalize_max_segment_gap=base_config.normalize_max_segment_gap, normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, normalize_max_segment_duration=base_config.normalize_max_segment_duration, normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, work_dir=tmp_path / "work", work_dir_retention="always", ) client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "I, uh, I think", "corrected_text": "I think", "confidence": 0.95, } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.97, "reason": "Reasonable dysfluency cleanup that preserves meaning.", } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.99, "reason": "Does not reverse the segment meaning.", } ] }, ] ) result = process_transcript_result( transcript, _glossary(), config, module_keys=["spoken_word"], llm_client=client, ) assert result.transcript[0].text == "I think we should go." assert [call["stage_name"] for call in client.calls] == [ "spoken_word:proposal", "spoken_word:spoken_word_review", "spoken_word:meaning_reversal_review", ] assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ "identical_text_guard", "original_text_present_guard", "proposal_confidence_guard", "protected_glossary_guard", "non_empty_segment_guard", "spoken_word_review", "meaning_reversal_review", ] def test_process_transcript_result_rejects_spoken_word_below_threshold_before_llm_validators(tmp_path): transcript = parse_source_transcript_json( """ [ {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."} ] """ ) base_config = AuditaConfig.from_sources(env={}) config = AuditaConfig( api_key=base_config.api_key, model=base_config.model, base_url=base_config.base_url, max_retries=base_config.max_retries, max_section_tokens=base_config.max_section_tokens, glossary_confidence_threshold=base_config.glossary_confidence_threshold, grammar_confidence_threshold=base_config.grammar_confidence_threshold, homophones_confidence_threshold=base_config.homophones_confidence_threshold, spoken_word_confidence_threshold=0.96, normalize_max_segment_gap=base_config.normalize_max_segment_gap, normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, normalize_max_segment_duration=base_config.normalize_max_segment_duration, normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, work_dir=tmp_path / "work", work_dir_retention="always", ) client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "I, uh, I think", "corrected_text": "I think", "confidence": 0.95, } ] } ] ) result = process_transcript_result( transcript, _glossary(), config, module_keys=["spoken_word"], llm_client=client, ) assert result.transcript[0].text == "I, uh, I think we should go." assert [call["stage_name"] for call in client.calls] == ["spoken_word:proposal"] assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold" assert result.report.modules[0].validators[3].candidate_count == 0 def test_process_transcript_result_runs_grammar_module_with_full_validator_chain(tmp_path): transcript = parse_source_transcript_json( """ [ {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} ] """ ) base_config = AuditaConfig.from_sources(env={}) config = AuditaConfig( api_key=base_config.api_key, model=base_config.model, base_url=base_config.base_url, max_retries=base_config.max_retries, max_section_tokens=base_config.max_section_tokens, glossary_confidence_threshold=base_config.glossary_confidence_threshold, grammar_confidence_threshold=base_config.grammar_confidence_threshold, homophones_confidence_threshold=base_config.homophones_confidence_threshold, spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, normalize_max_segment_gap=base_config.normalize_max_segment_gap, normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, normalize_max_segment_duration=base_config.normalize_max_segment_duration, normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, work_dir=tmp_path / "work", work_dir_retention="always", ) client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "hello world", "corrected_text": "Hello world.", "confidence": 0.95, } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.98, "reason": "Conservative grammar cleanup.", } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.99, "reason": "Does not reverse the segment meaning.", } ] }, ] ) result = process_transcript_result( transcript, _glossary(), config, module_keys=["grammar"], llm_client=client, ) assert result.transcript[0].text == "Hello world." assert [call["stage_name"] for call in client.calls] == [ "grammar:proposal", "grammar:grammar_only_guard", "grammar:meaning_reversal_review", ] assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ "identical_text_guard", "original_text_present_guard", "proposal_confidence_guard", "protected_glossary_guard", "non_empty_segment_guard", "grammar_only_guard", "meaning_reversal_review", ] def test_process_transcript_result_grammar_module_applies_indefinite_article_cleanup(tmp_path): transcript = parse_source_transcript_json( """ [ {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Give me a intelligence saving throw."} ] """ ) base_config = AuditaConfig.from_sources(env={}) config = AuditaConfig( api_key=base_config.api_key, model=base_config.model, base_url=base_config.base_url, max_retries=base_config.max_retries, max_section_tokens=base_config.max_section_tokens, glossary_confidence_threshold=base_config.glossary_confidence_threshold, grammar_confidence_threshold=base_config.grammar_confidence_threshold, homophones_confidence_threshold=base_config.homophones_confidence_threshold, spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, normalize_max_segment_gap=base_config.normalize_max_segment_gap, normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, normalize_max_segment_duration=base_config.normalize_max_segment_duration, normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, work_dir=tmp_path / "work", work_dir_retention="always", ) client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "a intelligence saving throw", "corrected_text": "an intelligence saving throw", "confidence": 0.95, } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.97, "reason": "Allowed article cleanup.", } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.99, "reason": "Does not reverse the segment meaning.", } ] }, ] ) result = process_transcript_result( transcript, _glossary(), config, module_keys=["grammar"], llm_client=client, ) assert result.transcript[0].text == "Give me an intelligence saving throw." assert [call["stage_name"] for call in client.calls] == [ "grammar:proposal", "grammar:grammar_only_guard", "grammar:meaning_reversal_review", ] def test_process_transcript_result_rejects_grammar_below_threshold_before_later_validators(tmp_path): transcript = parse_source_transcript_json( """ [ {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} ] """ ) base_config = AuditaConfig.from_sources(env={}) config = AuditaConfig( api_key=base_config.api_key, model=base_config.model, base_url=base_config.base_url, max_retries=base_config.max_retries, max_section_tokens=base_config.max_section_tokens, glossary_confidence_threshold=base_config.glossary_confidence_threshold, grammar_confidence_threshold=0.96, homophones_confidence_threshold=base_config.homophones_confidence_threshold, spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, normalize_max_segment_gap=base_config.normalize_max_segment_gap, normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, normalize_max_segment_duration=base_config.normalize_max_segment_duration, normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, work_dir=tmp_path / "work", work_dir_retention="always", ) client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "hello world", "corrected_text": "Hello world.", "confidence": 0.95, } ] } ] ) result = process_transcript_result( transcript, _glossary(), config, module_keys=["grammar"], llm_client=client, ) assert result.transcript[0].text == "hello world" assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"] assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold" assert result.report.modules[0].validators[3].candidate_count == 0 def test_process_transcript_result_grammar_module_allows_editorial_homophone_style_proposals(tmp_path): transcript = parse_source_transcript_json( """ [ {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."} ] """ ) base_config = AuditaConfig.from_sources(env={}) config = AuditaConfig( api_key=base_config.api_key, model=base_config.model, base_url=base_config.base_url, max_retries=base_config.max_retries, max_section_tokens=base_config.max_section_tokens, glossary_confidence_threshold=base_config.glossary_confidence_threshold, grammar_confidence_threshold=base_config.grammar_confidence_threshold, homophones_confidence_threshold=base_config.homophones_confidence_threshold, spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, normalize_max_segment_gap=base_config.normalize_max_segment_gap, normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, normalize_max_segment_duration=base_config.normalize_max_segment_duration, normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, work_dir=tmp_path / "work", work_dir_retention="always", ) client = FakeStructuredLLMClient( { "grammar:proposal": { "corrections": [ { "id": 1, "original_text": "dam", "corrected_text": "damn", "confidence": 0.95, } ] }, "grammar:grammar_only_guard": { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.94, "reason": "A low-risk editorial homophone correction that preserves meaning.", } ] }, "grammar:meaning_reversal_review": { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.99, "reason": "Does not reverse the segment meaning.", } ] }, } ) result = process_transcript_result( transcript, _glossary(), config, module_keys=["grammar"], llm_client=client, ) assert result.transcript[0].text == "ChatGPT still can't really do that with a damn." assert [call["stage_name"] for call in client.calls] == [ "grammar:proposal", "grammar:grammar_only_guard", "grammar:meaning_reversal_review", ] assert result.report.skipped_corrections == [] def test_process_transcript_result_grammar_module_allows_embedded_homophone_fix_with_grammar_cleanup(tmp_path): transcript = parse_source_transcript_json( """ [ {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't"} ] """ ) base_config = AuditaConfig.from_sources(env={}) config = AuditaConfig( api_key=base_config.api_key, llm_concurrency=base_config.llm_concurrency, module_keys=base_config.module_keys, model=base_config.model, base_url=base_config.base_url, max_retries=base_config.max_retries, max_section_tokens=base_config.max_section_tokens, glossary_confidence_threshold=base_config.glossary_confidence_threshold, grammar_confidence_threshold=base_config.grammar_confidence_threshold, homophones_confidence_threshold=base_config.homophones_confidence_threshold, spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, normalize_max_segment_gap=base_config.normalize_max_segment_gap, normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, normalize_max_segment_duration=base_config.normalize_max_segment_duration, normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, work_dir=tmp_path / "work", work_dir_retention="always", ) client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't", "corrected_text": "question: If he dies, does he stay there? The way that, yeah, the way that it's written, it's like, so if it stops, then he goes down; but what if it doesn't?", "confidence": 0.95, } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.95, "reason": "Primarily a grammatical revision with an embedded likely mistranscription recovery.", } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.99, "reason": "Does not reverse the segment meaning.", } ] }, ] ) result = process_transcript_result( transcript, _glossary(), config, module_keys=["grammar"], llm_client=client, ) assert result.transcript[0].text == ( "question: If he dies, does he stay there? The way that, yeah, the way that it's written, " "it's like, so if it stops, then he goes down; but what if it doesn't?" ) assert [call["stage_name"] for call in client.calls] == [ "grammar:proposal", "grammar:grammar_only_guard", "grammar:meaning_reversal_review", ] def test_process_transcript_result_rejects_spoken_word_whole_segment_deletion_before_llm_validators(tmp_path): transcript = parse_source_transcript_json( """ [ {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "uh"} ] """ ) base_config = AuditaConfig.from_sources(env={}) config = AuditaConfig( api_key=base_config.api_key, model=base_config.model, base_url=base_config.base_url, max_retries=base_config.max_retries, max_section_tokens=base_config.max_section_tokens, glossary_confidence_threshold=base_config.glossary_confidence_threshold, grammar_confidence_threshold=base_config.grammar_confidence_threshold, homophones_confidence_threshold=base_config.homophones_confidence_threshold, spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, normalize_max_segment_gap=base_config.normalize_max_segment_gap, normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, normalize_max_segment_duration=base_config.normalize_max_segment_duration, normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, work_dir=tmp_path / "work", work_dir_retention="always", ) client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "uh", "corrected_text": "", "confidence": 0.95, } ] } ] ) result = process_transcript_result( transcript, _glossary(), config, module_keys=["spoken_word"], llm_client=client, ) assert result.transcript[0].text == "uh" assert [call["stage_name"] for call in client.calls] == ["spoken_word:proposal"] assert result.report.skipped_corrections[0].source == "validator:non_empty_segment_guard" assert result.report.skipped_corrections[0].reason == "correction would leave the segment empty" assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ "identical_text_guard", "original_text_present_guard", "proposal_confidence_guard", "protected_glossary_guard", "non_empty_segment_guard", "spoken_word_review", "meaning_reversal_review", ] assert result.report.modules[0].validators[4].rejected_count == 1 assert result.report.modules[0].validators[5].candidate_count == 0 def test_process_transcript_result_rejects_identical_text_before_later_validators(tmp_path): transcript = parse_source_transcript_json( """ [ {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} ] """ ) config = AuditaConfig.from_sources( env={}, overrides=ConfigOverrides(work_dir=tmp_path / "work", work_dir_retention="always"), ) client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "hello", "corrected_text": "hello", "confidence": 0.95, } ] } ] ) result = process_transcript_result( transcript, _glossary(), config, module_keys=["grammar"], llm_client=client, ) assert result.transcript[0].text == "hello world" assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"] assert result.report.skipped_corrections[0].source == "validator:identical_text_guard" assert result.report.skipped_corrections[0].reason == "proposal original_text and corrected_text are identical" assert result.report.modules[0].validators[0].rejected_count == 1 assert result.report.modules[0].validators[1].candidate_count == 0 def test_process_transcript_result_rejects_missing_original_text_before_later_validators(tmp_path): transcript = parse_source_transcript_json( """ [ {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} ] """ ) config = AuditaConfig.from_sources( env={}, overrides=ConfigOverrides(work_dir=tmp_path / "work", work_dir_retention="always"), ) client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "goodbye", "corrected_text": "farewell", "confidence": 0.95, } ] } ] ) result = process_transcript_result( transcript, _glossary(), config, module_keys=["grammar"], llm_client=client, ) assert result.transcript[0].text == "hello world" assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"] assert result.report.skipped_corrections[0].source == "validator:original_text_present_guard" assert result.report.skipped_corrections[0].reason == "proposal original_text does not match segment text" assert result.report.modules[0].validators[0].approved_count == 1 assert result.report.modules[0].validators[1].rejected_count == 1 assert result.report.modules[0].validators[2].candidate_count == 0