from pathlib import Path from audita.core.chunking import chunk_transcript from audita.core.config import AuditaConfig from audita.core.errors import AuditaLLMError from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json, parse_transcript_json from audita.framework.models import ModuleContext, ModuleRunSpec from audita.modules.glossary import GlossaryModule from audita.modules.homophones import HomophonesModule from audita.modules.prompts import build_homophones_proposal_messages, build_spoken_word_proposal_messages from audita.modules.spoken_word import SpokenWordModule from audita.pipeline import process_transcript_result class FakeStructuredLLMClient: def __init__(self, responses): self._responses = list(responses) self.calls = [] def run_structured(self, *, stage_name, messages, response_model, config): self.calls.append( { "stage_name": stage_name, "messages": list(messages), "response_model": response_model, } ) if not self._responses: raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.") return response_model.model_validate(self._responses.pop(0)) def _glossary(): return parse_glossary_yaml( """ glossary: - name: "Jesters" aliases: - "Jester" category: faction summary: "A faction." - name: "Hrank" category: pc summary: "A player character." """ ) def test_glossary_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} ] """ ) section = chunk_transcript(transcript, max_section_tokens=1000)[0] module = GlossaryModule() client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95, } ] } ] ) context = ModuleContext( run_spec=ModuleRunSpec(instance_name="glossary_primary", module_key="glossary", module=module), glossary=_glossary(), config=AuditaConfig.from_sources(env={}), run_dir=tmp_path, llm_client=client, ) proposals = list(module.propose(section, context)) assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [ (1, "gestures", "Jesters", 0.95) ] assert (tmp_path / "prompt-0000.json").exists() assert (tmp_path / "corrections-0000.json").exists() prompt_text = client.calls[0]["messages"][1]["content"] assert "Glossary:" in prompt_text assert "exact text span" in prompt_text assert "gestures" in prompt_text def test_homophones_prompt_is_explicitly_scoped_to_spoken_form_corrections(): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't do that with a dam."} ] """ ) section = chunk_transcript(transcript, max_section_tokens=1000)[0] messages = build_homophones_proposal_messages(section, _glossary()) combined = messages[0]["content"] + messages[1]["content"] assert "homophone" in combined assert "mistranscription" in combined assert "Do not add or remove punctuation" in combined assert "visible" in combined assert '"id": 1' in messages[1]["content"] def test_spoken_word_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."} ] """ ) section = chunk_transcript(transcript, max_section_tokens=1000)[0] module = SpokenWordModule() client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "I, uh, I think", "corrected_text": "I think", "confidence": 0.95, } ] } ] ) context = ModuleContext( run_spec=ModuleRunSpec(instance_name="spoken_word", module_key="spoken_word", module=module), glossary=_glossary(), config=AuditaConfig.from_sources(env={}), run_dir=tmp_path, llm_client=client, ) proposals = list(module.propose(section, context)) assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [ (1, "I, uh, I think", "I think", 0.95) ] assert (tmp_path / "prompt-0000.json").exists() assert (tmp_path / "corrections-0000.json").exists() prompt_text = client.calls[0]["messages"][1]["content"] assert "spoken-word cleanup" in prompt_text assert "exact text span" in prompt_text assert "uh" in prompt_text def test_spoken_word_prompt_is_explicitly_scoped_to_dysfluency_cleanup(): transcript = parse_transcript_json( """ [ {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Well ... I think we should go."} ] """ ) section = chunk_transcript(transcript, max_section_tokens=1000)[0] messages = build_spoken_word_proposal_messages(section, _glossary()) combined = messages[0]["content"] + messages[1]["content"] assert "dysfluencies" in combined assert "punctuation" in combined assert "paraphrase" in combined assert '"id": 1' in messages[1]["content"] def test_process_transcript_result_uses_injected_fake_client_and_applies_sequential_module_updates(tmp_path): transcript = parse_source_transcript_json( """ [ {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."} ] """ ) config = AuditaConfig.from_sources( env={}, overrides=None, ) config = AuditaConfig( api_key=config.api_key, model=config.model, base_url=config.base_url, max_retries=config.max_retries, max_section_tokens=config.max_section_tokens, glossary_confidence_threshold=config.glossary_confidence_threshold, homophones_confidence_threshold=config.homophones_confidence_threshold, normalize_max_segment_gap=config.normalize_max_segment_gap, normalize_ellipsis_gap=config.normalize_ellipsis_gap, normalize_max_segment_duration=config.normalize_max_segment_duration, normalize_max_segment_tokens=config.normalize_max_segment_tokens, work_dir=tmp_path / "work", work_dir_retention="always", ) client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95, } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.97, "reason": "Likely spoken-form correction in context.", } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.99, "reason": "Does not reverse the segment meaning.", } ] }, { "corrections": [ { "id": 1, "original_text": "dam", "corrected_text": "damn", "confidence": 0.92, } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.98, "reason": "Likely spoken-form correction in context.", } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.99, "reason": "Does not reverse the segment meaning.", } ] }, {"corrections": []}, {"corrections": []}, ] ) result = process_transcript_result(transcript, _glossary(), config, llm_client=client) assert result.transcript[0].text == "There were Jesters at the damn." assert [call["stage_name"] for call in client.calls] == [ "glossary_1:proposal", "glossary_1:spoken_form_plausibility_review", "glossary_1:meaning_reversal_review", "homophones:proposal", "homophones:spoken_form_plausibility_review", "homophones:meaning_reversal_review", "glossary_2:proposal", "spoken_word:proposal", ] assert "There were Jesters at the dam." in client.calls[3]["messages"][1]["content"] def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_validators(tmp_path): transcript = parse_source_transcript_json( """ [ {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} ] """ ) base_config = AuditaConfig.from_sources(env={}) config = AuditaConfig( api_key=base_config.api_key, model=base_config.model, base_url=base_config.base_url, max_retries=base_config.max_retries, max_section_tokens=base_config.max_section_tokens, glossary_confidence_threshold=0.96, homophones_confidence_threshold=base_config.homophones_confidence_threshold, normalize_max_segment_gap=base_config.normalize_max_segment_gap, normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, normalize_max_segment_duration=base_config.normalize_max_segment_duration, normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, work_dir=tmp_path / "work", work_dir_retention="always", ) client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.95, } ] }, {"corrections": []}, {"corrections": []}, {"corrections": []}, ] ) result = process_transcript_result(transcript, _glossary(), config, llm_client=client) assert result.transcript[0].text == "There were gestures at the temple." assert [call["stage_name"] for call in client.calls] == [ "glossary_1:proposal", "homophones:proposal", "glossary_2:proposal", "spoken_word:proposal", ] assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold" assert result.report.skipped_corrections[0].source == "validator:proposal_confidence_guard" assert result.report.modules[0].validators[0].rejected_count == 1 assert result.report.modules[0].validators[1].candidate_count == 0 def test_process_transcript_result_runs_spoken_word_module_with_full_validator_chain(tmp_path): transcript = parse_source_transcript_json( """ [ {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."} ] """ ) base_config = AuditaConfig.from_sources(env={}) config = AuditaConfig( api_key=base_config.api_key, model=base_config.model, base_url=base_config.base_url, max_retries=base_config.max_retries, max_section_tokens=base_config.max_section_tokens, glossary_confidence_threshold=base_config.glossary_confidence_threshold, homophones_confidence_threshold=base_config.homophones_confidence_threshold, spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold, normalize_max_segment_gap=base_config.normalize_max_segment_gap, normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, normalize_max_segment_duration=base_config.normalize_max_segment_duration, normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, work_dir=tmp_path / "work", work_dir_retention="always", ) client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "I, uh, I think", "corrected_text": "I think", "confidence": 0.95, } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.97, "reason": "Reasonable dysfluency cleanup that preserves meaning.", } ] }, { "validations": [ { "correction_index": 0, "approved": True, "confidence": 0.99, "reason": "Does not reverse the segment meaning.", } ] }, ] ) result = process_transcript_result( transcript, _glossary(), config, module_keys=["spoken_word"], llm_client=client, ) assert result.transcript[0].text == "I think we should go." assert [call["stage_name"] for call in client.calls] == [ "spoken_word:proposal", "spoken_word:spoken_word_review", "spoken_word:meaning_reversal_review", ] assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [ "proposal_confidence_guard", "protected_glossary_guard", "spoken_word_review", "meaning_reversal_review", ] def test_process_transcript_result_rejects_spoken_word_below_threshold_before_llm_validators(tmp_path): transcript = parse_source_transcript_json( """ [ {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."} ] """ ) base_config = AuditaConfig.from_sources(env={}) config = AuditaConfig( api_key=base_config.api_key, model=base_config.model, base_url=base_config.base_url, max_retries=base_config.max_retries, max_section_tokens=base_config.max_section_tokens, glossary_confidence_threshold=base_config.glossary_confidence_threshold, homophones_confidence_threshold=base_config.homophones_confidence_threshold, spoken_word_confidence_threshold=0.96, normalize_max_segment_gap=base_config.normalize_max_segment_gap, normalize_ellipsis_gap=base_config.normalize_ellipsis_gap, normalize_max_segment_duration=base_config.normalize_max_segment_duration, normalize_max_segment_tokens=base_config.normalize_max_segment_tokens, work_dir=tmp_path / "work", work_dir_retention="always", ) client = FakeStructuredLLMClient( [ { "corrections": [ { "id": 1, "original_text": "I, uh, I think", "corrected_text": "I think", "confidence": 0.95, } ] } ] ) result = process_transcript_result( transcript, _glossary(), config, module_keys=["spoken_word"], llm_client=client, ) assert result.transcript[0].text == "I, uh, I think we should go." assert [call["stage_name"] for call in client.calls] == ["spoken_word:proposal"] assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold" assert result.report.modules[0].validators[1].candidate_count == 0