479 lines
17 KiB
Python
479 lines
17 KiB
Python
from pathlib import Path
|
|
|
|
from audita.core.chunking import chunk_transcript
|
|
from audita.core.config import AuditaConfig
|
|
from audita.core.errors import AuditaLLMError
|
|
from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json, parse_transcript_json
|
|
from audita.framework.models import ModuleContext, ModuleRunSpec
|
|
from audita.modules.glossary import GlossaryModule
|
|
from audita.modules.homophones import HomophonesModule
|
|
from audita.modules.prompts import build_homophones_proposal_messages, build_spoken_word_proposal_messages
|
|
from audita.modules.spoken_word import SpokenWordModule
|
|
from audita.pipeline import process_transcript_result
|
|
|
|
|
|
class FakeStructuredLLMClient:
|
|
def __init__(self, responses):
|
|
self._responses = list(responses)
|
|
self.calls = []
|
|
|
|
def run_structured(self, *, stage_name, messages, response_model, config):
|
|
self.calls.append(
|
|
{
|
|
"stage_name": stage_name,
|
|
"messages": list(messages),
|
|
"response_model": response_model,
|
|
}
|
|
)
|
|
if not self._responses:
|
|
raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.")
|
|
return response_model.model_validate(self._responses.pop(0))
|
|
|
|
|
|
def _glossary():
|
|
return parse_glossary_yaml(
|
|
"""
|
|
glossary:
|
|
- name: "Jesters"
|
|
aliases:
|
|
- "Jester"
|
|
category: faction
|
|
summary: "A faction."
|
|
- name: "Hrank"
|
|
category: pc
|
|
summary: "A player character."
|
|
"""
|
|
)
|
|
|
|
|
|
def test_glossary_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path):
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
|
|
]
|
|
"""
|
|
)
|
|
section = chunk_transcript(transcript, max_section_tokens=1000)[0]
|
|
module = GlossaryModule()
|
|
client = FakeStructuredLLMClient(
|
|
[
|
|
{
|
|
"corrections": [
|
|
{
|
|
"id": 1,
|
|
"original_text": "gestures",
|
|
"corrected_text": "Jesters",
|
|
"confidence": 0.95,
|
|
}
|
|
]
|
|
}
|
|
]
|
|
)
|
|
context = ModuleContext(
|
|
run_spec=ModuleRunSpec(instance_name="glossary_primary", module_key="glossary", module=module),
|
|
glossary=_glossary(),
|
|
config=AuditaConfig.from_sources(env={}),
|
|
run_dir=tmp_path,
|
|
llm_client=client,
|
|
)
|
|
|
|
proposals = list(module.propose(section, context))
|
|
|
|
assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [
|
|
(1, "gestures", "Jesters", 0.95)
|
|
]
|
|
assert (tmp_path / "prompt-0000.json").exists()
|
|
assert (tmp_path / "corrections-0000.json").exists()
|
|
prompt_text = client.calls[0]["messages"][1]["content"]
|
|
assert "Glossary:" in prompt_text
|
|
assert "exact text span" in prompt_text
|
|
assert "gestures" in prompt_text
|
|
|
|
|
|
def test_homophones_prompt_is_explicitly_scoped_to_spoken_form_corrections():
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't do that with a dam."}
|
|
]
|
|
"""
|
|
)
|
|
section = chunk_transcript(transcript, max_section_tokens=1000)[0]
|
|
|
|
messages = build_homophones_proposal_messages(section, _glossary())
|
|
combined = messages[0]["content"] + messages[1]["content"]
|
|
|
|
assert "homophone" in combined
|
|
assert "mistranscription" in combined
|
|
assert "Do not add or remove punctuation" in combined
|
|
assert "visible" in combined
|
|
assert '"id": 1' in messages[1]["content"]
|
|
|
|
|
|
def test_spoken_word_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path):
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."}
|
|
]
|
|
"""
|
|
)
|
|
section = chunk_transcript(transcript, max_section_tokens=1000)[0]
|
|
module = SpokenWordModule()
|
|
client = FakeStructuredLLMClient(
|
|
[
|
|
{
|
|
"corrections": [
|
|
{
|
|
"id": 1,
|
|
"original_text": "I, uh, I think",
|
|
"corrected_text": "I think",
|
|
"confidence": 0.95,
|
|
}
|
|
]
|
|
}
|
|
]
|
|
)
|
|
context = ModuleContext(
|
|
run_spec=ModuleRunSpec(instance_name="spoken_word", module_key="spoken_word", module=module),
|
|
glossary=_glossary(),
|
|
config=AuditaConfig.from_sources(env={}),
|
|
run_dir=tmp_path,
|
|
llm_client=client,
|
|
)
|
|
|
|
proposals = list(module.propose(section, context))
|
|
|
|
assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [
|
|
(1, "I, uh, I think", "I think", 0.95)
|
|
]
|
|
assert (tmp_path / "prompt-0000.json").exists()
|
|
assert (tmp_path / "corrections-0000.json").exists()
|
|
prompt_text = client.calls[0]["messages"][1]["content"]
|
|
assert "spoken-word cleanup" in prompt_text
|
|
assert "exact text span" in prompt_text
|
|
assert "uh" in prompt_text
|
|
|
|
|
|
def test_spoken_word_prompt_is_explicitly_scoped_to_dysfluency_cleanup():
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Well ... I think we should go."}
|
|
]
|
|
"""
|
|
)
|
|
section = chunk_transcript(transcript, max_section_tokens=1000)[0]
|
|
|
|
messages = build_spoken_word_proposal_messages(section, _glossary())
|
|
combined = messages[0]["content"] + messages[1]["content"]
|
|
|
|
assert "dysfluencies" in combined
|
|
assert "punctuation" in combined
|
|
assert "paraphrase" in combined
|
|
assert '"id": 1' in messages[1]["content"]
|
|
|
|
|
|
def test_process_transcript_result_uses_injected_fake_client_and_applies_sequential_module_updates(tmp_path):
|
|
transcript = parse_source_transcript_json(
|
|
"""
|
|
[
|
|
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."}
|
|
]
|
|
"""
|
|
)
|
|
config = AuditaConfig.from_sources(
|
|
env={},
|
|
overrides=None,
|
|
)
|
|
config = AuditaConfig(
|
|
api_key=config.api_key,
|
|
model=config.model,
|
|
base_url=config.base_url,
|
|
max_retries=config.max_retries,
|
|
max_section_tokens=config.max_section_tokens,
|
|
glossary_confidence_threshold=config.glossary_confidence_threshold,
|
|
homophones_confidence_threshold=config.homophones_confidence_threshold,
|
|
normalize_max_segment_gap=config.normalize_max_segment_gap,
|
|
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
|
|
normalize_max_segment_duration=config.normalize_max_segment_duration,
|
|
normalize_max_segment_tokens=config.normalize_max_segment_tokens,
|
|
work_dir=tmp_path / "work",
|
|
work_dir_retention="always",
|
|
)
|
|
client = FakeStructuredLLMClient(
|
|
[
|
|
{
|
|
"corrections": [
|
|
{
|
|
"id": 1,
|
|
"original_text": "gestures",
|
|
"corrected_text": "Jesters",
|
|
"confidence": 0.95,
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"validations": [
|
|
{
|
|
"correction_index": 0,
|
|
"approved": True,
|
|
"confidence": 0.97,
|
|
"reason": "Likely spoken-form correction in context.",
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"validations": [
|
|
{
|
|
"correction_index": 0,
|
|
"approved": True,
|
|
"confidence": 0.99,
|
|
"reason": "Does not reverse the segment meaning.",
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"corrections": [
|
|
{
|
|
"id": 1,
|
|
"original_text": "dam",
|
|
"corrected_text": "damn",
|
|
"confidence": 0.92,
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"validations": [
|
|
{
|
|
"correction_index": 0,
|
|
"approved": True,
|
|
"confidence": 0.98,
|
|
"reason": "Likely spoken-form correction in context.",
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"validations": [
|
|
{
|
|
"correction_index": 0,
|
|
"approved": True,
|
|
"confidence": 0.99,
|
|
"reason": "Does not reverse the segment meaning.",
|
|
}
|
|
]
|
|
},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
]
|
|
)
|
|
|
|
result = process_transcript_result(transcript, _glossary(), config, llm_client=client)
|
|
|
|
assert result.transcript[0].text == "There were Jesters at the damn."
|
|
assert [call["stage_name"] for call in client.calls] == [
|
|
"glossary_1:proposal",
|
|
"glossary_1:spoken_form_plausibility_review",
|
|
"glossary_1:meaning_reversal_review",
|
|
"homophones:proposal",
|
|
"homophones:spoken_form_plausibility_review",
|
|
"homophones:meaning_reversal_review",
|
|
"glossary_2:proposal",
|
|
"spoken_word:proposal",
|
|
]
|
|
assert "There were Jesters at the dam." in client.calls[3]["messages"][1]["content"]
|
|
|
|
|
|
def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_validators(tmp_path):
|
|
transcript = parse_source_transcript_json(
|
|
"""
|
|
[
|
|
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
|
|
]
|
|
"""
|
|
)
|
|
base_config = AuditaConfig.from_sources(env={})
|
|
config = AuditaConfig(
|
|
api_key=base_config.api_key,
|
|
model=base_config.model,
|
|
base_url=base_config.base_url,
|
|
max_retries=base_config.max_retries,
|
|
max_section_tokens=base_config.max_section_tokens,
|
|
glossary_confidence_threshold=0.96,
|
|
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
|
|
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
|
|
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
|
|
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
|
|
normalize_max_segment_tokens=base_config.normalize_max_segment_tokens,
|
|
work_dir=tmp_path / "work",
|
|
work_dir_retention="always",
|
|
)
|
|
client = FakeStructuredLLMClient(
|
|
[
|
|
{
|
|
"corrections": [
|
|
{
|
|
"id": 1,
|
|
"original_text": "gestures",
|
|
"corrected_text": "Jesters",
|
|
"confidence": 0.95,
|
|
}
|
|
]
|
|
},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
]
|
|
)
|
|
|
|
result = process_transcript_result(transcript, _glossary(), config, llm_client=client)
|
|
|
|
assert result.transcript[0].text == "There were gestures at the temple."
|
|
assert [call["stage_name"] for call in client.calls] == [
|
|
"glossary_1:proposal",
|
|
"homophones:proposal",
|
|
"glossary_2:proposal",
|
|
"spoken_word:proposal",
|
|
]
|
|
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
|
|
assert result.report.skipped_corrections[0].source == "validator:proposal_confidence_guard"
|
|
assert result.report.modules[0].validators[0].rejected_count == 1
|
|
assert result.report.modules[0].validators[1].candidate_count == 0
|
|
|
|
|
|
def test_process_transcript_result_runs_spoken_word_module_with_full_validator_chain(tmp_path):
|
|
transcript = parse_source_transcript_json(
|
|
"""
|
|
[
|
|
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."}
|
|
]
|
|
"""
|
|
)
|
|
base_config = AuditaConfig.from_sources(env={})
|
|
config = AuditaConfig(
|
|
api_key=base_config.api_key,
|
|
model=base_config.model,
|
|
base_url=base_config.base_url,
|
|
max_retries=base_config.max_retries,
|
|
max_section_tokens=base_config.max_section_tokens,
|
|
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
|
|
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
|
|
spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold,
|
|
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
|
|
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
|
|
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
|
|
normalize_max_segment_tokens=base_config.normalize_max_segment_tokens,
|
|
work_dir=tmp_path / "work",
|
|
work_dir_retention="always",
|
|
)
|
|
client = FakeStructuredLLMClient(
|
|
[
|
|
{
|
|
"corrections": [
|
|
{
|
|
"id": 1,
|
|
"original_text": "I, uh, I think",
|
|
"corrected_text": "I think",
|
|
"confidence": 0.95,
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"validations": [
|
|
{
|
|
"correction_index": 0,
|
|
"approved": True,
|
|
"confidence": 0.97,
|
|
"reason": "Reasonable dysfluency cleanup that preserves meaning.",
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"validations": [
|
|
{
|
|
"correction_index": 0,
|
|
"approved": True,
|
|
"confidence": 0.99,
|
|
"reason": "Does not reverse the segment meaning.",
|
|
}
|
|
]
|
|
},
|
|
]
|
|
)
|
|
|
|
result = process_transcript_result(
|
|
transcript,
|
|
_glossary(),
|
|
config,
|
|
module_keys=["spoken_word"],
|
|
llm_client=client,
|
|
)
|
|
|
|
assert result.transcript[0].text == "I think we should go."
|
|
assert [call["stage_name"] for call in client.calls] == [
|
|
"spoken_word:proposal",
|
|
"spoken_word:spoken_word_review",
|
|
"spoken_word:meaning_reversal_review",
|
|
]
|
|
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
|
|
"proposal_confidence_guard",
|
|
"protected_glossary_guard",
|
|
"spoken_word_review",
|
|
"meaning_reversal_review",
|
|
]
|
|
|
|
|
|
def test_process_transcript_result_rejects_spoken_word_below_threshold_before_llm_validators(tmp_path):
|
|
transcript = parse_source_transcript_json(
|
|
"""
|
|
[
|
|
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."}
|
|
]
|
|
"""
|
|
)
|
|
base_config = AuditaConfig.from_sources(env={})
|
|
config = AuditaConfig(
|
|
api_key=base_config.api_key,
|
|
model=base_config.model,
|
|
base_url=base_config.base_url,
|
|
max_retries=base_config.max_retries,
|
|
max_section_tokens=base_config.max_section_tokens,
|
|
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
|
|
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
|
|
spoken_word_confidence_threshold=0.96,
|
|
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
|
|
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
|
|
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
|
|
normalize_max_segment_tokens=base_config.normalize_max_segment_tokens,
|
|
work_dir=tmp_path / "work",
|
|
work_dir_retention="always",
|
|
)
|
|
client = FakeStructuredLLMClient(
|
|
[
|
|
{
|
|
"corrections": [
|
|
{
|
|
"id": 1,
|
|
"original_text": "I, uh, I think",
|
|
"corrected_text": "I think",
|
|
"confidence": 0.95,
|
|
}
|
|
]
|
|
}
|
|
]
|
|
)
|
|
|
|
result = process_transcript_result(
|
|
transcript,
|
|
_glossary(),
|
|
config,
|
|
module_keys=["spoken_word"],
|
|
llm_client=client,
|
|
)
|
|
|
|
assert result.transcript[0].text == "I, uh, I think we should go."
|
|
assert [call["stage_name"] for call in client.calls] == ["spoken_word:proposal"]
|
|
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
|
|
assert result.report.modules[0].validators[1].candidate_count == 0
|