eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Implemented the spoken_word LLM review module

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-25 08:12:36 -05:00
parent 92c8c371a6
commit 52d29f7228
13 changed files with 484 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

207
tests/test_module_proposals.py
View File

@@ -7,7 +7,8 @@ from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_jso
from audita.framework.models import ModuleContext, ModuleRunSpec
from audita.modules.glossary import GlossaryModule
from audita.modules.homophones import HomophonesModule
from audita.modules.prompts import build_homophones_proposal_messages
from audita.modules.prompts import build_homophones_proposal_messages, build_spoken_word_proposal_messages
from audita.modules.spoken_word import SpokenWordModule
from audita.pipeline import process_transcript_result
@@ -110,6 +111,70 @@ def test_homophones_prompt_is_explicitly_scoped_to_spoken_form_corrections():
assert '"id": 1' in messages[1]["content"]
def test_spoken_word_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."}
]
"""
)
section = chunk_transcript(transcript, max_section_tokens=1000)[0]
module = SpokenWordModule()
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "I, uh, I think",
"corrected_text": "I think",
"confidence": 0.95,
}
]
}
]
)
context = ModuleContext(
run_spec=ModuleRunSpec(instance_name="spoken_word", module_key="spoken_word", module=module),
glossary=_glossary(),
config=AuditaConfig.from_sources(env={}),
run_dir=tmp_path,
llm_client=client,
)
proposals = list(module.propose(section, context))
assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [
(1, "I, uh, I think", "I think", 0.95)
]
assert (tmp_path / "prompt-0000.json").exists()
assert (tmp_path / "corrections-0000.json").exists()
prompt_text = client.calls[0]["messages"][1]["content"]
assert "spoken-word cleanup" in prompt_text
assert "exact text span" in prompt_text
assert "uh" in prompt_text
def test_spoken_word_prompt_is_explicitly_scoped_to_dysfluency_cleanup():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Well ... I think we should go."}
]
"""
)
section = chunk_transcript(transcript, max_section_tokens=1000)[0]
messages = build_spoken_word_proposal_messages(section, _glossary())
combined = messages[0]["content"] + messages[1]["content"]
assert "dysfluencies" in combined
assert "punctuation" in combined
assert "paraphrase" in combined
assert '"id": 1' in messages[1]["content"]
def test_process_transcript_result_uses_injected_fake_client_and_applies_sequential_module_updates(tmp_path):
transcript = parse_source_transcript_json(
"""
@@ -200,6 +265,7 @@ def test_process_transcript_result_uses_injected_fake_client_and_applies_sequent
]
},
{"corrections": []},
{"corrections": []},
]
)
@@ -214,6 +280,7 @@ def test_process_transcript_result_uses_injected_fake_client_and_applies_sequent
"homophones:spoken_form_plausibility_review",
"homophones:meaning_reversal_review",
"glossary_2:proposal",
"spoken_word:proposal",
]
assert "There were Jesters at the dam." in client.calls[3]["messages"][1]["content"]
@@ -256,6 +323,7 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_
},
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
@@ -266,8 +334,145 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_
"glossary_1:proposal",
"homophones:proposal",
"glossary_2:proposal",
"spoken_word:proposal",
]
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
assert result.report.skipped_corrections[0].source == "validator:proposal_confidence_guard"
assert result.report.modules[0].validators[0].rejected_count == 1
assert result.report.modules[0].validators[1].candidate_count == 0
def test_process_transcript_result_runs_spoken_word_module_with_full_validator_chain(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."}
]
"""
)
base_config = AuditaConfig.from_sources(env={})
config = AuditaConfig(
api_key=base_config.api_key,
model=base_config.model,
base_url=base_config.base_url,
max_retries=base_config.max_retries,
max_section_tokens=base_config.max_section_tokens,
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold,
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
normalize_max_segment_tokens=base_config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="always",
)
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "I, uh, I think",
"corrected_text": "I think",
"confidence": 0.95,
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.97,
"reason": "Reasonable dysfluency cleanup that preserves meaning.",
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.99,
"reason": "Does not reverse the segment meaning.",
}
]
},
]
)
result = process_transcript_result(
transcript,
_glossary(),
config,
module_keys=["spoken_word"],
llm_client=client,
)
assert result.transcript[0].text == "I think we should go."
assert [call["stage_name"] for call in client.calls] == [
"spoken_word:proposal",
"spoken_word:spoken_word_review",
"spoken_word:meaning_reversal_review",
]
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"spoken_word_review",
"meaning_reversal_review",
]
def test_process_transcript_result_rejects_spoken_word_below_threshold_before_llm_validators(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."}
]
"""
)
base_config = AuditaConfig.from_sources(env={})
config = AuditaConfig(
api_key=base_config.api_key,
model=base_config.model,
base_url=base_config.base_url,
max_retries=base_config.max_retries,
max_section_tokens=base_config.max_section_tokens,
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
spoken_word_confidence_threshold=0.96,
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
normalize_max_segment_tokens=base_config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="always",
)
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "I, uh, I think",
"corrected_text": "I think",
"confidence": 0.95,
}
]
}
]
)
result = process_transcript_result(
transcript,
_glossary(),
config,
module_keys=["spoken_word"],
llm_client=client,
)
assert result.transcript[0].text == "I, uh, I think we should go."
assert [call["stage_name"] for call in client.calls] == ["spoken_word:proposal"]
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
assert result.report.modules[0].validators[1].candidate_count == 0