eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Implemented the spoken_word LLM review module

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-25 08:12:36 -05:00
parent 92c8c371a6
commit 52d29f7228
13 changed files with 484 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@@ -4,7 +4,7 @@ Audita is a framework-first transcript correction application. The public `audit
- deterministic transcript normalization
- token-batched module orchestration
- concrete `glossary` and `homophones` modules built on reusable proposal / validator contracts
- concrete `glossary`, `homophones`, and `spoken_word` modules built on reusable proposal / validator contracts
- structured run reporting and work-dir diagnostics
The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`.
@@ -44,8 +44,8 @@ Resolved run instance names are auto-numbered for repeats, so the default report
The default module sequence is partially implemented today:
- `glossary`, `homophones`, and the second `glossary` pass run real LLM-backed proposal and validation stages
- `spoken_word` and `grammar` remain stubs and currently propose no corrections
- `glossary`, `homophones`, the second `glossary` pass, and `spoken_word` run real LLM-backed proposal and validation stages
- `grammar` remains a stub and currently proposes no corrections
To run a custom module sequence, pass `--modules`:
@@ -77,7 +77,7 @@ audita process transcript.json --glossary glossary.yaml --output corrected.json
Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr.
`--report-json` writes a separate machine-readable run report and never mixes report data into stdout.
Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Normal runs now require `OPENROUTER_API_KEY`, because the `glossary` and `homophones` modules make real LLM calls.
Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Normal runs now require `OPENROUTER_API_KEY`, because the `glossary`, `homophones`, and `spoken_word` modules make real LLM calls.
| Environment variable | CLI flag | Default | Purpose |
| --- | --- | --- | --- |
@@ -88,6 +88,7 @@ Useful configuration can be supplied by CLI flag or environment variable. CLI fl
| `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `6144` | Maximum estimated tokens per transcript batch |
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation |
| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation |
| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | `--spoken-word-confidence-threshold` | `0.8` | Minimum confidence required for spoken-word proposals to survive validation |
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging |
| `AUDITA_NORMALIZE_ELLIPSIS_GAP` | `--normalize-ellipsis-gap` | `3.5` | Same-speaker gaps above this value are joined with ` ... ` |
| `AUDITA_NORMALIZE_MAX_SEGMENT_DURATION` | `--normalize-max-segment-duration` | `60.0` | Maximum merged segment duration |

6
src/audita/cli.py
View File

@@ -45,6 +45,11 @@ def _build_parser() -> argparse.ArgumentParser:
type=float,
help="minimum confidence required for homophone proposals to survive validation",
)
process.add_argument(
"--spoken-word-confidence-threshold",
type=float,
help="minimum confidence required for spoken-word proposals to survive validation",
)
process.add_argument(
"--normalize-max-segment-gap",
type=float,
@@ -85,6 +90,7 @@ def _process(args: argparse.Namespace) -> int:
max_section_tokens=args.max_section_tokens,
glossary_confidence_threshold=args.glossary_confidence_threshold,
homophones_confidence_threshold=args.homophones_confidence_threshold,
spoken_word_confidence_threshold=args.spoken_word_confidence_threshold,
normalize_max_segment_gap=args.normalize_max_segment_gap,
normalize_ellipsis_gap=args.normalize_ellipsis_gap,
normalize_max_segment_duration=args.normalize_max_segment_duration,

12
src/audita/core/config.py
View File

@@ -15,6 +15,7 @@ DEFAULT_MAX_RETRIES = 3
DEFAULT_MAX_SECTION_TOKENS = 6144
DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD = 0.80
DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD = 0.80
DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD = 0.80
DEFAULT_WORK_DIR = "/tmp/audita"
DEFAULT_WORK_DIR_RETENTION = "auto"
DEFAULT_NORMALIZE_MAX_SEGMENT_GAP = 4.0
@@ -32,6 +33,7 @@ class ConfigOverrides:
max_section_tokens: Optional[int] = None
glossary_confidence_threshold: Optional[float] = None
homophones_confidence_threshold: Optional[float] = None
spoken_word_confidence_threshold: Optional[float] = None
normalize_max_segment_gap: Optional[float] = None
normalize_ellipsis_gap: Optional[float] = None
normalize_max_segment_duration: Optional[float] = None
@@ -50,6 +52,7 @@ class AuditaConfig:
max_section_tokens: int = DEFAULT_MAX_SECTION_TOKENS
glossary_confidence_threshold: float = DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
homophones_confidence_threshold: float = DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD
spoken_word_confidence_threshold: float = DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD
normalize_max_segment_gap: float = DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
normalize_ellipsis_gap: float = DEFAULT_NORMALIZE_ELLIPSIS_GAP
normalize_max_segment_duration: float = DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION
@@ -99,6 +102,12 @@ class AuditaConfig:
DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD,
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD",
),
spoken_word_confidence_threshold=_select_float(
selected.spoken_word_confidence_threshold,
source.get("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD"),
DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD,
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD",
),
normalize_max_segment_gap=_select_float(
selected.normalize_max_segment_gap,
source.get("AUDITA_NORMALIZE_MAX_SEGMENT_GAP"),
@@ -149,6 +158,8 @@ class AuditaConfig:
raise AuditaConfigError("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.")
if not 0.0 <= self.homophones_confidence_threshold <= 1.0:
raise AuditaConfigError("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.")
if not 0.0 <= self.spoken_word_confidence_threshold <= 1.0:
raise AuditaConfigError("AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.")
if not math.isfinite(self.normalize_max_segment_gap):
raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_GAP must be finite.")
if not math.isfinite(self.normalize_ellipsis_gap):
@@ -180,6 +191,7 @@ class AuditaConfig:
"max_section_tokens": self.max_section_tokens,
"glossary_confidence_threshold": self.glossary_confidence_threshold,
"homophones_confidence_threshold": self.homophones_confidence_threshold,
"spoken_word_confidence_threshold": self.spoken_word_confidence_threshold,
"normalize_max_segment_gap": self.normalize_max_segment_gap,
"normalize_ellipsis_gap": self.normalize_ellipsis_gap,
"normalize_max_segment_duration": self.normalize_max_segment_duration,

37
src/audita/modules/prompts.py
View File

@@ -85,3 +85,40 @@ def build_homophones_proposal_messages(section: TranscriptSection, glossary: Glo
f"Transcript section:\n{section_json}"
)
return [{"role": "system", "content": system}, {"role": "user", "content": user}]
def build_spoken_word_proposal_messages(section: TranscriptSection, glossary: Glossary) -> List[Message]:
glossary_json = json.dumps(glossary.model_dump(mode="json", exclude_none=True), ensure_ascii=False, indent=2)
section_json = json.dumps(section.prompt_payload(), ensure_ascii=False, indent=2)
system = (
"You are Audita, a conservative spoken-word cleanup assistant. "
"Identify only low-risk cleanup of repeated words or short phrases, filler words, hesitation artifacts, "
"and similar dysfluencies that commonly appear in spoken English transcripts. "
"Preserve substantive meaning, named entities, and transcript content."
)
user = (
"Review this transcript section and return only spoken-word cleanup corrections that should be applied.\n\n"
"Rules:\n"
"- Approve only conservative cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar spoken dysfluencies.\n"
"- You may collapse adjacent repetition such as \"I I think\" to \"I think\" or remove filler spans such as \"you know\" or \"uh\" when local context supports that cleanup.\n"
"- You may include low-risk punctuation, spacing, or capitalization cleanup when it is part of removing a dysfluency, such as removing ellipses or hesitation punctuation that no longer belongs after the cleanup.\n"
"- Do not paraphrase, summarize, reorder ideas, replace content with different wording, or make substantive semantic edits.\n"
"- Do not change clear content words just because a different phrasing reads better.\n"
"- Treat glossary names and aliases as protected spellings and context.\n"
"- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.\n"
"- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n"
"- Use the exact id from the input segment.\n"
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n"
"- Choose an original_text span that appears exactly once in the current segment text.\n"
"- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n"
"- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n"
"- Do not return corrections where original_text and corrected_text are identical.\n"
"- Do not return speaker, start, or end fields.\n"
"- Return only changed segments; do not return entries for unchanged segments.\n"
"- confidence must be between 0.0 and 1.0.\n"
"- If no corrections are needed, return an empty corrections list.\n\n"
f"Protected glossary/context:\n{glossary_json}\n\n"
f"Transcript section:\n{section_json}"
)
return [{"role": "system", "content": system}, {"role": "user", "content": user}]

25
src/audita/modules/spoken_word.py
View File

@@ -2,19 +2,36 @@ from typing import Sequence
from audita.core.chunking import TranscriptSection
from audita.framework.models import CorrectionProposal, ModuleContext
from audita.validators import ProtectedGlossaryTermsValidator, Validator
from audita.framework.proposal_generation import generate_llm_correction_proposals
from audita.modules.prompts import build_spoken_word_proposal_messages
from audita.validators import (
MeaningReversalValidator,
ProposalConfidenceValidator,
ProtectedGlossaryTermsValidator,
SpokenWordValidator,
Validator,
)
class SpokenWordModule:
module_key = "spoken_word"
replacement_policy = "replace_all"
replacement_policy = "require_unique"
def validators(self) -> Sequence[Validator]:
return [ProtectedGlossaryTermsValidator("protected_glossary_guard")]
return [
ProposalConfidenceValidator("proposal_confidence_guard", "spoken_word_confidence_threshold"),
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
SpokenWordValidator("spoken_word_review"),
MeaningReversalValidator("meaning_reversal_review"),
]
def propose(
self,
transcript_section: TranscriptSection,
context: ModuleContext,
) -> Sequence[CorrectionProposal]:
return []
return generate_llm_correction_proposals(
section=transcript_section,
context=context,
prompt_builder=build_spoken_word_proposal_messages,
)

3
src/audita/validators/__init__.py
View File

@@ -1,6 +1,6 @@
from .base import ValidationContext, ValidationDecision, ValidationResult, Validator
from .deterministic import ProposalConfidenceValidator, ProtectedGlossaryTermsValidator
from .llm import MeaningReversalValidator, SpokenFormPlausibilityValidator
from .llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
from .protection import ProtectedVocabulary
__all__ = [
@@ -12,5 +12,6 @@ __all__ = [
"ProtectedGlossaryTermsValidator",
"ProtectedVocabulary",
"SpokenFormPlausibilityValidator",
"SpokenWordValidator",
"MeaningReversalValidator",
]

8
src/audita/validators/llm.py
View File

@@ -12,7 +12,7 @@ from audita.core.errors import AuditaLLMError
from audita.framework.proposals import ProposalPreview, ProposalPreviewError, preview_proposal
from .base import ValidationContext, ValidationDecision, ValidationResult
from .prompts import build_meaning_reversal_messages, build_spoken_form_plausibility_messages
from .prompts import build_meaning_reversal_messages, build_spoken_form_plausibility_messages, build_spoken_word_messages
class _LLMValidationDecisionModel(BaseModel):
@@ -141,5 +141,11 @@ class MeaningReversalValidator(_BaseLLMValidator):
prompt_builder: PromptBuilder = build_meaning_reversal_messages
@dataclass(frozen=True)
class SpokenWordValidator(_BaseLLMValidator):
name: str = "spoken_word_review"
prompt_builder: PromptBuilder = build_spoken_word_messages
def _write_json(path: Path, payload: dict) -> None:
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")

24
src/audita/validators/prompts.py
View File

@@ -54,3 +54,27 @@ def build_meaning_reversal_messages(validation_payload: List[dict]) -> List[Mess
f"Corrections to validate:\n{payload_json}"
)
return [{"role": "system", "content": system}, {"role": "user", "content": user}]
def build_spoken_word_messages(validation_payload: List[dict]) -> List[Message]:
payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2)
system = (
"You are Audita, a conservative spoken-word cleanup validation assistant. "
"Evaluate whether each proposed correction is a reasonable cleanup of repeated words, repeated short phrases, "
"filler words, hesitation artifacts, or similar spoken dysfluencies. "
"Approve only low-risk cleanup that preserves the segment's substantive meaning."
)
user = (
"Review these proposed transcript corrections and decide whether each one is a valid spoken-word cleanup.\n\n"
"Rules:\n"
"- Return one validation decision for every correction_index in the input.\n"
"- Approve cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar common spoken dysfluencies.\n"
"- Approve minor punctuation, spacing, or capitalization cleanup only when it is plausibly part of removing a dysfluency.\n"
"- Reject free-standing stylistic polishing, readability edits, paraphrases, and general rewriting.\n"
"- Reject edits that materially change the segment's substantive meaning, even if they are not literal antonyms.\n"
"- Evaluate the full original_segment_text and corrected_segment_text, not only the replacement span.\n"
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n"
"- confidence must be between 0.0 and 1.0.\n\n"
f"Corrections to validate:\n{payload_json}"
)
return [{"role": "system", "content": system}, {"role": "user", "content": user}]

142
tests/test_llm_validators.py
View File

@@ -6,10 +6,11 @@ from audita.core.errors import AuditaLLMError
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
from audita.framework.models import CorrectionProposal, ModuleRunSpec
from audita.validators.base import ValidationContext
from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator
from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
from audita.validators.prompts import (
build_meaning_reversal_messages,
build_spoken_form_plausibility_messages,
build_spoken_word_messages,
)
import audita.validators.llm as llm_module
@@ -204,6 +205,110 @@ def test_meaning_reversal_validator_rejects_reversal_and_approves_nonreversal(tm
assert "The figure became invisible in the doorway." in prompt_text
def test_spoken_word_validator_approves_cleanup_and_rejects_rewrite(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "We should maybe proceed carefully."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="spoken_word",
module_key="spoken_word",
id=1,
original_text="I, uh, I think",
corrected_text="I think",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="spoken_word",
module_key="spoken_word",
id=2,
original_text="maybe proceed carefully",
corrected_text="go now",
confidence=0.95,
),
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.97,
"reason": "Reasonable dysfluency cleanup that preserves meaning.",
},
{
"correction_index": 1,
"approved": False,
"confidence": 0.99,
"reason": "This changes the substance of the segment rather than cleaning a dysfluency.",
},
]
}
]
)
result = SpokenWordValidator("spoken_word_review").validate(
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
(0, True),
(1, False),
]
prompt_text = client.calls[0]["messages"][1]["content"]
assert "I think we should go." in prompt_text
assert "go now" in prompt_text
def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Well ... I think we should go."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="spoken_word",
module_key="spoken_word",
id=1,
original_text="Well ... ",
corrected_text="",
confidence=0.95,
)
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.95,
"reason": "Removes a hesitation artifact without changing substantive meaning.",
}
]
}
]
)
result = SpokenWordValidator("spoken_word_review").validate(
_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)]
@pytest.mark.parametrize(
("validator", "payload", "message_fragment"),
[
@@ -246,6 +351,20 @@ def test_meaning_reversal_validator_rejects_reversal_and_approves_nonreversal(tm
},
"unknown correction_index",
),
(
SpokenWordValidator("spoken_word_review"),
{
"validations": [
{
"correction_index": 99,
"approved": True,
"confidence": 0.9,
"reason": "unknown",
}
]
},
"unknown correction_index",
),
],
)
def test_llm_validators_reject_bad_correction_indexes(tmp_path, validator, payload, message_fragment):
@@ -356,6 +475,27 @@ def test_meaning_reversal_prompt_emphasizes_antonyms_and_segment_context():
assert "original_segment_text" in messages[1]["content"]
def test_spoken_word_prompt_emphasizes_dysfluency_cleanup():
messages = build_spoken_word_messages(
[
{
"correction_index": 0,
"id": 1,
"original_segment_text": "I, uh, I think we should go.",
"corrected_segment_text": "I think we should go.",
"original_text": "I, uh, I think",
"corrected_text": "I think",
}
]
)
combined = messages[0]["content"] + messages[1]["content"]
assert "dysfluencies" in combined
assert "punctuation" in combined
assert "substantive meaning" in combined
assert "original_segment_text" in messages[1]["content"]
def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path):
transcript = parse_transcript_json(
"""

207
tests/test_module_proposals.py
View File

@@ -7,7 +7,8 @@ from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_jso
from audita.framework.models import ModuleContext, ModuleRunSpec
from audita.modules.glossary import GlossaryModule
from audita.modules.homophones import HomophonesModule
from audita.modules.prompts import build_homophones_proposal_messages
from audita.modules.prompts import build_homophones_proposal_messages, build_spoken_word_proposal_messages
from audita.modules.spoken_word import SpokenWordModule
from audita.pipeline import process_transcript_result
@@ -110,6 +111,70 @@ def test_homophones_prompt_is_explicitly_scoped_to_spoken_form_corrections():
assert '"id": 1' in messages[1]["content"]
def test_spoken_word_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."}
]
"""
)
section = chunk_transcript(transcript, max_section_tokens=1000)[0]
module = SpokenWordModule()
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "I, uh, I think",
"corrected_text": "I think",
"confidence": 0.95,
}
]
}
]
)
context = ModuleContext(
run_spec=ModuleRunSpec(instance_name="spoken_word", module_key="spoken_word", module=module),
glossary=_glossary(),
config=AuditaConfig.from_sources(env={}),
run_dir=tmp_path,
llm_client=client,
)
proposals = list(module.propose(section, context))
assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [
(1, "I, uh, I think", "I think", 0.95)
]
assert (tmp_path / "prompt-0000.json").exists()
assert (tmp_path / "corrections-0000.json").exists()
prompt_text = client.calls[0]["messages"][1]["content"]
assert "spoken-word cleanup" in prompt_text
assert "exact text span" in prompt_text
assert "uh" in prompt_text
def test_spoken_word_prompt_is_explicitly_scoped_to_dysfluency_cleanup():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Well ... I think we should go."}
]
"""
)
section = chunk_transcript(transcript, max_section_tokens=1000)[0]
messages = build_spoken_word_proposal_messages(section, _glossary())
combined = messages[0]["content"] + messages[1]["content"]
assert "dysfluencies" in combined
assert "punctuation" in combined
assert "paraphrase" in combined
assert '"id": 1' in messages[1]["content"]
def test_process_transcript_result_uses_injected_fake_client_and_applies_sequential_module_updates(tmp_path):
transcript = parse_source_transcript_json(
"""
@@ -200,6 +265,7 @@ def test_process_transcript_result_uses_injected_fake_client_and_applies_sequent
]
},
{"corrections": []},
{"corrections": []},
]
)
@@ -214,6 +280,7 @@ def test_process_transcript_result_uses_injected_fake_client_and_applies_sequent
"homophones:spoken_form_plausibility_review",
"homophones:meaning_reversal_review",
"glossary_2:proposal",
"spoken_word:proposal",
]
assert "There were Jesters at the dam." in client.calls[3]["messages"][1]["content"]
@@ -256,6 +323,7 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_
},
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
@@ -266,8 +334,145 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_
"glossary_1:proposal",
"homophones:proposal",
"glossary_2:proposal",
"spoken_word:proposal",
]
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
assert result.report.skipped_corrections[0].source == "validator:proposal_confidence_guard"
assert result.report.modules[0].validators[0].rejected_count == 1
assert result.report.modules[0].validators[1].candidate_count == 0
def test_process_transcript_result_runs_spoken_word_module_with_full_validator_chain(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."}
]
"""
)
base_config = AuditaConfig.from_sources(env={})
config = AuditaConfig(
api_key=base_config.api_key,
model=base_config.model,
base_url=base_config.base_url,
max_retries=base_config.max_retries,
max_section_tokens=base_config.max_section_tokens,
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold,
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
normalize_max_segment_tokens=base_config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="always",
)
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "I, uh, I think",
"corrected_text": "I think",
"confidence": 0.95,
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.97,
"reason": "Reasonable dysfluency cleanup that preserves meaning.",
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.99,
"reason": "Does not reverse the segment meaning.",
}
]
},
]
)
result = process_transcript_result(
transcript,
_glossary(),
config,
module_keys=["spoken_word"],
llm_client=client,
)
assert result.transcript[0].text == "I think we should go."
assert [call["stage_name"] for call in client.calls] == [
"spoken_word:proposal",
"spoken_word:spoken_word_review",
"spoken_word:meaning_reversal_review",
]
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"spoken_word_review",
"meaning_reversal_review",
]
def test_process_transcript_result_rejects_spoken_word_below_threshold_before_llm_validators(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."}
]
"""
)
base_config = AuditaConfig.from_sources(env={})
config = AuditaConfig(
api_key=base_config.api_key,
model=base_config.model,
base_url=base_config.base_url,
max_retries=base_config.max_retries,
max_section_tokens=base_config.max_section_tokens,
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
spoken_word_confidence_threshold=0.96,
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
normalize_max_segment_tokens=base_config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="always",
)
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "I, uh, I think",
"corrected_text": "I think",
"confidence": 0.95,
}
]
}
]
)
result = process_transcript_result(
transcript,
_glossary(),
config,
module_keys=["spoken_word"],
llm_client=client,
)
assert result.transcript[0].text == "I, uh, I think we should go."
assert [call["stage_name"] for call in client.calls] == ["spoken_word:proposal"]
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
assert result.report.modules[0].validators[1].candidate_count == 0

1
tests/test_new_cli.py
View File

@@ -27,6 +27,7 @@ def test_process_help_exposes_framework_flags(capsys):
assert "--max-section-tokens" in output
assert "--glossary-confidence-threshold" in output
assert "--homophones-confidence-threshold" in output
assert "--spoken-word-confidence-threshold" in output
assert "--work-dir-retention" in output
assert "--normalize-max-segment-gap" in output
assert "--grammar-validation-enabled" not in output

6
tests/test_new_config.py
View File

@@ -6,6 +6,7 @@ from audita.core.config import (
DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD,
DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD,
DEFAULT_NORMALIZE_MAX_SEGMENT_GAP,
DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD,
DEFAULT_WORK_DIR_RETENTION,
)
from audita.core.errors import AuditaConfigError
@@ -19,6 +20,7 @@ def test_default_config_allows_missing_api_key():
assert config.module_keys == DEFAULT_MODULE_KEYS
assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
assert config.homophones_confidence_threshold == DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD
assert config.spoken_word_confidence_threshold == DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD
assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
assert config.work_dir_retention == DEFAULT_WORK_DIR_RETENTION
@@ -57,15 +59,18 @@ def test_threshold_overrides_take_precedence():
env={
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.6",
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.7",
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.75",
},
overrides=ConfigOverrides(
glossary_confidence_threshold=0.85,
homophones_confidence_threshold=0.9,
spoken_word_confidence_threshold=0.95,
),
)
assert config.glossary_confidence_threshold == 0.85
assert config.homophones_confidence_threshold == 0.9
assert config.spoken_word_confidence_threshold == 0.95
@pytest.mark.parametrize(
@@ -86,6 +91,7 @@ def test_invalid_module_sequences_are_rejected(value):
[
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD",
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD",
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD",
],
)
def test_invalid_thresholds_are_rejected(env_name):

17
tests/test_new_pipeline.py
View File

@@ -60,6 +60,7 @@ def test_process_transcript_runs_noop_framework(tmp_path):
{"corrections": []},
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
revised = process_transcript(
@@ -76,6 +77,7 @@ def test_process_transcript_runs_noop_framework(tmp_path):
"glossary_1:proposal",
"homophones:proposal",
"glossary_2:proposal",
"spoken_word:proposal",
]
@@ -105,6 +107,7 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
{"corrections": []},
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
@@ -126,6 +129,12 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
assert [validator["name"] for validator in result.report.modules[3].to_dict()["validators"]] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"spoken_word_review",
"meaning_reversal_review",
]
def test_external_report_can_be_written(tmp_path):
@@ -135,6 +144,7 @@ def test_external_report_can_be_written(tmp_path):
{"corrections": []},
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
@@ -175,7 +185,12 @@ def test_default_module_specs_expose_final_validator_order():
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
assert [validator.name for validator in specs[3].module.validators()] == ["protected_glossary_guard"]
assert [validator.name for validator in specs[3].module.validators()] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"spoken_word_review",
"meaning_reversal_review",
]
assert [validator.name for validator in specs[4].module.validators()] == ["protected_glossary_guard"]