eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Completed the MVP for the new Audita refactor

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-24 13:17:36 -05:00
parent 34b3c09e43
commit 2d1d21d314
20 changed files with 1112 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
tests/test_framework_runner.py
View File

@@ -3,7 +3,12 @@ from audita.core.errors import AuditaLLMError
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
from audita.framework.models import CorrectionProposal, ModuleContext, ModuleRunSpec
from audita.framework.runner import PipelineRunner
from audita.validators import MeaningReversalValidator, ProtectedGlossaryTermsValidator, SpokenFormPlausibilityValidator
from audita.validators import (
MeaningReversalValidator,
ProposalConfidenceValidator,
ProtectedGlossaryTermsValidator,
SpokenFormPlausibilityValidator,
)
from audita.validators.base import ValidationContext, ValidationDecision, ValidationResult
@@ -67,7 +72,7 @@ class RecordingModule:
return list(self._validators)
def propose(self, transcript_section, context: ModuleContext):
self._recorder.append(("propose", [segment.text for segment in transcript_section]))
self._recorder.append(("propose", [item.segment.text for item in transcript_section.segments]))
return list(self._proposals)
@@ -277,6 +282,7 @@ def test_pipeline_runner_supports_real_llm_validators_in_one_chain(tmp_path):
)
],
[
ProposalConfidenceValidator("proposal_confidence_guard", "glossary_confidence_threshold"),
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
MeaningReversalValidator("meaning_reversal_review"),
@@ -320,6 +326,7 @@ def test_pipeline_runner_supports_real_llm_validators_in_one_chain(tmp_path):
assert result.transcript[0].text == "There were Jesters at the dam."
assert [report.execution_kind for report in result.module_reports[0].validators] == [
"deterministic",
"deterministic",
"llm",
"llm",

273
tests/test_module_proposals.py Normal file
View File

@@ -0,0 +1,273 @@
from pathlib import Path
from audita.core.chunking import chunk_transcript
from audita.core.config import AuditaConfig
from audita.core.errors import AuditaLLMError
from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json, parse_transcript_json
from audita.framework.models import ModuleContext, ModuleRunSpec
from audita.modules.glossary import GlossaryModule
from audita.modules.homophones import HomophonesModule
from audita.modules.prompts import build_homophones_proposal_messages
from audita.pipeline import process_transcript_result
class FakeStructuredLLMClient:
def __init__(self, responses):
self._responses = list(responses)
self.calls = []
def run_structured(self, *, stage_name, messages, response_model, config):
self.calls.append(
{
"stage_name": stage_name,
"messages": list(messages),
"response_model": response_model,
}
)
if not self._responses:
raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.")
return response_model.model_validate(self._responses.pop(0))
def _glossary():
return parse_glossary_yaml(
"""
glossary:
- name: "Jesters"
aliases:
- "Jester"
category: faction
summary: "A faction."
- name: "Hrank"
category: pc
summary: "A player character."
"""
)
def test_glossary_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
]
"""
)
section = chunk_transcript(transcript, max_section_tokens=1000)[0]
module = GlossaryModule()
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "gestures",
"corrected_text": "Jesters",
"confidence": 0.95,
}
]
}
]
)
context = ModuleContext(
run_spec=ModuleRunSpec(instance_name="glossary_primary", module_key="glossary", module=module),
glossary=_glossary(),
config=AuditaConfig.from_sources(env={}),
run_dir=tmp_path,
llm_client=client,
)
proposals = list(module.propose(section, context))
assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [
(1, "gestures", "Jesters", 0.95)
]
assert (tmp_path / "prompt-0000.json").exists()
assert (tmp_path / "corrections-0000.json").exists()
prompt_text = client.calls[0]["messages"][1]["content"]
assert "Glossary:" in prompt_text
assert "exact text span" in prompt_text
assert "gestures" in prompt_text
def test_homophones_prompt_is_explicitly_scoped_to_spoken_form_corrections():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't do that with a dam."}
]
"""
)
section = chunk_transcript(transcript, max_section_tokens=1000)[0]
messages = build_homophones_proposal_messages(section, _glossary())
combined = messages[0]["content"] + messages[1]["content"]
assert "homophone" in combined
assert "mistranscription" in combined
assert "Do not add or remove punctuation" in combined
assert "visible" in combined
assert '"id": 1' in messages[1]["content"]
def test_process_transcript_result_uses_injected_fake_client_and_applies_sequential_module_updates(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."}
]
"""
)
config = AuditaConfig.from_sources(
env={},
overrides=None,
)
config = AuditaConfig(
api_key=config.api_key,
model=config.model,
base_url=config.base_url,
max_retries=config.max_retries,
max_section_tokens=config.max_section_tokens,
glossary_confidence_threshold=config.glossary_confidence_threshold,
homophones_confidence_threshold=config.homophones_confidence_threshold,
normalize_max_segment_gap=config.normalize_max_segment_gap,
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
normalize_max_segment_duration=config.normalize_max_segment_duration,
normalize_max_segment_tokens=config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="always",
)
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "gestures",
"corrected_text": "Jesters",
"confidence": 0.95,
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.97,
"reason": "Likely spoken-form correction in context.",
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.99,
"reason": "Does not reverse the segment meaning.",
}
]
},
{
"corrections": [
{
"id": 1,
"original_text": "dam",
"corrected_text": "damn",
"confidence": 0.92,
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.98,
"reason": "Likely spoken-form correction in context.",
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.99,
"reason": "Does not reverse the segment meaning.",
}
]
},
{"corrections": []},
]
)
result = process_transcript_result(transcript, _glossary(), config, llm_client=client)
assert result.transcript[0].text == "There were Jesters at the damn."
assert [call["stage_name"] for call in client.calls] == [
"glossary_primary:proposal",
"glossary_primary:spoken_form_plausibility_review",
"glossary_primary:meaning_reversal_review",
"homophones:proposal",
"homophones:spoken_form_plausibility_review",
"homophones:meaning_reversal_review",
"glossary_secondary:proposal",
]
assert "There were Jesters at the dam." in client.calls[3]["messages"][1]["content"]
def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_validators(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
]
"""
)
base_config = AuditaConfig.from_sources(env={})
config = AuditaConfig(
api_key=base_config.api_key,
model=base_config.model,
base_url=base_config.base_url,
max_retries=base_config.max_retries,
max_section_tokens=base_config.max_section_tokens,
glossary_confidence_threshold=0.96,
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
normalize_max_segment_tokens=base_config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="always",
)
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "gestures",
"corrected_text": "Jesters",
"confidence": 0.95,
}
]
},
{"corrections": []},
{"corrections": []},
]
)
result = process_transcript_result(transcript, _glossary(), config, llm_client=client)
assert result.transcript[0].text == "There were gestures at the temple."
assert [call["stage_name"] for call in client.calls] == [
"glossary_primary:proposal",
"homophones:proposal",
"glossary_secondary:proposal",
]
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
assert result.report.skipped_corrections[0].source == "validator:proposal_confidence_guard"
assert result.report.modules[0].validators[0].rejected_count == 1
assert result.report.modules[0].validators[1].candidate_count == 0

3
tests/test_new_cli.py
View File

@@ -24,9 +24,10 @@ def test_process_help_exposes_framework_flags(capsys):
assert "--base-url" in output
assert "--max-retries" in output
assert "--max-section-tokens" in output
assert "--glossary-confidence-threshold" in output
assert "--homophones-confidence-threshold" in output
assert "--work-dir-retention" in output
assert "--normalize-max-segment-gap" in output
assert "--glossary-confidence-threshold" not in output
assert "--grammar-validation-enabled" not in output

32
tests/test_new_config.py
View File

@@ -3,6 +3,8 @@ import pytest
from audita.core.config import (
AuditaConfig,
ConfigOverrides,
DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD,
DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD,
DEFAULT_NORMALIZE_MAX_SEGMENT_GAP,
DEFAULT_WORK_DIR_RETENTION,
)
@@ -13,6 +15,8 @@ def test_default_config_allows_missing_api_key():
config = AuditaConfig.from_sources(env={})
assert config.api_key is None
assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
assert config.homophones_confidence_threshold == DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD
assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
assert config.work_dir_retention == DEFAULT_WORK_DIR_RETENTION
@@ -29,3 +33,31 @@ def test_cli_overrides_take_precedence():
def test_invalid_work_dir_retention_is_rejected():
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(env={"AUDITA_WORK_DIR_RETENTION": "sometimes"})
def test_threshold_overrides_take_precedence():
config = AuditaConfig.from_sources(
env={
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.6",
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.7",
},
overrides=ConfigOverrides(
glossary_confidence_threshold=0.85,
homophones_confidence_threshold=0.9,
),
)
assert config.glossary_confidence_threshold == 0.85
assert config.homophones_confidence_threshold == 0.9
@pytest.mark.parametrize(
"env_name",
[
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD",
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD",
],
)
def test_invalid_thresholds_are_rejected(env_name):
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(env={env_name: "1.5"})

253
tests/test_new_pipeline.py
View File

@@ -1,12 +1,36 @@
import json
import pytest
from audita.core.config import AuditaConfig
from audita.core.errors import AuditaLLMError
from audita.core.io import write_report
from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json
from audita.modules import default_module_specs
from audita.pipeline import process_transcript, process_transcript_result
class FakeStructuredLLMClient:
def __init__(self, responses):
self._responses = list(responses)
self.calls = []
def run_structured(self, *, stage_name, messages, response_model, config):
self.calls.append(
{
"stage_name": stage_name,
"messages": list(messages),
"response_model": response_model,
}
)
if not self._responses:
raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.")
response = self._responses.pop(0)
if isinstance(response, Exception):
raise response
return response_model.model_validate(response)
def _glossary():
return parse_glossary_yaml(
"""
@@ -31,15 +55,28 @@ def _transcript():
def test_process_transcript_runs_noop_framework(tmp_path):
llm_client = FakeStructuredLLMClient(
[
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
revised = process_transcript(
_transcript(),
_glossary(),
AuditaConfig.from_sources(env={}, overrides=None),
llm_client=llm_client,
)
assert [segment.id for segment in revised] == [1, 2]
assert revised[0].text == "Hello. Again."
assert revised[1].text == "Done."
assert [call["stage_name"] for call in llm_client.calls] == [
"glossary_primary:proposal",
"homophones:proposal",
"glossary_secondary:proposal",
]
def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(tmp_path):
@@ -53,6 +90,8 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
base_url=config.base_url,
max_retries=config.max_retries,
max_section_tokens=config.max_section_tokens,
glossary_confidence_threshold=config.glossary_confidence_threshold,
homophones_confidence_threshold=config.homophones_confidence_threshold,
normalize_max_segment_gap=config.normalize_max_segment_gap,
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
normalize_max_segment_duration=config.normalize_max_segment_duration,
@@ -61,7 +100,14 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
work_dir_retention="always",
)
result = process_transcript_result(_transcript(), _glossary(), config)
llm_client = FakeStructuredLLMClient(
[
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
assert result.work_dir_retained is True
assert result.report.pipeline == [
@@ -75,6 +121,7 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
assert (result.run_dir / "report.json").exists()
assert (result.run_dir / "normalization" / "summary.json").exists()
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"spoken_form_plausibility_review",
"meaning_reversal_review",
@@ -83,7 +130,14 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
def test_external_report_can_be_written(tmp_path):
config = AuditaConfig.from_sources(env={})
result = process_transcript_result(_transcript(), _glossary(), config)
llm_client = FakeStructuredLLMClient(
[
{"corrections": []},
{"corrections": []},
{"corrections": []},
]
)
result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
report_path = tmp_path / "report.json"
write_report(report_path, result.report)
@@ -96,19 +150,214 @@ def test_default_module_specs_expose_final_validator_order():
specs = default_module_specs()
assert [validator.name for validator in specs[0].module.validators()] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
assert [validator.name for validator in specs[1].module.validators()] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
assert [validator.name for validator in specs[2].module.validators()] == [
"proposal_confidence_guard",
"protected_glossary_guard",
"spoken_form_plausibility_review",
"meaning_reversal_review",
]
assert [validator.name for validator in specs[3].module.validators()] == ["protected_glossary_guard"]
assert [validator.name for validator in specs[4].module.validators()] == ["protected_glossary_guard"]
def test_process_transcript_result_missing_api_key_writes_failed_report(tmp_path):
config = AuditaConfig.from_sources(
env={},
overrides=None,
)
config = AuditaConfig(
api_key=None,
model=config.model,
base_url=config.base_url,
max_retries=config.max_retries,
max_section_tokens=config.max_section_tokens,
glossary_confidence_threshold=config.glossary_confidence_threshold,
homophones_confidence_threshold=config.homophones_confidence_threshold,
normalize_max_segment_gap=config.normalize_max_segment_gap,
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
normalize_max_segment_duration=config.normalize_max_segment_duration,
normalize_max_segment_tokens=config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="never",
)
with pytest.raises(AuditaLLMError, match="OPENROUTER_API_KEY"):
process_transcript_result(_transcript(), _glossary(), config)
run_dir = next((tmp_path / "work").iterdir())
report = json.loads((run_dir / "report.json").read_text(encoding="utf-8"))
assert report["status"] == "failed"
assert report["normalization"]["normalized_segment_count"] == 2
assert report["pipeline"] == [
"glossary_primary",
"homophones",
"glossary_secondary",
"spoken_word",
"grammar",
]
assert report["modules"] == []
assert report["applied_changes"] == []
assert report["skipped_corrections"] == []
assert report["work_dir_retained"] is True
assert report["work_dir"] == str(run_dir)
assert "OPENROUTER_API_KEY" in report["error"]
def test_process_transcript_result_preserves_partial_progress_when_later_module_fails(tmp_path):
transcript = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."}
]
"""
)
config = AuditaConfig.from_sources(
env={},
overrides=None,
)
config = AuditaConfig(
api_key=config.api_key,
model=config.model,
base_url=config.base_url,
max_retries=config.max_retries,
max_section_tokens=config.max_section_tokens,
glossary_confidence_threshold=config.glossary_confidence_threshold,
homophones_confidence_threshold=config.homophones_confidence_threshold,
normalize_max_segment_gap=config.normalize_max_segment_gap,
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
normalize_max_segment_duration=config.normalize_max_segment_duration,
normalize_max_segment_tokens=config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="never",
)
llm_client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "gestures",
"corrected_text": "Jesters",
"confidence": 0.95,
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.97,
"reason": "Likely spoken-form correction in context.",
}
]
},
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.99,
"reason": "Does not reverse the segment meaning.",
}
]
},
AuditaLLMError("Simulated homophones proposal failure."),
]
)
with pytest.raises(AuditaLLMError, match="Simulated homophones proposal failure"):
process_transcript_result(transcript, _glossary(), config, llm_client=llm_client)
run_dir = next((tmp_path / "work").iterdir())
report = json.loads((run_dir / "report.json").read_text(encoding="utf-8"))
assert report["status"] == "failed"
assert report["normalization"]["normalized_segment_count"] == 1
assert [module["instance_name"] for module in report["modules"]] == ["glossary_primary"]
assert report["applied_changes"][0]["corrected_text"] == "Jesters"
assert report["applied_changes"][0]["segment_text_after"] == "There were Jesters at the dam."
assert report["totals"]["applied_change_count"] == 1
assert report["skipped_corrections"] == []
assert report["pipeline"][1] == "homophones"
assert "Simulated homophones proposal failure." in report["error"]
def test_process_transcript_result_preserves_partial_skips_and_validator_diagnostics_on_failure(tmp_path):
config = AuditaConfig.from_sources(
env={},
overrides=None,
)
config = AuditaConfig(
api_key=config.api_key,
model=config.model,
base_url=config.base_url,
max_retries=config.max_retries,
max_section_tokens=config.max_section_tokens,
glossary_confidence_threshold=0.8,
homophones_confidence_threshold=config.homophones_confidence_threshold,
normalize_max_segment_gap=config.normalize_max_segment_gap,
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
normalize_max_segment_duration=config.normalize_max_segment_duration,
normalize_max_segment_tokens=config.normalize_max_segment_tokens,
work_dir=tmp_path / "work",
work_dir_retention="never",
)
llm_client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "Hello",
"corrected_text": "Jesters",
"confidence": 0.40,
},
{
"id": 1,
"original_text": "Hello",
"corrected_text": "Jesters",
"confidence": 0.95,
},
]
},
{
"validations": [
{
"correction_index": 99,
"approved": True,
"confidence": 0.98,
"reason": "Malformed response for testing.",
}
]
},
]
)
with pytest.raises(AuditaLLMError, match="unknown correction_index"):
process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
run_dir = next((tmp_path / "work").iterdir())
report = json.loads((run_dir / "report.json").read_text(encoding="utf-8"))
validator_dir = run_dir / "glossary_primary"
assert report["status"] == "failed"
assert report["modules"] == []
assert len(report["skipped_corrections"]) == 1
assert report["skipped_corrections"][0]["reason"] == "proposal confidence below threshold"
assert report["skipped_corrections"][0]["source"] == "validator:proposal_confidence_guard"
assert "unknown correction_index" in report["error"]
assert (validator_dir / "spoken_form_plausibility_review-prompt-0000.json").exists()
assert (validator_dir / "spoken_form_plausibility_review-response-0000.json").exists()