eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Implemented a --modules CLI flag to allow runtime selection of the modules to be run

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-25 07:48:21 -05:00
parent 2d1d21d314
commit 92c8c371a6
9 changed files with 277 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
tests/test_module_proposals.py
View File

@@ -207,13 +207,13 @@ def test_process_transcript_result_uses_injected_fake_client_and_applies_sequent
assert result.transcript[0].text == "There were Jesters at the damn."
assert [call["stage_name"] for call in client.calls] == [
"glossary_primary:proposal",
"glossary_primary:spoken_form_plausibility_review",
"glossary_primary:meaning_reversal_review",
"glossary_1:proposal",
"glossary_1:spoken_form_plausibility_review",
"glossary_1:meaning_reversal_review",
"homophones:proposal",
"homophones:spoken_form_plausibility_review",
"homophones:meaning_reversal_review",
"glossary_secondary:proposal",
"glossary_2:proposal",
]
assert "There were Jesters at the dam." in client.calls[3]["messages"][1]["content"]
@@ -263,9 +263,9 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_
assert result.transcript[0].text == "There were gestures at the temple."
assert [call["stage_name"] for call in client.calls] == [
"glossary_primary:proposal",
"glossary_1:proposal",
"homophones:proposal",
"glossary_secondary:proposal",
"glossary_2:proposal",
]
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
assert result.report.skipped_corrections[0].source == "validator:proposal_confidence_guard"

57
tests/test_new_cli.py
View File

@@ -20,6 +20,7 @@ def test_process_help_exposes_framework_flags(capsys):
assert exc.value.code == 0
output = capsys.readouterr().out
assert "--report-json" in output
assert "--modules" in output
assert "--model" in output
assert "--base-url" in output
assert "--max-retries" in output
@@ -43,7 +44,7 @@ def test_cli_process_writes_report_json(monkeypatch, tmp_path):
status="success",
config={"model": "m", "base_url": "b"},
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
pipeline=["glossary_primary", "homophones", "glossary_secondary", "spoken_word", "grammar"],
pipeline=["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
modules=[],
applied_changes=[],
skipped_corrections=[],
@@ -82,3 +83,57 @@ def test_cli_process_writes_report_json(monkeypatch, tmp_path):
assert exit_code == 0
assert report_path.exists()
def test_cli_process_passes_modules_override_to_config(monkeypatch, tmp_path):
captured = {}
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
]
"""
)
report = RunReport(
status="success",
config={"model": "m", "base_url": "b"},
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
pipeline=["grammar"],
modules=[],
applied_changes=[],
skipped_corrections=[],
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
work_dir_retention="auto",
work_dir_retained=False,
work_dir=None,
error=None,
)
result = ProcessResult(
transcript=transcript,
report=report,
run_dir=tmp_path / "run",
work_dir_retained=False,
)
def _fake_from_sources(*, overrides=None):
captured["module_keys"] = overrides.module_keys
return object()
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--modules",
"grammar",
]
)
assert exit_code == 0
assert captured["module_keys"] == "grammar"

30
tests/test_new_config.py
View File

@@ -9,12 +9,14 @@ from audita.core.config import (
DEFAULT_WORK_DIR_RETENTION,
)
from audita.core.errors import AuditaConfigError
from audita.modules import DEFAULT_MODULE_KEYS
def test_default_config_allows_missing_api_key():
config = AuditaConfig.from_sources(env={})
assert config.api_key is None
assert config.module_keys == DEFAULT_MODULE_KEYS
assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
assert config.homophones_confidence_threshold == DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD
assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
@@ -30,6 +32,21 @@ def test_cli_overrides_take_precedence():
assert config.max_section_tokens == 2000
def test_module_key_overrides_take_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_MODULES": "grammar"},
overrides=ConfigOverrides(module_keys="homophones,grammar"),
)
assert config.module_keys == ("homophones", "grammar")
def test_module_key_env_is_parsed_and_trimmed():
config = AuditaConfig.from_sources(env={"AUDITA_MODULES": " glossary , grammar "})
assert config.module_keys == ("glossary", "grammar")
def test_invalid_work_dir_retention_is_rejected():
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(env={"AUDITA_WORK_DIR_RETENTION": "sometimes"})
@@ -51,6 +68,19 @@ def test_threshold_overrides_take_precedence():
assert config.homophones_confidence_threshold == 0.9
@pytest.mark.parametrize(
"value",
[
"",
"grammar,,homophones",
"bogus",
],
)
def test_invalid_module_sequences_are_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_MODULES"):
AuditaConfig.from_sources(env={"AUDITA_MODULES": value})
@pytest.mark.parametrize(
"env_name",
[

56
tests/test_new_pipeline.py
View File

@@ -2,11 +2,11 @@ import json
import pytest
from audita.core.config import AuditaConfig
from audita.core.config import AuditaConfig, ConfigOverrides
from audita.core.errors import AuditaLLMError
from audita.core.io import write_report
from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json
from audita.modules import default_module_specs
from audita.modules import DEFAULT_MODULE_KEYS, default_module_specs, resolve_module_specs
from audita.pipeline import process_transcript, process_transcript_result
@@ -73,9 +73,9 @@ def test_process_transcript_runs_noop_framework(tmp_path):
assert revised[0].text == "Hello. Again."
assert revised[1].text == "Done."
assert [call["stage_name"] for call in llm_client.calls] == [
"glossary_primary:proposal",
"glossary_1:proposal",
"homophones:proposal",
"glossary_secondary:proposal",
"glossary_2:proposal",
]
@@ -111,9 +111,9 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
assert result.work_dir_retained is True
assert result.report.pipeline == [
"glossary_primary",
"glossary_1",
"homophones",
"glossary_secondary",
"glossary_2",
"spoken_word",
"grammar",
]
@@ -142,13 +142,21 @@ def test_external_report_can_be_written(tmp_path):
write_report(report_path, result.report)
payload = json.loads(report_path.read_text(encoding="utf-8"))
assert payload["pipeline"][0] == "glossary_primary"
assert payload["pipeline"][0] == "glossary_1"
assert payload["totals"]["applied_change_count"] == 0
def test_default_module_specs_expose_final_validator_order():
specs = default_module_specs()
assert DEFAULT_MODULE_KEYS == ("glossary", "homophones", "glossary", "spoken_word", "grammar")
assert [spec.instance_name for spec in specs] == [
"glossary_1",
"homophones",
"glossary_2",
"spoken_word",
"grammar",
]
assert [validator.name for validator in specs[0].module.validators()] == [
"proposal_confidence_guard",
"protected_glossary_guard",
@@ -201,9 +209,9 @@ def test_process_transcript_result_missing_api_key_writes_failed_report(tmp_path
assert report["status"] == "failed"
assert report["normalization"]["normalized_segment_count"] == 2
assert report["pipeline"] == [
"glossary_primary",
"glossary_1",
"homophones",
"glossary_secondary",
"glossary_2",
"spoken_word",
"grammar",
]
@@ -286,7 +294,7 @@ def test_process_transcript_result_preserves_partial_progress_when_later_module_
assert report["status"] == "failed"
assert report["normalization"]["normalized_segment_count"] == 1
assert [module["instance_name"] for module in report["modules"]] == ["glossary_primary"]
assert [module["instance_name"] for module in report["modules"]] == ["glossary_1"]
assert report["applied_changes"][0]["corrected_text"] == "Jesters"
assert report["applied_changes"][0]["segment_text_after"] == "There were Jesters at the dam."
assert report["totals"]["applied_change_count"] == 1
@@ -351,7 +359,7 @@ def test_process_transcript_result_preserves_partial_skips_and_validator_diagnos
run_dir = next((tmp_path / "work").iterdir())
report = json.loads((run_dir / "report.json").read_text(encoding="utf-8"))
validator_dir = run_dir / "glossary_primary"
validator_dir = run_dir / "glossary_1"
assert report["status"] == "failed"
assert report["modules"] == []
@@ -361,3 +369,29 @@ def test_process_transcript_result_preserves_partial_skips_and_validator_diagnos
assert "unknown correction_index" in report["error"]
assert (validator_dir / "spoken_form_plausibility_review-prompt-0000.json").exists()
assert (validator_dir / "spoken_form_plausibility_review-response-0000.json").exists()
def test_resolve_module_specs_numbers_repeated_keys():
specs = resolve_module_specs(["glossary", "homophones", "glossary"])
assert [spec.instance_name for spec in specs] == ["glossary_1", "homophones", "glossary_2"]
assert [spec.module_key for spec in specs] == ["glossary", "homophones", "glossary"]
def test_process_transcript_result_supports_grammar_only_module_override(tmp_path):
config = AuditaConfig.from_sources(
env={},
overrides=ConfigOverrides(work_dir=tmp_path / "work", work_dir_retention="always"),
)
result = process_transcript_result(
_transcript(),
_glossary(),
config,
module_keys=["grammar"],
llm_client=FakeStructuredLLMClient([]),
)
assert [segment.id for segment in result.transcript] == [1, 2]
assert result.report.pipeline == ["grammar"]
assert result.report.totals["applied_change_count"] == 0