499 lines
16 KiB
Python
499 lines
16 KiB
Python
import pytest
|
|
|
|
from audita.cli import main
|
|
from audita.core.reporting import ProcessResult, RunReport
|
|
from audita.core.schemas import parse_transcript_json
|
|
|
|
|
|
def test_cli_help_uses_audita_program_name(capsys):
|
|
with pytest.raises(SystemExit) as exc:
|
|
main(["--help"])
|
|
|
|
assert exc.value.code == 0
|
|
assert capsys.readouterr().out.startswith("usage: audita ")
|
|
|
|
|
|
def test_process_help_exposes_framework_flags(capsys):
|
|
with pytest.raises(SystemExit) as exc:
|
|
main(["process", "--help"])
|
|
|
|
assert exc.value.code == 0
|
|
output = capsys.readouterr().out
|
|
assert "--report-json" in output
|
|
assert "--llm-api-key" in output
|
|
assert "--llm-concurrency" in output
|
|
assert "--llm-timeout-seconds" in output
|
|
assert "--validation-llm-api-key" in output
|
|
assert "--validation-llm-concurrency" in output
|
|
assert "--validation-llm-timeout-seconds" in output
|
|
assert "--validation-model" in output
|
|
assert "--validation-base-url" in output
|
|
assert "--validation-max-retries" in output
|
|
assert "--target-sections" in output
|
|
assert "--modules" in output
|
|
assert "--model" in output
|
|
assert "--base-url" in output
|
|
assert "--max-retries" in output
|
|
assert "--max-section-tokens" in output
|
|
assert "--min-section-tokens" in output
|
|
assert "--glossary-confidence-threshold" in output
|
|
assert "--grammar-confidence-threshold" in output
|
|
assert "--homophones-confidence-threshold" in output
|
|
assert "--spoken-word-confidence-threshold" in output
|
|
assert "--work-dir-retention" in output
|
|
assert "--normalize-max-segment-gap" in output
|
|
assert "--grammar-validation-enabled" not in output
|
|
|
|
|
|
def test_cli_process_writes_report_json(monkeypatch, tmp_path):
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
|
|
]
|
|
"""
|
|
)
|
|
report = RunReport(
|
|
status="success",
|
|
config={"model": "m", "base_url": "b"},
|
|
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
|
|
pipeline=["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
|
|
modules=[],
|
|
applied_changes=[],
|
|
skipped_corrections=[],
|
|
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
|
|
work_dir_retention="auto",
|
|
work_dir_retained=False,
|
|
work_dir=None,
|
|
error=None,
|
|
)
|
|
result = ProcessResult(
|
|
transcript=transcript,
|
|
report=report,
|
|
run_dir=tmp_path / "run",
|
|
work_dir_retained=False,
|
|
)
|
|
|
|
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", lambda overrides=None: object())
|
|
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
|
|
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
|
|
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
|
|
|
|
output_path = tmp_path / "out.json"
|
|
report_path = tmp_path / "report.json"
|
|
exit_code = main(
|
|
[
|
|
"process",
|
|
"transcript.json",
|
|
"--glossary",
|
|
"glossary.yaml",
|
|
"--output",
|
|
str(output_path),
|
|
"--report-json",
|
|
str(report_path),
|
|
]
|
|
)
|
|
|
|
assert exit_code == 0
|
|
assert report_path.exists()
|
|
|
|
|
|
def test_cli_process_passes_modules_override_to_config(monkeypatch, tmp_path):
|
|
captured = {}
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
|
|
]
|
|
"""
|
|
)
|
|
report = RunReport(
|
|
status="success",
|
|
config={"model": "m", "base_url": "b"},
|
|
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
|
|
pipeline=["grammar"],
|
|
modules=[],
|
|
applied_changes=[],
|
|
skipped_corrections=[],
|
|
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
|
|
work_dir_retention="auto",
|
|
work_dir_retained=False,
|
|
work_dir=None,
|
|
error=None,
|
|
)
|
|
result = ProcessResult(
|
|
transcript=transcript,
|
|
report=report,
|
|
run_dir=tmp_path / "run",
|
|
work_dir_retained=False,
|
|
)
|
|
|
|
def _fake_from_sources(*, overrides=None):
|
|
captured["module_keys"] = overrides.module_keys
|
|
return object()
|
|
|
|
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
|
|
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
|
|
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
|
|
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
|
|
|
|
exit_code = main(
|
|
[
|
|
"process",
|
|
"transcript.json",
|
|
"--glossary",
|
|
"glossary.yaml",
|
|
"--modules",
|
|
"grammar",
|
|
]
|
|
)
|
|
|
|
assert exit_code == 0
|
|
assert captured["module_keys"] == "grammar"
|
|
|
|
|
|
def test_cli_process_passes_llm_api_key_override_to_config(monkeypatch, tmp_path):
|
|
captured = {}
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
|
|
]
|
|
"""
|
|
)
|
|
report = RunReport(
|
|
status="success",
|
|
config={"model": "m", "base_url": "b"},
|
|
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
|
|
pipeline=["grammar"],
|
|
modules=[],
|
|
applied_changes=[],
|
|
skipped_corrections=[],
|
|
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
|
|
work_dir_retention="auto",
|
|
work_dir_retained=False,
|
|
work_dir=None,
|
|
error=None,
|
|
)
|
|
result = ProcessResult(
|
|
transcript=transcript,
|
|
report=report,
|
|
run_dir=tmp_path / "run",
|
|
work_dir_retained=False,
|
|
)
|
|
|
|
def _fake_from_sources(*, overrides=None):
|
|
captured["llm_api_key"] = overrides.llm_api_key
|
|
return object()
|
|
|
|
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
|
|
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
|
|
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
|
|
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
|
|
|
|
exit_code = main(
|
|
[
|
|
"process",
|
|
"transcript.json",
|
|
"--glossary",
|
|
"glossary.yaml",
|
|
"--llm-api-key",
|
|
"cli-key",
|
|
]
|
|
)
|
|
|
|
assert exit_code == 0
|
|
assert captured["llm_api_key"] == "cli-key"
|
|
|
|
|
|
def test_cli_process_passes_llm_concurrency_override_to_config(monkeypatch, tmp_path):
|
|
captured = {}
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
|
|
]
|
|
"""
|
|
)
|
|
report = RunReport(
|
|
status="success",
|
|
config={"model": "m", "base_url": "b"},
|
|
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
|
|
pipeline=["grammar"],
|
|
modules=[],
|
|
applied_changes=[],
|
|
skipped_corrections=[],
|
|
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
|
|
work_dir_retention="auto",
|
|
work_dir_retained=False,
|
|
work_dir=None,
|
|
error=None,
|
|
)
|
|
result = ProcessResult(
|
|
transcript=transcript,
|
|
report=report,
|
|
run_dir=tmp_path / "run",
|
|
work_dir_retained=False,
|
|
)
|
|
|
|
def _fake_from_sources(*, overrides=None):
|
|
captured["llm_concurrency"] = overrides.llm_concurrency
|
|
return object()
|
|
|
|
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
|
|
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
|
|
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
|
|
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
|
|
|
|
exit_code = main(
|
|
[
|
|
"process",
|
|
"transcript.json",
|
|
"--glossary",
|
|
"glossary.yaml",
|
|
"--llm-concurrency",
|
|
"3",
|
|
]
|
|
)
|
|
|
|
assert exit_code == 0
|
|
assert captured["llm_concurrency"] == 3
|
|
|
|
|
|
def test_cli_process_passes_llm_timeout_seconds_override_to_config(monkeypatch, tmp_path):
|
|
captured = {}
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
|
|
]
|
|
"""
|
|
)
|
|
report = RunReport(
|
|
status="success",
|
|
config={"model": "m", "base_url": "b"},
|
|
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
|
|
pipeline=["grammar"],
|
|
modules=[],
|
|
applied_changes=[],
|
|
skipped_corrections=[],
|
|
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
|
|
work_dir_retention="auto",
|
|
work_dir_retained=False,
|
|
work_dir=None,
|
|
error=None,
|
|
)
|
|
result = ProcessResult(
|
|
transcript=transcript,
|
|
report=report,
|
|
run_dir=tmp_path / "run",
|
|
work_dir_retained=False,
|
|
)
|
|
|
|
def _fake_from_sources(*, overrides=None):
|
|
captured["llm_timeout_seconds"] = overrides.llm_timeout_seconds
|
|
return object()
|
|
|
|
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
|
|
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
|
|
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
|
|
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
|
|
|
|
exit_code = main(
|
|
[
|
|
"process",
|
|
"transcript.json",
|
|
"--glossary",
|
|
"glossary.yaml",
|
|
"--llm-timeout-seconds",
|
|
"900",
|
|
]
|
|
)
|
|
|
|
assert exit_code == 0
|
|
assert captured["llm_timeout_seconds"] == 900.0
|
|
|
|
|
|
def test_cli_process_passes_validation_llm_overrides_to_config(monkeypatch, tmp_path):
|
|
captured = {}
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
|
|
]
|
|
"""
|
|
)
|
|
report = RunReport(
|
|
status="success",
|
|
config={"model": "m", "base_url": "b"},
|
|
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
|
|
pipeline=["grammar"],
|
|
modules=[],
|
|
applied_changes=[],
|
|
skipped_corrections=[],
|
|
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
|
|
work_dir_retention="auto",
|
|
work_dir_retained=False,
|
|
work_dir=None,
|
|
error=None,
|
|
)
|
|
result = ProcessResult(
|
|
transcript=transcript,
|
|
report=report,
|
|
run_dir=tmp_path / "run",
|
|
work_dir_retained=False,
|
|
)
|
|
|
|
def _fake_from_sources(*, overrides=None):
|
|
captured["validation_llm_api_key"] = overrides.validation_llm_api_key
|
|
captured["validation_llm_concurrency"] = overrides.validation_llm_concurrency
|
|
captured["validation_llm_timeout_seconds"] = overrides.validation_llm_timeout_seconds
|
|
captured["validation_model"] = overrides.validation_model
|
|
captured["validation_base_url"] = overrides.validation_base_url
|
|
captured["validation_max_retries"] = overrides.validation_max_retries
|
|
return object()
|
|
|
|
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
|
|
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
|
|
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
|
|
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
|
|
|
|
exit_code = main(
|
|
[
|
|
"process",
|
|
"transcript.json",
|
|
"--glossary",
|
|
"glossary.yaml",
|
|
"--validation-llm-api-key",
|
|
"validator-key",
|
|
"--validation-llm-concurrency",
|
|
"4",
|
|
"--validation-llm-timeout-seconds",
|
|
"180",
|
|
"--validation-model",
|
|
"validator-model",
|
|
"--validation-base-url",
|
|
"http://localhost:9000/v1",
|
|
"--validation-max-retries",
|
|
"2",
|
|
]
|
|
)
|
|
|
|
assert exit_code == 0
|
|
assert captured == {
|
|
"validation_llm_api_key": "validator-key",
|
|
"validation_llm_concurrency": 4,
|
|
"validation_llm_timeout_seconds": 180.0,
|
|
"validation_model": "validator-model",
|
|
"validation_base_url": "http://localhost:9000/v1",
|
|
"validation_max_retries": 2,
|
|
}
|
|
|
|
|
|
def test_cli_process_passes_target_sections_override_to_config(monkeypatch, tmp_path):
|
|
captured = {}
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
|
|
]
|
|
"""
|
|
)
|
|
report = RunReport(
|
|
status="success",
|
|
config={"model": "m", "base_url": "b"},
|
|
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
|
|
pipeline=["grammar"],
|
|
modules=[],
|
|
applied_changes=[],
|
|
skipped_corrections=[],
|
|
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
|
|
work_dir_retention="auto",
|
|
work_dir_retained=False,
|
|
work_dir=None,
|
|
error=None,
|
|
)
|
|
result = ProcessResult(
|
|
transcript=transcript,
|
|
report=report,
|
|
run_dir=tmp_path / "run",
|
|
work_dir_retained=False,
|
|
)
|
|
|
|
def _fake_from_sources(*, overrides=None):
|
|
captured["target_sections"] = overrides.target_sections
|
|
return object()
|
|
|
|
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
|
|
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
|
|
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
|
|
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
|
|
|
|
exit_code = main(
|
|
[
|
|
"process",
|
|
"transcript.json",
|
|
"--glossary",
|
|
"glossary.yaml",
|
|
"--target-sections",
|
|
"4",
|
|
]
|
|
)
|
|
|
|
assert exit_code == 0
|
|
assert captured["target_sections"] == 4
|
|
|
|
|
|
def test_cli_process_passes_min_section_tokens_override_to_config(monkeypatch, tmp_path):
|
|
captured = {}
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
|
|
]
|
|
"""
|
|
)
|
|
report = RunReport(
|
|
status="success",
|
|
config={"model": "m", "base_url": "b"},
|
|
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
|
|
pipeline=["grammar"],
|
|
modules=[],
|
|
applied_changes=[],
|
|
skipped_corrections=[],
|
|
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
|
|
work_dir_retention="auto",
|
|
work_dir_retained=False,
|
|
work_dir=None,
|
|
error=None,
|
|
)
|
|
result = ProcessResult(
|
|
transcript=transcript,
|
|
report=report,
|
|
run_dir=tmp_path / "run",
|
|
work_dir_retained=False,
|
|
)
|
|
|
|
def _fake_from_sources(*, overrides=None):
|
|
captured["min_section_tokens"] = overrides.min_section_tokens
|
|
return object()
|
|
|
|
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
|
|
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
|
|
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
|
|
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
|
|
|
|
exit_code = main(
|
|
[
|
|
"process",
|
|
"transcript.json",
|
|
"--glossary",
|
|
"glossary.yaml",
|
|
"--min-section-tokens",
|
|
"5000",
|
|
]
|
|
)
|
|
|
|
assert exit_code == 0
|
|
assert captured["min_section_tokens"] == 5000
|