Implemented invocation-level failure diagnostics
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
import json
|
||||
import pytest
|
||||
|
||||
from audita.cli import main
|
||||
from audita.core.errors import AuditaConfigError
|
||||
from audita.core.reporting import ProcessResult, RunReport
|
||||
from audita.core.schemas import parse_transcript_json
|
||||
|
||||
@@ -501,3 +503,69 @@ def test_cli_process_passes_min_section_tokens_override_to_config(monkeypatch, t
|
||||
|
||||
assert exit_code == 0
|
||||
assert captured["min_section_tokens"] == 5000
|
||||
|
||||
|
||||
def test_cli_process_writes_failure_diagnostics_and_keeps_stdout_empty(monkeypatch, tmp_path, capsys):
|
||||
monkeypatch.setattr(
|
||||
"audita.cli.AuditaConfig.from_sources",
|
||||
lambda overrides=None: (_ for _ in ()).throw(AuditaConfigError("bad config")),
|
||||
)
|
||||
|
||||
report_path = tmp_path / "external-report.json"
|
||||
exit_code = main(
|
||||
[
|
||||
"process",
|
||||
"transcript.json",
|
||||
"--glossary",
|
||||
"glossary.yaml",
|
||||
"--work-dir",
|
||||
str(tmp_path / "work"),
|
||||
"--report-json",
|
||||
str(report_path),
|
||||
]
|
||||
)
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert exit_code == 1
|
||||
assert captured.out == ""
|
||||
assert "audita: error: bad config" in captured.err
|
||||
assert "audita: exit code: 1" in captured.err
|
||||
run_dir = next((tmp_path / "work").iterdir())
|
||||
assert f"audita: run directory: {run_dir}" in captured.err
|
||||
assert f"audita: error log: {run_dir / 'error.log'}" in captured.err
|
||||
assert f"audita: report: {run_dir / 'report.json'}" in captured.err
|
||||
assert (run_dir / "error.log").exists()
|
||||
report = json.loads((run_dir / "report.json").read_text(encoding="utf-8"))
|
||||
assert report["status"] == "failed"
|
||||
assert report["error"] == "bad config"
|
||||
assert report["error_details"]["phase"] == "config"
|
||||
assert report["error_details"]["type"] == "AuditaConfigError"
|
||||
external = json.loads(report_path.read_text(encoding="utf-8"))
|
||||
assert external["error"] == "bad config"
|
||||
|
||||
|
||||
def test_cli_process_writes_failure_diagnostics_for_unexpected_exceptions(monkeypatch, tmp_path, capsys):
|
||||
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", lambda overrides=None: object())
|
||||
monkeypatch.setattr("audita.cli.load_transcript", lambda path: (_ for _ in ()).throw(RuntimeError("boom")))
|
||||
|
||||
exit_code = main(
|
||||
[
|
||||
"process",
|
||||
"transcript.json",
|
||||
"--glossary",
|
||||
"glossary.yaml",
|
||||
"--work-dir",
|
||||
str(tmp_path / "work"),
|
||||
]
|
||||
)
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert exit_code == 1
|
||||
assert captured.out == ""
|
||||
assert "audita: error: boom" in captured.err
|
||||
run_dir = next((tmp_path / "work").iterdir())
|
||||
report = json.loads((run_dir / "report.json").read_text(encoding="utf-8"))
|
||||
assert report["status"] == "failed"
|
||||
assert report["error_details"]["phase"] == "transcript_load"
|
||||
assert report["error_details"]["type"] == "RuntimeError"
|
||||
assert "RuntimeError: boom" in (run_dir / "error.log").read_text(encoding="utf-8")
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -29,3 +30,46 @@ def test_root_launcher_help_smoke():
|
||||
|
||||
assert result.returncode == 0
|
||||
assert result.stdout.startswith("usage: audita ")
|
||||
|
||||
|
||||
def test_root_launcher_writes_error_log_when_uv_is_missing(tmp_path):
|
||||
work_dir = tmp_path / "work"
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(LAUNCHER), "process", "transcript.json", "--glossary", "glossary.yaml", "--work-dir", str(work_dir)],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env={**os.environ, "PATH": ""},
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert result.stdout == ""
|
||||
assert "audita: error: uv is required to run this launcher" in result.stderr
|
||||
run_dir = next(work_dir.iterdir())
|
||||
assert f"audita: run directory: {run_dir}" in result.stderr
|
||||
assert (run_dir / "error.log").exists()
|
||||
|
||||
|
||||
def test_root_launcher_preserves_child_exit_code_and_writes_fallback_error_log(tmp_path):
|
||||
work_dir = tmp_path / "work"
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
fake_uv = fake_bin / "uv"
|
||||
fake_uv.write_text("#!/bin/sh\nexit 120\n", encoding="utf-8")
|
||||
fake_uv.chmod(0o755)
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(LAUNCHER), "process", "transcript.json", "--glossary", "glossary.yaml", "--work-dir", str(work_dir)],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env={**os.environ, "PATH": str(fake_bin)},
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 120
|
||||
assert result.stdout == ""
|
||||
assert "audita: subprocess exited with status 120" in result.stderr
|
||||
run_dir = next(work_dir.iterdir())
|
||||
assert (run_dir / "error.log").exists()
|
||||
|
||||
@@ -414,6 +414,10 @@ def test_process_transcript_result_missing_api_key_writes_failed_report(tmp_path
|
||||
assert "AUDITA_LLM_API_KEY" in report["error"]
|
||||
assert "OPENROUTER_API_KEY" in report["error"]
|
||||
assert "OpenRouter endpoint" in report["error"]
|
||||
assert report["error_details"]["type"] == "AuditaLLMError"
|
||||
assert report["error_details"]["phase"] == "pipeline"
|
||||
assert report["error_details"]["error_log"] == str(run_dir / "error.log")
|
||||
assert (run_dir / "error.log").exists()
|
||||
|
||||
|
||||
def test_process_transcript_result_allows_missing_api_key_for_nondefault_proposal_endpoint(tmp_path):
|
||||
@@ -590,6 +594,9 @@ def test_process_transcript_result_preserves_partial_progress_when_later_module_
|
||||
assert report["skipped_corrections"] == []
|
||||
assert report["pipeline"][1] == "homophones"
|
||||
assert "Simulated homophones proposal failure." in report["error"]
|
||||
assert report["error_details"]["module_instance"] == "homophones"
|
||||
assert report["error_details"]["phase"] == "pipeline"
|
||||
assert (run_dir / "error.log").exists()
|
||||
|
||||
|
||||
def test_process_transcript_result_preserves_partial_skips_and_validator_diagnostics_on_failure(tmp_path):
|
||||
@@ -658,6 +665,9 @@ def test_process_transcript_result_preserves_partial_skips_and_validator_diagnos
|
||||
assert report["skipped_corrections"][0]["reason"] == "proposal confidence below threshold"
|
||||
assert report["skipped_corrections"][0]["source"] == "validator:proposal_confidence_guard"
|
||||
assert "unknown correction_index" in report["error"]
|
||||
assert report["error_details"]["module_instance"] == "glossary_1"
|
||||
assert report["error_details"]["phase"] == "pipeline"
|
||||
assert (run_dir / "error.log").exists()
|
||||
assert (validator_dir / "spoken_form_plausibility_review-prompt-0000.json").exists()
|
||||
assert (validator_dir / "spoken_form_plausibility_review-response-0000.json").exists()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user