699 lines
25 KiB
Python
699 lines
25 KiB
Python
import json
|
|
import threading
|
|
|
|
import pytest
|
|
|
|
from audita.core.config import AuditaConfig, ConfigOverrides
|
|
from audita.core.errors import AuditaLLMError
|
|
from audita.core.io import write_report
|
|
from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json
|
|
from audita.modules import DEFAULT_MODULE_KEYS, default_module_specs, resolve_module_specs
|
|
from audita.pipeline import process_transcript, process_transcript_result
|
|
|
|
|
|
class FakeStructuredLLMClient:
|
|
def __init__(self, responses):
|
|
self._responses = responses
|
|
self._lock = threading.Lock()
|
|
self.calls = []
|
|
|
|
def run_structured(self, *, stage_name, messages, response_model, config):
|
|
with self._lock:
|
|
self.calls.append(
|
|
{
|
|
"stage_name": stage_name,
|
|
"messages": list(messages),
|
|
"response_model": response_model,
|
|
"config": config,
|
|
}
|
|
)
|
|
response = _pop_llm_response(self._responses, stage_name)
|
|
if isinstance(response, Exception):
|
|
raise response
|
|
return response_model.model_validate(response)
|
|
|
|
|
|
def _pop_llm_response(responses, stage_name):
|
|
if isinstance(responses, dict):
|
|
if stage_name not in responses:
|
|
raise AuditaLLMError(f"FakeStructuredLLMClient received unexpected stage_name: {stage_name}")
|
|
payloads = responses[stage_name]
|
|
if isinstance(payloads, list):
|
|
if not payloads:
|
|
raise AuditaLLMError(f"FakeStructuredLLMClient received too many calls for stage_name: {stage_name}")
|
|
return payloads.pop(0)
|
|
payload = payloads
|
|
del responses[stage_name]
|
|
return payload
|
|
if not responses:
|
|
raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.")
|
|
return responses.pop(0)
|
|
|
|
|
|
def _glossary():
|
|
return parse_glossary_yaml(
|
|
"""
|
|
glossary:
|
|
- name: "Jesters"
|
|
category: faction
|
|
summary: "A faction."
|
|
"""
|
|
)
|
|
|
|
|
|
def _transcript():
|
|
return parse_source_transcript_json(
|
|
"""
|
|
[
|
|
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hello.", "categories": ["intro"]},
|
|
{"speaker": "Eric", "start": 1.5, "end": 2.0, "text": "Again.", "categories": ["intro", "aside"]},
|
|
{"speaker": "Mike", "start": 10.0, "end": 11.0, "text": "Done.", "categories": ["response"]}
|
|
]
|
|
"""
|
|
)
|
|
|
|
|
|
def test_process_transcript_runs_noop_framework(tmp_path):
|
|
llm_client = FakeStructuredLLMClient(
|
|
[
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
]
|
|
)
|
|
revised = process_transcript(
|
|
_transcript(),
|
|
_glossary(),
|
|
AuditaConfig.from_sources(env={}, overrides=None),
|
|
llm_client=llm_client,
|
|
)
|
|
|
|
assert [segment.id for segment in revised] == [1, 2]
|
|
assert revised[0].text == "Hello. Again."
|
|
assert revised[1].text == "Done."
|
|
assert revised[0].categories == ["intro", "aside"]
|
|
assert revised[1].categories == ["response"]
|
|
assert [call["stage_name"] for call in llm_client.calls] == [
|
|
"glossary_1:proposal",
|
|
"homophones:proposal",
|
|
"glossary_2:proposal",
|
|
"spoken_word:proposal",
|
|
"grammar:proposal",
|
|
]
|
|
|
|
|
|
def test_process_transcript_result_can_use_different_validation_llm_settings(tmp_path):
|
|
llm_client = FakeStructuredLLMClient(
|
|
{
|
|
"grammar:proposal": {
|
|
"corrections": [
|
|
{
|
|
"id": 1,
|
|
"original_text": "hello world",
|
|
"corrected_text": "Hello world.",
|
|
"confidence": 0.95,
|
|
}
|
|
]
|
|
},
|
|
"grammar:grammar_only_guard": {
|
|
"validations": [
|
|
{
|
|
"correction_index": 0,
|
|
"approved": True,
|
|
"confidence": 0.98,
|
|
"reason": "ok",
|
|
}
|
|
]
|
|
},
|
|
"grammar:meaning_reversal_review": {
|
|
"validations": [
|
|
{
|
|
"correction_index": 0,
|
|
"approved": True,
|
|
"confidence": 0.99,
|
|
"reason": "ok",
|
|
}
|
|
]
|
|
},
|
|
}
|
|
)
|
|
config = AuditaConfig.from_sources(
|
|
env={"OPENROUTER_API_KEY": "primary-key"},
|
|
overrides=ConfigOverrides(
|
|
model="primary-model",
|
|
base_url="http://localhost:8000/v1",
|
|
max_retries=7,
|
|
llm_timeout_seconds=120,
|
|
validation_llm_api_key="validation-key",
|
|
validation_model="validation-model",
|
|
validation_base_url="http://localhost:9000/v1",
|
|
validation_max_retries=2,
|
|
validation_llm_timeout_seconds=240,
|
|
validation_llm_concurrency=3,
|
|
work_dir=tmp_path / "work",
|
|
work_dir_retention="always",
|
|
),
|
|
)
|
|
|
|
result = process_transcript_result(
|
|
parse_source_transcript_json(
|
|
"""
|
|
[
|
|
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
|
|
]
|
|
"""
|
|
),
|
|
_glossary(),
|
|
config,
|
|
module_keys=["grammar"],
|
|
llm_client=llm_client,
|
|
)
|
|
|
|
assert result.transcript[0].text == "Hello world."
|
|
calls_by_stage = {call["stage_name"]: call["config"] for call in llm_client.calls}
|
|
assert calls_by_stage["grammar:proposal"].model == "primary-model"
|
|
assert calls_by_stage["grammar:proposal"].base_url == "http://localhost:8000/v1"
|
|
assert calls_by_stage["grammar:proposal"].api_key == "primary-key"
|
|
assert calls_by_stage["grammar:grammar_only_guard"].model == "validation-model"
|
|
assert calls_by_stage["grammar:grammar_only_guard"].base_url == "http://localhost:9000/v1"
|
|
assert calls_by_stage["grammar:grammar_only_guard"].api_key == "validation-key"
|
|
assert calls_by_stage["grammar:grammar_only_guard"].max_retries == 2
|
|
assert calls_by_stage["grammar:grammar_only_guard"].llm_timeout_seconds == 240
|
|
|
|
|
|
def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(tmp_path):
|
|
config = AuditaConfig.from_sources(
|
|
env={},
|
|
overrides=None,
|
|
)
|
|
config = AuditaConfig(
|
|
api_key=config.api_key,
|
|
model=config.model,
|
|
base_url=config.base_url,
|
|
max_retries=config.max_retries,
|
|
max_section_tokens=config.max_section_tokens,
|
|
glossary_confidence_threshold=config.glossary_confidence_threshold,
|
|
grammar_confidence_threshold=config.grammar_confidence_threshold,
|
|
homophones_confidence_threshold=config.homophones_confidence_threshold,
|
|
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
|
|
normalize_max_segment_gap=config.normalize_max_segment_gap,
|
|
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
|
|
normalize_max_segment_duration=config.normalize_max_segment_duration,
|
|
normalize_max_segment_tokens=config.normalize_max_segment_tokens,
|
|
work_dir=tmp_path / "work",
|
|
work_dir_retention="always",
|
|
)
|
|
|
|
llm_client = FakeStructuredLLMClient(
|
|
[
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
]
|
|
)
|
|
result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
|
|
|
|
assert result.work_dir_retained is True
|
|
assert result.report.pipeline == [
|
|
"glossary_1",
|
|
"homophones",
|
|
"glossary_2",
|
|
"spoken_word",
|
|
"grammar",
|
|
]
|
|
assert result.report.totals["applied_change_count"] == 0
|
|
assert (result.run_dir / "report.json").exists()
|
|
assert (result.run_dir / "normalization" / "summary.json").exists()
|
|
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
|
|
"identical_text_guard",
|
|
"original_text_present_guard",
|
|
"proposal_confidence_guard",
|
|
"glossary_stage_protected_glossary_guard",
|
|
"non_empty_segment_guard",
|
|
"spoken_form_plausibility_review",
|
|
"meaning_reversal_review",
|
|
]
|
|
assert [validator["name"] for validator in result.report.modules[2].to_dict()["validators"]] == [
|
|
"identical_text_guard",
|
|
"original_text_present_guard",
|
|
"proposal_confidence_guard",
|
|
"glossary_stage_protected_glossary_guard",
|
|
"non_empty_segment_guard",
|
|
"spoken_form_plausibility_review",
|
|
"meaning_reversal_review",
|
|
]
|
|
assert [validator["name"] for validator in result.report.modules[3].to_dict()["validators"]] == [
|
|
"identical_text_guard",
|
|
"original_text_present_guard",
|
|
"proposal_confidence_guard",
|
|
"protected_glossary_guard",
|
|
"non_empty_segment_guard",
|
|
"spoken_word_review",
|
|
"meaning_reversal_review",
|
|
]
|
|
assert [validator["name"] for validator in result.report.modules[4].to_dict()["validators"]] == [
|
|
"identical_text_guard",
|
|
"original_text_present_guard",
|
|
"proposal_confidence_guard",
|
|
"protected_glossary_guard",
|
|
"non_empty_segment_guard",
|
|
"grammar_only_guard",
|
|
"meaning_reversal_review",
|
|
]
|
|
|
|
|
|
def test_external_report_can_be_written(tmp_path):
|
|
config = AuditaConfig.from_sources(env={})
|
|
llm_client = FakeStructuredLLMClient(
|
|
[
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
]
|
|
)
|
|
result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
|
|
report_path = tmp_path / "report.json"
|
|
write_report(report_path, result.report)
|
|
|
|
payload = json.loads(report_path.read_text(encoding="utf-8"))
|
|
assert payload["pipeline"][0] == "glossary_1"
|
|
assert payload["totals"]["applied_change_count"] == 0
|
|
|
|
|
|
def test_process_transcript_preserves_categories_in_llm_prompt_payloads(tmp_path):
|
|
llm_client = FakeStructuredLLMClient(
|
|
[
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
]
|
|
)
|
|
|
|
process_transcript(
|
|
_transcript(),
|
|
_glossary(),
|
|
AuditaConfig.from_sources(env={}, overrides=None),
|
|
llm_client=llm_client,
|
|
)
|
|
|
|
proposal_prompt = llm_client.calls[0]["messages"][1]["content"]
|
|
assert '"categories": [' in proposal_prompt
|
|
assert '"intro"' in proposal_prompt
|
|
assert '"aside"' in proposal_prompt
|
|
|
|
|
|
def test_default_module_specs_expose_final_validator_order():
|
|
specs = default_module_specs()
|
|
|
|
assert DEFAULT_MODULE_KEYS == ("glossary", "homophones", "glossary", "spoken_word", "grammar")
|
|
assert [spec.instance_name for spec in specs] == [
|
|
"glossary_1",
|
|
"homophones",
|
|
"glossary_2",
|
|
"spoken_word",
|
|
"grammar",
|
|
]
|
|
assert [validator.name for validator in specs[0].module.validators()] == [
|
|
"identical_text_guard",
|
|
"original_text_present_guard",
|
|
"proposal_confidence_guard",
|
|
"glossary_stage_protected_glossary_guard",
|
|
"non_empty_segment_guard",
|
|
"spoken_form_plausibility_review",
|
|
"meaning_reversal_review",
|
|
]
|
|
assert [validator.name for validator in specs[1].module.validators()] == [
|
|
"identical_text_guard",
|
|
"original_text_present_guard",
|
|
"proposal_confidence_guard",
|
|
"protected_glossary_guard",
|
|
"non_empty_segment_guard",
|
|
"spoken_form_plausibility_review",
|
|
"meaning_reversal_review",
|
|
]
|
|
assert [validator.name for validator in specs[2].module.validators()] == [
|
|
"identical_text_guard",
|
|
"original_text_present_guard",
|
|
"proposal_confidence_guard",
|
|
"glossary_stage_protected_glossary_guard",
|
|
"non_empty_segment_guard",
|
|
"spoken_form_plausibility_review",
|
|
"meaning_reversal_review",
|
|
]
|
|
assert [validator.name for validator in specs[3].module.validators()] == [
|
|
"identical_text_guard",
|
|
"original_text_present_guard",
|
|
"proposal_confidence_guard",
|
|
"protected_glossary_guard",
|
|
"non_empty_segment_guard",
|
|
"spoken_word_review",
|
|
"meaning_reversal_review",
|
|
]
|
|
assert [validator.name for validator in specs[4].module.validators()] == [
|
|
"identical_text_guard",
|
|
"original_text_present_guard",
|
|
"proposal_confidence_guard",
|
|
"protected_glossary_guard",
|
|
"non_empty_segment_guard",
|
|
"grammar_only_guard",
|
|
"meaning_reversal_review",
|
|
]
|
|
|
|
|
|
def test_process_transcript_result_missing_api_key_writes_failed_report(tmp_path):
|
|
config = AuditaConfig.from_sources(
|
|
env={},
|
|
overrides=None,
|
|
)
|
|
config = AuditaConfig(
|
|
api_key=None,
|
|
model=config.model,
|
|
base_url=config.base_url,
|
|
max_retries=config.max_retries,
|
|
max_section_tokens=config.max_section_tokens,
|
|
glossary_confidence_threshold=config.glossary_confidence_threshold,
|
|
grammar_confidence_threshold=config.grammar_confidence_threshold,
|
|
homophones_confidence_threshold=config.homophones_confidence_threshold,
|
|
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
|
|
normalize_max_segment_gap=config.normalize_max_segment_gap,
|
|
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
|
|
normalize_max_segment_duration=config.normalize_max_segment_duration,
|
|
normalize_max_segment_tokens=config.normalize_max_segment_tokens,
|
|
work_dir=tmp_path / "work",
|
|
work_dir_retention="never",
|
|
)
|
|
|
|
with pytest.raises(AuditaLLMError, match="OpenRouter endpoint"):
|
|
process_transcript_result(_transcript(), _glossary(), config)
|
|
|
|
run_dir = next((tmp_path / "work").iterdir())
|
|
report = json.loads((run_dir / "report.json").read_text(encoding="utf-8"))
|
|
|
|
assert report["status"] == "failed"
|
|
assert report["normalization"]["normalized_segment_count"] == 2
|
|
assert report["pipeline"] == [
|
|
"glossary_1",
|
|
"homophones",
|
|
"glossary_2",
|
|
"spoken_word",
|
|
"grammar",
|
|
]
|
|
assert report["modules"] == []
|
|
assert report["applied_changes"] == []
|
|
assert report["skipped_corrections"] == []
|
|
assert report["work_dir_retained"] is True
|
|
assert report["work_dir"] == str(run_dir)
|
|
assert "AUDITA_LLM_API_KEY" in report["error"]
|
|
assert "OPENROUTER_API_KEY" in report["error"]
|
|
assert "OpenRouter endpoint" in report["error"]
|
|
assert report["error_details"]["type"] == "AuditaLLMError"
|
|
assert report["error_details"]["phase"] == "pipeline"
|
|
assert report["error_details"]["error_log"] == str(run_dir / "error.log")
|
|
assert (run_dir / "error.log").exists()
|
|
|
|
|
|
def test_process_transcript_result_allows_missing_api_key_for_nondefault_proposal_endpoint(tmp_path):
|
|
llm_client = FakeStructuredLLMClient(
|
|
[
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
{"corrections": []},
|
|
]
|
|
)
|
|
config = AuditaConfig.from_sources(
|
|
env={},
|
|
overrides=ConfigOverrides(
|
|
base_url="http://localhost:8000/v1",
|
|
model="meta-llama/Llama-3.1-8B-Instruct",
|
|
work_dir=tmp_path / "work",
|
|
work_dir_retention="always",
|
|
),
|
|
)
|
|
|
|
result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
|
|
|
|
assert result.report.status == "success"
|
|
assert llm_client.calls[0]["config"].api_key is None
|
|
assert llm_client.calls[0]["config"].base_url == "http://localhost:8000/v1"
|
|
|
|
|
|
def test_process_transcript_result_allows_missing_validation_api_key_for_nondefault_validation_endpoint(tmp_path):
|
|
llm_client = FakeStructuredLLMClient(
|
|
{
|
|
"grammar:proposal": {
|
|
"corrections": [
|
|
{
|
|
"id": 1,
|
|
"original_text": "hello world",
|
|
"corrected_text": "Hello world.",
|
|
"confidence": 0.95,
|
|
}
|
|
]
|
|
},
|
|
"grammar:grammar_only_guard": {
|
|
"validations": [
|
|
{
|
|
"correction_index": 0,
|
|
"approved": True,
|
|
"confidence": 0.98,
|
|
"reason": "ok",
|
|
}
|
|
]
|
|
},
|
|
"grammar:meaning_reversal_review": {
|
|
"validations": [
|
|
{
|
|
"correction_index": 0,
|
|
"approved": True,
|
|
"confidence": 0.99,
|
|
"reason": "ok",
|
|
}
|
|
]
|
|
},
|
|
}
|
|
)
|
|
config = AuditaConfig.from_sources(
|
|
env={"OPENROUTER_API_KEY": "primary-key"},
|
|
overrides=ConfigOverrides(
|
|
work_dir=tmp_path / "work",
|
|
work_dir_retention="always",
|
|
validation_llm_api_key=" ",
|
|
validation_base_url="http://localhost:9000/v1",
|
|
validation_model="meta-llama/Llama-3.1-8B-Instruct",
|
|
),
|
|
)
|
|
|
|
result = process_transcript_result(
|
|
parse_source_transcript_json(
|
|
"""
|
|
[
|
|
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
|
|
]
|
|
"""
|
|
),
|
|
_glossary(),
|
|
config,
|
|
module_keys=["grammar"],
|
|
llm_client=llm_client,
|
|
)
|
|
|
|
assert result.transcript[0].text == "Hello world."
|
|
calls_by_stage = {call["stage_name"]: call["config"] for call in llm_client.calls}
|
|
assert calls_by_stage["grammar:proposal"].api_key == "primary-key"
|
|
assert calls_by_stage["grammar:grammar_only_guard"].api_key == ""
|
|
assert calls_by_stage["grammar:grammar_only_guard"].base_url == "http://localhost:9000/v1"
|
|
|
|
|
|
def test_process_transcript_result_preserves_partial_progress_when_later_module_fails(tmp_path):
|
|
transcript = parse_source_transcript_json(
|
|
"""
|
|
[
|
|
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the dam."}
|
|
]
|
|
"""
|
|
)
|
|
config = AuditaConfig.from_sources(
|
|
env={},
|
|
overrides=None,
|
|
)
|
|
config = AuditaConfig(
|
|
api_key=config.api_key,
|
|
model=config.model,
|
|
base_url=config.base_url,
|
|
max_retries=config.max_retries,
|
|
max_section_tokens=config.max_section_tokens,
|
|
glossary_confidence_threshold=config.glossary_confidence_threshold,
|
|
grammar_confidence_threshold=config.grammar_confidence_threshold,
|
|
homophones_confidence_threshold=config.homophones_confidence_threshold,
|
|
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
|
|
normalize_max_segment_gap=config.normalize_max_segment_gap,
|
|
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
|
|
normalize_max_segment_duration=config.normalize_max_segment_duration,
|
|
normalize_max_segment_tokens=config.normalize_max_segment_tokens,
|
|
work_dir=tmp_path / "work",
|
|
work_dir_retention="never",
|
|
)
|
|
llm_client = FakeStructuredLLMClient(
|
|
[
|
|
{
|
|
"corrections": [
|
|
{
|
|
"id": 1,
|
|
"original_text": "gestures",
|
|
"corrected_text": "Jesters",
|
|
"confidence": 0.95,
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"validations": [
|
|
{
|
|
"correction_index": 0,
|
|
"approved": True,
|
|
"confidence": 0.97,
|
|
"reason": "Likely spoken-form correction in context.",
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"validations": [
|
|
{
|
|
"correction_index": 0,
|
|
"approved": True,
|
|
"confidence": 0.99,
|
|
"reason": "Does not reverse the segment meaning.",
|
|
}
|
|
]
|
|
},
|
|
AuditaLLMError("Simulated homophones proposal failure."),
|
|
]
|
|
)
|
|
|
|
with pytest.raises(AuditaLLMError, match="Simulated homophones proposal failure"):
|
|
process_transcript_result(transcript, _glossary(), config, llm_client=llm_client)
|
|
|
|
run_dir = next((tmp_path / "work").iterdir())
|
|
report = json.loads((run_dir / "report.json").read_text(encoding="utf-8"))
|
|
|
|
assert report["status"] == "failed"
|
|
assert report["normalization"]["normalized_segment_count"] == 1
|
|
assert [module["instance_name"] for module in report["modules"]] == ["glossary_1"]
|
|
assert report["applied_changes"][0]["corrected_text"] == "Jesters"
|
|
assert report["applied_changes"][0]["segment_text_after"] == "There were Jesters at the dam."
|
|
assert report["totals"]["applied_change_count"] == 1
|
|
assert report["skipped_corrections"] == []
|
|
assert report["pipeline"][1] == "homophones"
|
|
assert "Simulated homophones proposal failure." in report["error"]
|
|
assert report["error_details"]["module_instance"] == "homophones"
|
|
assert report["error_details"]["phase"] == "pipeline"
|
|
assert (run_dir / "error.log").exists()
|
|
|
|
|
|
def test_process_transcript_result_preserves_partial_skips_and_validator_diagnostics_on_failure(tmp_path):
|
|
config = AuditaConfig.from_sources(
|
|
env={},
|
|
overrides=None,
|
|
)
|
|
config = AuditaConfig(
|
|
api_key=config.api_key,
|
|
model=config.model,
|
|
base_url=config.base_url,
|
|
max_retries=config.max_retries,
|
|
max_section_tokens=config.max_section_tokens,
|
|
glossary_confidence_threshold=0.8,
|
|
grammar_confidence_threshold=config.grammar_confidence_threshold,
|
|
homophones_confidence_threshold=config.homophones_confidence_threshold,
|
|
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
|
|
normalize_max_segment_gap=config.normalize_max_segment_gap,
|
|
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
|
|
normalize_max_segment_duration=config.normalize_max_segment_duration,
|
|
normalize_max_segment_tokens=config.normalize_max_segment_tokens,
|
|
work_dir=tmp_path / "work",
|
|
work_dir_retention="never",
|
|
)
|
|
llm_client = FakeStructuredLLMClient(
|
|
[
|
|
{
|
|
"corrections": [
|
|
{
|
|
"id": 1,
|
|
"original_text": "Hello",
|
|
"corrected_text": "Jesters",
|
|
"confidence": 0.40,
|
|
},
|
|
{
|
|
"id": 1,
|
|
"original_text": "Hello",
|
|
"corrected_text": "Jesters",
|
|
"confidence": 0.95,
|
|
},
|
|
]
|
|
},
|
|
{
|
|
"validations": [
|
|
{
|
|
"correction_index": 99,
|
|
"approved": True,
|
|
"confidence": 0.98,
|
|
"reason": "Malformed response for testing.",
|
|
}
|
|
]
|
|
},
|
|
]
|
|
)
|
|
|
|
with pytest.raises(AuditaLLMError, match="unknown correction_index"):
|
|
process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
|
|
|
|
run_dir = next((tmp_path / "work").iterdir())
|
|
report = json.loads((run_dir / "report.json").read_text(encoding="utf-8"))
|
|
validator_dir = run_dir / "glossary_1"
|
|
|
|
assert report["status"] == "failed"
|
|
assert report["modules"] == []
|
|
assert len(report["skipped_corrections"]) == 1
|
|
assert report["skipped_corrections"][0]["reason"] == "proposal confidence below threshold"
|
|
assert report["skipped_corrections"][0]["source"] == "validator:proposal_confidence_guard"
|
|
assert "unknown correction_index" in report["error"]
|
|
assert report["error_details"]["module_instance"] == "glossary_1"
|
|
assert report["error_details"]["phase"] == "pipeline"
|
|
assert (run_dir / "error.log").exists()
|
|
assert (validator_dir / "spoken_form_plausibility_review-prompt-0000.json").exists()
|
|
assert (validator_dir / "spoken_form_plausibility_review-response-0000.json").exists()
|
|
|
|
|
|
def test_resolve_module_specs_numbers_repeated_keys():
|
|
specs = resolve_module_specs(["glossary", "homophones", "glossary"])
|
|
|
|
assert [spec.instance_name for spec in specs] == ["glossary_1", "homophones", "glossary_2"]
|
|
assert [spec.module_key for spec in specs] == ["glossary", "homophones", "glossary"]
|
|
|
|
|
|
def test_process_transcript_result_supports_grammar_only_module_override(tmp_path):
|
|
config = AuditaConfig.from_sources(
|
|
env={},
|
|
overrides=ConfigOverrides(work_dir=tmp_path / "work", work_dir_retention="always"),
|
|
)
|
|
|
|
result = process_transcript_result(
|
|
_transcript(),
|
|
_glossary(),
|
|
config,
|
|
module_keys=["grammar"],
|
|
llm_client=FakeStructuredLLMClient([{"corrections": []}]),
|
|
)
|
|
|
|
assert [segment.id for segment in result.transcript] == [1, 2]
|
|
assert result.report.pipeline == ["grammar"]
|
|
assert result.report.totals["applied_change_count"] == 0
|