Added validation-specific LLM configuration options
This commit is contained in:
@@ -24,6 +24,7 @@ class FakeStructuredLLMClient:
|
||||
"stage_name": stage_name,
|
||||
"messages": list(messages),
|
||||
"response_model": response_model,
|
||||
"config": config,
|
||||
}
|
||||
)
|
||||
response = _pop_llm_response(self._responses, stage_name)
|
||||
@@ -103,6 +104,85 @@ def test_process_transcript_runs_noop_framework(tmp_path):
|
||||
]
|
||||
|
||||
|
||||
def test_process_transcript_result_can_use_different_validation_llm_settings(tmp_path):
|
||||
llm_client = FakeStructuredLLMClient(
|
||||
{
|
||||
"grammar:proposal": {
|
||||
"corrections": [
|
||||
{
|
||||
"id": 1,
|
||||
"original_text": "hello world",
|
||||
"corrected_text": "Hello world.",
|
||||
"confidence": 0.95,
|
||||
}
|
||||
]
|
||||
},
|
||||
"grammar:grammar_only_guard": {
|
||||
"validations": [
|
||||
{
|
||||
"correction_index": 0,
|
||||
"approved": True,
|
||||
"confidence": 0.98,
|
||||
"reason": "ok",
|
||||
}
|
||||
]
|
||||
},
|
||||
"grammar:meaning_reversal_review": {
|
||||
"validations": [
|
||||
{
|
||||
"correction_index": 0,
|
||||
"approved": True,
|
||||
"confidence": 0.99,
|
||||
"reason": "ok",
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
)
|
||||
config = AuditaConfig.from_sources(
|
||||
env={"OPENROUTER_API_KEY": "primary-key"},
|
||||
overrides=ConfigOverrides(
|
||||
model="primary-model",
|
||||
base_url="http://localhost:8000/v1",
|
||||
max_retries=7,
|
||||
llm_timeout_seconds=120,
|
||||
validation_llm_api_key="validation-key",
|
||||
validation_model="validation-model",
|
||||
validation_base_url="http://localhost:9000/v1",
|
||||
validation_max_retries=2,
|
||||
validation_llm_timeout_seconds=240,
|
||||
validation_llm_concurrency=3,
|
||||
work_dir=tmp_path / "work",
|
||||
work_dir_retention="always",
|
||||
),
|
||||
)
|
||||
|
||||
result = process_transcript_result(
|
||||
parse_source_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
|
||||
]
|
||||
"""
|
||||
),
|
||||
_glossary(),
|
||||
config,
|
||||
module_keys=["grammar"],
|
||||
llm_client=llm_client,
|
||||
)
|
||||
|
||||
assert result.transcript[0].text == "Hello world."
|
||||
calls_by_stage = {call["stage_name"]: call["config"] for call in llm_client.calls}
|
||||
assert calls_by_stage["grammar:proposal"].model == "primary-model"
|
||||
assert calls_by_stage["grammar:proposal"].base_url == "http://localhost:8000/v1"
|
||||
assert calls_by_stage["grammar:proposal"].api_key == "primary-key"
|
||||
assert calls_by_stage["grammar:grammar_only_guard"].model == "validation-model"
|
||||
assert calls_by_stage["grammar:grammar_only_guard"].base_url == "http://localhost:9000/v1"
|
||||
assert calls_by_stage["grammar:grammar_only_guard"].api_key == "validation-key"
|
||||
assert calls_by_stage["grammar:grammar_only_guard"].max_retries == 2
|
||||
assert calls_by_stage["grammar:grammar_only_guard"].llm_timeout_seconds == 240
|
||||
|
||||
|
||||
def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(tmp_path):
|
||||
config = AuditaConfig.from_sources(
|
||||
env={},
|
||||
|
||||
Reference in New Issue
Block a user