eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Added validation-specific LLM configuration options

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-29 20:45:51 -05:00
parent 4c899122d9
commit 54ce3d08c9
13 changed files with 714 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tests/test_framework_llm.py
View File

@@ -19,6 +19,12 @@ def _config(**overrides):
"api_key": "test-key",
"llm_concurrency": base.llm_concurrency,
"llm_timeout_seconds": base.llm_timeout_seconds,
"validation_llm_api_key": base.validation_llm_api_key,
"validation_llm_concurrency": base.validation_llm_concurrency,
"validation_llm_timeout_seconds": base.validation_llm_timeout_seconds,
"validation_model": base.validation_model,
"validation_base_url": base.validation_base_url,
"validation_max_retries": base.validation_max_retries,
"module_keys": base.module_keys,
"model": base.model,
"base_url": base.base_url,
@@ -193,7 +199,7 @@ def test_client_cache_identity_uses_timeout(monkeypatch):
def test_missing_api_key_error_is_provider_neutral():
client = OpenAICompatibleStructuredLLMClient()
with pytest.raises(AuditaLLMError, match="AUDITA_LLM_API_KEY"):
with pytest.raises(AuditaLLMError, match="AUDITA_VALIDATION_LLM_API_KEY"):
client.run_structured(
stage_name="test-stage",
messages=[{"role": "user", "content": "Hello"}],

169
tests/test_llm_validators.py
View File

@@ -40,6 +40,7 @@ class FakeStructuredLLMClient:
"stage_name": stage_name,
"messages": list(messages),
"response_model": response_model,
"config": config,
}
)
if not self._responses:
@@ -62,6 +63,7 @@ class CoordinatedStructuredLLMClient:
"stage_name": stage_name,
"messages": list(messages),
"response_model": response_model,
"config": config,
}
)
if not self._responses:
@@ -97,15 +99,20 @@ def _context(
tmp_path,
replacement_policy="require_unique",
llm_concurrency=1,
validation_llm_concurrency=None,
):
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "test-key"},
overrides=ConfigOverrides(
llm_concurrency=llm_concurrency,
validation_llm_concurrency=validation_llm_concurrency,
),
)
return ValidationContext(
proposals=proposals,
transcript=transcript,
glossary=_glossary(),
config=AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "test-key"},
overrides=ConfigOverrides(llm_concurrency=llm_concurrency),
),
config=config.validation_llm_config(),
run_spec=ModuleRunSpec(
instance_name="homophones",
module_key="homophones",
@@ -1190,3 +1197,157 @@ def test_llm_validators_process_batches_concurrently_and_preserve_proposal_order
(1, True),
]
assert len(client.calls) == 2
def test_validator_uses_validation_llm_config(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
)
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.97,
"reason": "ok",
}
]
}
]
)
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "primary-key"},
overrides=ConfigOverrides(
model="primary-model",
base_url="http://localhost:8000/v1",
max_retries=7,
llm_timeout_seconds=120,
validation_llm_api_key="validation-key",
validation_model="validation-model",
validation_base_url="http://localhost:9000/v1",
validation_max_retries=2,
validation_llm_timeout_seconds=240,
validation_llm_concurrency=3,
),
)
context = ValidationContext(
proposals=proposals,
transcript=transcript,
glossary=_glossary(),
config=config.validation_llm_config(),
run_spec=ModuleRunSpec(
instance_name="homophones",
module_key="homophones",
module=_Module("require_unique"),
),
run_dir=tmp_path,
llm_client=client,
)
SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(context)
validation_config = client.calls[0]["config"]
assert validation_config.api_key == "validation-key"
assert validation_config.model == "validation-model"
assert validation_config.base_url == "http://localhost:9000/v1"
assert validation_config.max_retries == 2
assert validation_config.llm_timeout_seconds == 240
assert validation_config.llm_concurrency == 3
def test_validator_uses_validation_llm_concurrency_override(tmp_path, monkeypatch):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "ChatGPT still can't do that with a dam."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="homophones",
module_key="homophones",
id=2,
original_text="dam",
corrected_text="damn",
confidence=0.95,
),
]
client = CoordinatedStructuredLLMClient(
[
lambda **kwargs: {
"validations": [
{
"correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1,
"approved": True,
"confidence": 0.97,
"reason": "ok",
}
]
},
lambda **kwargs: {
"validations": [
{
"correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1,
"approved": True,
"confidence": 0.94,
"reason": "ok",
}
]
},
],
threading.Barrier(2, timeout=1.0),
)
def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message):
return [
TokenBatch(batch_index=0, items=list(items[:1]), token_count=1),
TokenBatch(batch_index=1, items=list(items[1:]), token_count=1),
]
monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items)
result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
_context(
proposals=proposals,
transcript=transcript,
llm_client=client,
tmp_path=tmp_path,
llm_concurrency=1,
validation_llm_concurrency=2,
)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
(0, True),
(1, True),
]
assert len(client.calls) == 2

58
tests/test_module_proposals.py
View File

@@ -31,6 +31,7 @@ class FakeStructuredLLMClient:
"stage_name": stage_name,
"messages": list(messages),
"response_model": response_model,
"config": config,
}
)
payload = _pop_llm_response(self._responses, stage_name)
@@ -115,6 +116,63 @@ def test_glossary_module_propose_writes_diagnostics_and_returns_proposals_withou
assert "gestures" in prompt_text
def test_module_propose_uses_primary_llm_config(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
]
"""
)
section = chunk_transcript(transcript, max_section_tokens=1000)[0]
module = GlossaryModule()
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "gestures",
"corrected_text": "Jesters",
"confidence": 0.95,
}
]
}
]
)
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "primary-key"},
overrides=ConfigOverrides(
model="primary-model",
base_url="http://localhost:8000/v1",
max_retries=7,
llm_timeout_seconds=120,
validation_llm_api_key="validation-key",
validation_model="validation-model",
validation_base_url="http://localhost:9000/v1",
validation_max_retries=2,
validation_llm_timeout_seconds=240,
validation_llm_concurrency=3,
),
)
context = ModuleContext(
run_spec=ModuleRunSpec(instance_name="glossary_primary", module_key="glossary", module=module),
glossary=_glossary(),
config=config,
run_dir=tmp_path,
llm_client=client,
)
module.propose(section, context)
proposal_config = client.calls[0]["config"]
assert proposal_config.api_key == "primary-key"
assert proposal_config.model == "primary-model"
assert proposal_config.base_url == "http://localhost:8000/v1"
assert proposal_config.max_retries == 7
assert proposal_config.llm_timeout_seconds == 120
def test_homophones_prompt_is_explicitly_scoped_to_spoken_form_corrections():
transcript = parse_transcript_json(
"""

82
tests/test_new_cli.py
View File

@@ -23,6 +23,12 @@ def test_process_help_exposes_framework_flags(capsys):
assert "--llm-api-key" in output
assert "--llm-concurrency" in output
assert "--llm-timeout-seconds" in output
assert "--validation-llm-api-key" in output
assert "--validation-llm-concurrency" in output
assert "--validation-llm-timeout-seconds" in output
assert "--validation-model" in output
assert "--validation-base-url" in output
assert "--validation-max-retries" in output
assert "--target-sections" in output
assert "--modules" in output
assert "--model" in output
@@ -308,6 +314,82 @@ def test_cli_process_passes_llm_timeout_seconds_override_to_config(monkeypatch,
assert captured["llm_timeout_seconds"] == 900.0
def test_cli_process_passes_validation_llm_overrides_to_config(monkeypatch, tmp_path):
captured = {}
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
]
"""
)
report = RunReport(
status="success",
config={"model": "m", "base_url": "b"},
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
pipeline=["grammar"],
modules=[],
applied_changes=[],
skipped_corrections=[],
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
work_dir_retention="auto",
work_dir_retained=False,
work_dir=None,
error=None,
)
result = ProcessResult(
transcript=transcript,
report=report,
run_dir=tmp_path / "run",
work_dir_retained=False,
)
def _fake_from_sources(*, overrides=None):
captured["validation_llm_api_key"] = overrides.validation_llm_api_key
captured["validation_llm_concurrency"] = overrides.validation_llm_concurrency
captured["validation_llm_timeout_seconds"] = overrides.validation_llm_timeout_seconds
captured["validation_model"] = overrides.validation_model
captured["validation_base_url"] = overrides.validation_base_url
captured["validation_max_retries"] = overrides.validation_max_retries
return object()
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--validation-llm-api-key",
"validator-key",
"--validation-llm-concurrency",
"4",
"--validation-llm-timeout-seconds",
"180",
"--validation-model",
"validator-model",
"--validation-base-url",
"http://localhost:9000/v1",
"--validation-max-retries",
"2",
]
)
assert exit_code == 0
assert captured == {
"validation_llm_api_key": "validator-key",
"validation_llm_concurrency": 4,
"validation_llm_timeout_seconds": 180.0,
"validation_model": "validator-model",
"validation_base_url": "http://localhost:9000/v1",
"validation_max_retries": 2,
}
def test_cli_process_passes_target_sections_override_to_config(monkeypatch, tmp_path):
captured = {}
transcript = parse_transcript_json(

143
tests/test_new_config.py
View File

@@ -24,6 +24,12 @@ def test_default_config_allows_missing_api_key():
assert config.api_key is None
assert config.llm_concurrency == DEFAULT_LLM_CONCURRENCY
assert config.llm_timeout_seconds == DEFAULT_LLM_TIMEOUT_SECONDS
assert config.validation_llm_api_key is None
assert config.validation_llm_concurrency is None
assert config.validation_llm_timeout_seconds is None
assert config.validation_model is None
assert config.validation_base_url is None
assert config.validation_max_retries is None
assert config.target_sections is None
assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS
assert config.min_section_tokens == DEFAULT_MIN_SECTION_TOKENS
@@ -84,6 +90,54 @@ def test_llm_timeout_seconds_env_is_parsed():
assert config.llm_timeout_seconds == 120.5
def test_validation_llm_concurrency_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_VALIDATION_LLM_CONCURRENCY": "2"},
overrides=ConfigOverrides(validation_llm_concurrency=4),
)
assert config.validation_llm_concurrency == 4
def test_validation_llm_concurrency_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_CONCURRENCY": "3"})
assert config.validation_llm_concurrency == 3
def test_validation_llm_timeout_seconds_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "120"},
overrides=ConfigOverrides(validation_llm_timeout_seconds=900.0),
)
assert config.validation_llm_timeout_seconds == 900.0
def test_validation_llm_timeout_seconds_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "120.5"})
assert config.validation_llm_timeout_seconds == 120.5
def test_validation_model_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MODEL": "validator-model"})
assert config.validation_model == "validator-model"
def test_validation_base_url_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_BASE_URL": "http://localhost:9000/v1"})
assert config.validation_base_url == "http://localhost:9000/v1"
def test_validation_max_retries_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_RETRIES": "7"})
assert config.validation_max_retries == 7
def test_target_sections_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_TARGET_SECTIONS": "2"},
@@ -143,6 +197,62 @@ def test_blank_llm_api_key_override_resolves_to_none():
assert config.api_key is None
def test_validation_llm_api_key_env_is_read():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_API_KEY": "validation-key"})
assert config.validation_llm_api_key == "validation-key"
def test_effective_validation_fields_fall_back_to_primary_settings():
config = AuditaConfig.from_sources(
env={
"AUDITA_LLM_API_KEY": "primary-key",
"AUDITA_MODEL": "primary-model",
"AUDITA_BASE_URL": "http://localhost:8000/v1",
"AUDITA_LLM_TIMEOUT_SECONDS": "120",
"AUDITA_LLM_CONCURRENCY": "5",
"AUDITA_MAX_RETRIES": "9",
}
)
validation = config.validation_llm_config()
assert validation.api_key == "primary-key"
assert validation.model == "primary-model"
assert validation.base_url == "http://localhost:8000/v1"
assert validation.llm_timeout_seconds == 120.0
assert validation.llm_concurrency == 5
assert validation.max_retries == 9
def test_effective_validation_fields_use_overrides_when_set():
config = AuditaConfig.from_sources(
env={
"AUDITA_LLM_API_KEY": "primary-key",
"AUDITA_MODEL": "primary-model",
"AUDITA_BASE_URL": "http://localhost:8000/v1",
"AUDITA_LLM_TIMEOUT_SECONDS": "120",
"AUDITA_LLM_CONCURRENCY": "5",
"AUDITA_MAX_RETRIES": "9",
"AUDITA_VALIDATION_LLM_API_KEY": "validation-key",
"AUDITA_VALIDATION_MODEL": "validation-model",
"AUDITA_VALIDATION_BASE_URL": "http://localhost:9000/v1",
"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "240",
"AUDITA_VALIDATION_LLM_CONCURRENCY": "3",
"AUDITA_VALIDATION_MAX_RETRIES": "2",
}
)
validation = config.validation_llm_config()
assert validation.api_key == "validation-key"
assert validation.model == "validation-model"
assert validation.base_url == "http://localhost:9000/v1"
assert validation.llm_timeout_seconds == 240.0
assert validation.llm_concurrency == 3
assert validation.max_retries == 2
def test_module_key_overrides_take_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_MODULES": "grammar"},
@@ -175,6 +285,21 @@ def test_report_dict_includes_target_sections():
assert config.to_report_dict()["target_sections"] == 7
def test_report_dict_includes_effective_validation_llm_config():
config = AuditaConfig.from_sources(
env={
"AUDITA_LLM_API_KEY": "primary-key",
"AUDITA_VALIDATION_MODEL": "validation-model",
}
)
report = config.to_report_dict()
assert report["validation_model"] == "validation-model"
assert report["effective_validation_llm"]["api_key_configured"] is True
assert report["effective_validation_llm"]["model"] == "validation-model"
def test_threshold_overrides_take_precedence():
config = AuditaConfig.from_sources(
env={
@@ -236,6 +361,24 @@ def test_invalid_llm_timeout_seconds_is_rejected(value):
AuditaConfig.from_sources(env={"AUDITA_LLM_TIMEOUT_SECONDS": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_validation_llm_concurrency_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_LLM_CONCURRENCY"):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_CONCURRENCY": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_validation_llm_timeout_seconds_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": value})
@pytest.mark.parametrize("value", ["-1", "many"])
def test_invalid_validation_max_retries_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_MAX_RETRIES"):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_RETRIES": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_target_sections_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_TARGET_SECTIONS"):

80
tests/test_new_pipeline.py
View File

@@ -24,6 +24,7 @@ class FakeStructuredLLMClient:
"stage_name": stage_name,
"messages": list(messages),
"response_model": response_model,
"config": config,
}
)
response = _pop_llm_response(self._responses, stage_name)
@@ -103,6 +104,85 @@ def test_process_transcript_runs_noop_framework(tmp_path):
]
def test_process_transcript_result_can_use_different_validation_llm_settings(tmp_path):
llm_client = FakeStructuredLLMClient(
{
"grammar:proposal": {
"corrections": [
{
"id": 1,
"original_text": "hello world",
"corrected_text": "Hello world.",
"confidence": 0.95,
}
]
},
"grammar:grammar_only_guard": {
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.98,
"reason": "ok",
}
]
},
"grammar:meaning_reversal_review": {
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.99,
"reason": "ok",
}
]
},
}
)
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "primary-key"},
overrides=ConfigOverrides(
model="primary-model",
base_url="http://localhost:8000/v1",
max_retries=7,
llm_timeout_seconds=120,
validation_llm_api_key="validation-key",
validation_model="validation-model",
validation_base_url="http://localhost:9000/v1",
validation_max_retries=2,
validation_llm_timeout_seconds=240,
validation_llm_concurrency=3,
work_dir=tmp_path / "work",
work_dir_retention="always",
),
)
result = process_transcript_result(
parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
]
"""
),
_glossary(),
config,
module_keys=["grammar"],
llm_client=llm_client,
)
assert result.transcript[0].text == "Hello world."
calls_by_stage = {call["stage_name"]: call["config"] for call in llm_client.calls}
assert calls_by_stage["grammar:proposal"].model == "primary-model"
assert calls_by_stage["grammar:proposal"].base_url == "http://localhost:8000/v1"
assert calls_by_stage["grammar:proposal"].api_key == "primary-key"
assert calls_by_stage["grammar:grammar_only_guard"].model == "validation-model"
assert calls_by_stage["grammar:grammar_only_guard"].base_url == "http://localhost:9000/v1"
assert calls_by_stage["grammar:grammar_only_guard"].api_key == "validation-key"
assert calls_by_stage["grammar:grammar_only_guard"].max_retries == 2
assert calls_by_stage["grammar:grammar_only_guard"].llm_timeout_seconds == 240
def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(tmp_path):
config = AuditaConfig.from_sources(
env={},