Added validation-specific LLM configuration options

This commit is contained in:
2026-04-29 20:45:51 -05:00
parent 4c899122d9
commit 54ce3d08c9
13 changed files with 714 additions and 12 deletions

View File

@@ -24,6 +24,12 @@ def test_default_config_allows_missing_api_key():
assert config.api_key is None
assert config.llm_concurrency == DEFAULT_LLM_CONCURRENCY
assert config.llm_timeout_seconds == DEFAULT_LLM_TIMEOUT_SECONDS
assert config.validation_llm_api_key is None
assert config.validation_llm_concurrency is None
assert config.validation_llm_timeout_seconds is None
assert config.validation_model is None
assert config.validation_base_url is None
assert config.validation_max_retries is None
assert config.target_sections is None
assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS
assert config.min_section_tokens == DEFAULT_MIN_SECTION_TOKENS
@@ -84,6 +90,54 @@ def test_llm_timeout_seconds_env_is_parsed():
assert config.llm_timeout_seconds == 120.5
def test_validation_llm_concurrency_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_VALIDATION_LLM_CONCURRENCY": "2"},
overrides=ConfigOverrides(validation_llm_concurrency=4),
)
assert config.validation_llm_concurrency == 4
def test_validation_llm_concurrency_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_CONCURRENCY": "3"})
assert config.validation_llm_concurrency == 3
def test_validation_llm_timeout_seconds_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "120"},
overrides=ConfigOverrides(validation_llm_timeout_seconds=900.0),
)
assert config.validation_llm_timeout_seconds == 900.0
def test_validation_llm_timeout_seconds_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "120.5"})
assert config.validation_llm_timeout_seconds == 120.5
def test_validation_model_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MODEL": "validator-model"})
assert config.validation_model == "validator-model"
def test_validation_base_url_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_BASE_URL": "http://localhost:9000/v1"})
assert config.validation_base_url == "http://localhost:9000/v1"
def test_validation_max_retries_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_RETRIES": "7"})
assert config.validation_max_retries == 7
def test_target_sections_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_TARGET_SECTIONS": "2"},
@@ -143,6 +197,62 @@ def test_blank_llm_api_key_override_resolves_to_none():
assert config.api_key is None
def test_validation_llm_api_key_env_is_read():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_API_KEY": "validation-key"})
assert config.validation_llm_api_key == "validation-key"
def test_effective_validation_fields_fall_back_to_primary_settings():
config = AuditaConfig.from_sources(
env={
"AUDITA_LLM_API_KEY": "primary-key",
"AUDITA_MODEL": "primary-model",
"AUDITA_BASE_URL": "http://localhost:8000/v1",
"AUDITA_LLM_TIMEOUT_SECONDS": "120",
"AUDITA_LLM_CONCURRENCY": "5",
"AUDITA_MAX_RETRIES": "9",
}
)
validation = config.validation_llm_config()
assert validation.api_key == "primary-key"
assert validation.model == "primary-model"
assert validation.base_url == "http://localhost:8000/v1"
assert validation.llm_timeout_seconds == 120.0
assert validation.llm_concurrency == 5
assert validation.max_retries == 9
def test_effective_validation_fields_use_overrides_when_set():
config = AuditaConfig.from_sources(
env={
"AUDITA_LLM_API_KEY": "primary-key",
"AUDITA_MODEL": "primary-model",
"AUDITA_BASE_URL": "http://localhost:8000/v1",
"AUDITA_LLM_TIMEOUT_SECONDS": "120",
"AUDITA_LLM_CONCURRENCY": "5",
"AUDITA_MAX_RETRIES": "9",
"AUDITA_VALIDATION_LLM_API_KEY": "validation-key",
"AUDITA_VALIDATION_MODEL": "validation-model",
"AUDITA_VALIDATION_BASE_URL": "http://localhost:9000/v1",
"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "240",
"AUDITA_VALIDATION_LLM_CONCURRENCY": "3",
"AUDITA_VALIDATION_MAX_RETRIES": "2",
}
)
validation = config.validation_llm_config()
assert validation.api_key == "validation-key"
assert validation.model == "validation-model"
assert validation.base_url == "http://localhost:9000/v1"
assert validation.llm_timeout_seconds == 240.0
assert validation.llm_concurrency == 3
assert validation.max_retries == 2
def test_module_key_overrides_take_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_MODULES": "grammar"},
@@ -175,6 +285,21 @@ def test_report_dict_includes_target_sections():
assert config.to_report_dict()["target_sections"] == 7
def test_report_dict_includes_effective_validation_llm_config():
config = AuditaConfig.from_sources(
env={
"AUDITA_LLM_API_KEY": "primary-key",
"AUDITA_VALIDATION_MODEL": "validation-model",
}
)
report = config.to_report_dict()
assert report["validation_model"] == "validation-model"
assert report["effective_validation_llm"]["api_key_configured"] is True
assert report["effective_validation_llm"]["model"] == "validation-model"
def test_threshold_overrides_take_precedence():
config = AuditaConfig.from_sources(
env={
@@ -236,6 +361,24 @@ def test_invalid_llm_timeout_seconds_is_rejected(value):
AuditaConfig.from_sources(env={"AUDITA_LLM_TIMEOUT_SECONDS": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_validation_llm_concurrency_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_LLM_CONCURRENCY"):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_CONCURRENCY": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_validation_llm_timeout_seconds_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": value})
@pytest.mark.parametrize("value", ["-1", "many"])
def test_invalid_validation_max_retries_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_MAX_RETRIES"):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_RETRIES": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_target_sections_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_TARGET_SECTIONS"):