From fac162941704bd703dfaae6f26f7976a4bb59aba Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 4 May 2026 11:11:06 -0500 Subject: [PATCH] An API key is no longer required if base_url is pointed to a non-default endpoint --- README.md | 9 ++-- src/audita/core/config.py | 10 +++- src/audita/framework/llm.py | 8 ++- tests/test_framework_llm.py | 28 ++++++++++- tests/test_new_config.py | 10 ++++ tests/test_new_pipeline.py | 97 ++++++++++++++++++++++++++++++++++++- 6 files changed, 153 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index e13d571..19220a5 100644 --- a/README.md +++ b/README.md @@ -79,13 +79,13 @@ audita process transcript.json --glossary glossary.yaml --output corrected.json Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr. `--report-json` writes a separate machine-readable run report and never mixes report data into stdout. -Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Normal runs now require LLM API credentials, because the `glossary`, `homophones`, `spoken_word`, and `grammar` modules make real LLM calls. `AUDITA_LLM_API_KEY` and `--llm-api-key` are the preferred provider-neutral credential surfaces, while `OPENROUTER_API_KEY` remains supported as a backward-compatible fallback. +Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Default OpenRouter runs require LLM API credentials, because the `glossary`, `homophones`, `spoken_word`, and `grammar` modules make real LLM calls. Self-hosted or other non-default OpenAI-compatible endpoints may not require credentials. `AUDITA_LLM_API_KEY` and `--llm-api-key` are the preferred provider-neutral credential surfaces, while `OPENROUTER_API_KEY` remains supported as a backward-compatible fallback. | Environment variable | CLI flag | Default | Purpose | | --- | --- | --- | --- | | `AUDITA_MODULES` | `--modules` | `glossary,homophones,glossary,spoken_word,grammar` | Comma-separated logical module keys to run; CLI overrides the environment value | -| `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; CLI overrides both environment-key variants | -| `AUDITA_VALIDATION_LLM_API_KEY` | `--validation-llm-api-key` | unset | Validation-phase LLM API credential; defaults to the primary LLM API key | +| `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; required for the default OpenRouter endpoint and optional for non-default endpoints; CLI overrides both environment-key variants | +| `AUDITA_VALIDATION_LLM_API_KEY` | `--validation-llm-api-key` | unset | Validation-phase LLM API credential; defaults to the primary LLM API key and is optional for non-default validation endpoints | | `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model name sent to the configured OpenAI-compatible endpoint | | `AUDITA_VALIDATION_MODEL` | `--validation-model` | unset | Validation-phase LLM model; defaults to `AUDITA_MODEL` | | `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL | @@ -124,12 +124,13 @@ audita process transcript.json --glossary glossary.yaml --output corrected.json You can point Audita at any OpenAI-compatible endpoint by changing `AUDITA_BASE_URL` and, if needed, `AUDITA_MODEL`. For example, a local vLLM server: ```sh -export AUDITA_LLM_API_KEY=local-dev-key export AUDITA_BASE_URL=http://localhost:8000/v1 export AUDITA_MODEL=meta-llama/Llama-3.1-8B-Instruct audita process transcript.json --glossary glossary.yaml --output corrected.json ``` +If your self-hosted endpoint requires authentication, you can still set `AUDITA_LLM_API_KEY`; Audita simply no longer requires it for non-default endpoints. + Or the actual OpenAI API: ```sh diff --git a/src/audita/core/config.py b/src/audita/core/config.py index 94f6c58..da492cb 100644 --- a/src/audita/core/config.py +++ b/src/audita/core/config.py @@ -116,7 +116,7 @@ class AuditaConfig: DEFAULT_LLM_TIMEOUT_SECONDS, "AUDITA_LLM_TIMEOUT_SECONDS", ), - validation_llm_api_key=_select_optional_string_override( + validation_llm_api_key=_select_optional_api_key_override( selected.validation_llm_api_key, source.get("AUDITA_VALIDATION_LLM_API_KEY"), ), @@ -407,6 +407,14 @@ def _select_optional_string_override(cli_value: Optional[str], env_value: Option return _select_optional_string(env_value) +def _select_optional_api_key_override(cli_value: Optional[str], env_value: Optional[str]) -> Optional[str]: + if cli_value is not None: + return cli_value.strip() + if env_value is not None: + return env_value.strip() + return None + + def _select_api_key( cli_value: Optional[str], generic_env_value: Optional[str], diff --git a/src/audita/framework/llm.py b/src/audita/framework/llm.py index 39632ca..0350915 100644 --- a/src/audita/framework/llm.py +++ b/src/audita/framework/llm.py @@ -21,9 +21,9 @@ class OpenAICompatibleStructuredLLMClient: response_model: Any, config: AuditaConfig, ) -> Any: - if not config.api_key: + if _requires_api_key(config) and not config.api_key: raise AuditaLLMError( - f"Structured LLM stage '{stage_name}' requires LLM API credentials to be configured " + f"Structured LLM stage '{stage_name}' requires LLM API credentials for the configured OpenRouter endpoint " "via --llm-api-key, --validation-llm-api-key, AUDITA_LLM_API_KEY, " "AUDITA_VALIDATION_LLM_API_KEY, or OPENROUTER_API_KEY." ) @@ -88,5 +88,9 @@ def _uses_openrouter_shape(config: AuditaConfig) -> bool: ) +def _requires_api_key(config: AuditaConfig) -> bool: + return _normalized_base_url(config.base_url) == _normalized_base_url(DEFAULT_BASE_URL) + + def _normalized_base_url(base_url: str) -> str: return base_url.rstrip("/") diff --git a/tests/test_framework_llm.py b/tests/test_framework_llm.py index 870daf2..0f34279 100644 --- a/tests/test_framework_llm.py +++ b/tests/test_framework_llm.py @@ -199,7 +199,7 @@ def test_client_cache_identity_uses_timeout(monkeypatch): def test_missing_api_key_error_is_provider_neutral(): client = OpenAICompatibleStructuredLLMClient() - with pytest.raises(AuditaLLMError, match="AUDITA_VALIDATION_LLM_API_KEY"): + with pytest.raises(AuditaLLMError, match="OpenRouter endpoint"): client.run_structured( stage_name="test-stage", messages=[{"role": "user", "content": "Hello"}], @@ -208,6 +208,32 @@ def test_missing_api_key_error_is_provider_neutral(): ) +def test_missing_api_key_is_allowed_for_nondefault_endpoint(monkeypatch): + create_calls, openai_inits = _install_fake_llm_modules(monkeypatch) + client = OpenAICompatibleStructuredLLMClient() + + client.run_structured( + stage_name="test-stage", + messages=[{"role": "user", "content": "Hello"}], + response_model=DummyResponseModel, + config=_config( + api_key=None, + model="meta-llama/Llama-3.1-8B-Instruct", + base_url="http://localhost:8000/v1", + ), + ) + + assert openai_inits == [{"api_key": None, "base_url": "http://localhost:8000/v1", "timeout": 600}] + assert create_calls == [ + { + "model": "meta-llama/Llama-3.1-8B-Instruct", + "messages": [{"role": "user", "content": "Hello"}], + "response_model": DummyResponseModel, + "max_retries": 3, + } + ] + + def test_client_initialization_is_safe_under_concurrent_calls(monkeypatch): _, openai_inits = _install_fake_llm_modules(monkeypatch) client = OpenAICompatibleStructuredLLMClient() diff --git a/tests/test_new_config.py b/tests/test_new_config.py index 5c4829e..4bdceee 100644 --- a/tests/test_new_config.py +++ b/tests/test_new_config.py @@ -220,6 +220,16 @@ def test_validation_llm_api_key_env_is_read(): assert config.validation_llm_api_key == "validation-key" +def test_blank_validation_llm_api_key_override_disables_primary_fallback(): + config = AuditaConfig.from_sources( + env={"AUDITA_LLM_API_KEY": "primary-key"}, + overrides=ConfigOverrides(validation_llm_api_key=" "), + ) + + assert config.validation_llm_api_key == "" + assert config.validation_llm_config().api_key == "" + + def test_effective_validation_fields_fall_back_to_primary_settings(): config = AuditaConfig.from_sources( env={ diff --git a/tests/test_new_pipeline.py b/tests/test_new_pipeline.py index c56ac9a..11a0fd8 100644 --- a/tests/test_new_pipeline.py +++ b/tests/test_new_pipeline.py @@ -391,7 +391,7 @@ def test_process_transcript_result_missing_api_key_writes_failed_report(tmp_path work_dir_retention="never", ) - with pytest.raises(AuditaLLMError, match="AUDITA_LLM_API_KEY"): + with pytest.raises(AuditaLLMError, match="OpenRouter endpoint"): process_transcript_result(_transcript(), _glossary(), config) run_dir = next((tmp_path / "work").iterdir()) @@ -413,6 +413,101 @@ def test_process_transcript_result_missing_api_key_writes_failed_report(tmp_path assert report["work_dir"] == str(run_dir) assert "AUDITA_LLM_API_KEY" in report["error"] assert "OPENROUTER_API_KEY" in report["error"] + assert "OpenRouter endpoint" in report["error"] + + +def test_process_transcript_result_allows_missing_api_key_for_nondefault_proposal_endpoint(tmp_path): + llm_client = FakeStructuredLLMClient( + [ + {"corrections": []}, + {"corrections": []}, + {"corrections": []}, + {"corrections": []}, + {"corrections": []}, + ] + ) + config = AuditaConfig.from_sources( + env={}, + overrides=ConfigOverrides( + base_url="http://localhost:8000/v1", + model="meta-llama/Llama-3.1-8B-Instruct", + work_dir=tmp_path / "work", + work_dir_retention="always", + ), + ) + + result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client) + + assert result.report.status == "success" + assert llm_client.calls[0]["config"].api_key is None + assert llm_client.calls[0]["config"].base_url == "http://localhost:8000/v1" + + +def test_process_transcript_result_allows_missing_validation_api_key_for_nondefault_validation_endpoint(tmp_path): + llm_client = FakeStructuredLLMClient( + { + "grammar:proposal": { + "corrections": [ + { + "id": 1, + "original_text": "hello world", + "corrected_text": "Hello world.", + "confidence": 0.95, + } + ] + }, + "grammar:grammar_only_guard": { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.98, + "reason": "ok", + } + ] + }, + "grammar:meaning_reversal_review": { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.99, + "reason": "ok", + } + ] + }, + } + ) + config = AuditaConfig.from_sources( + env={"OPENROUTER_API_KEY": "primary-key"}, + overrides=ConfigOverrides( + work_dir=tmp_path / "work", + work_dir_retention="always", + validation_llm_api_key=" ", + validation_base_url="http://localhost:9000/v1", + validation_model="meta-llama/Llama-3.1-8B-Instruct", + ), + ) + + result = process_transcript_result( + parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} + ] + """ + ), + _glossary(), + config, + module_keys=["grammar"], + llm_client=llm_client, + ) + + assert result.transcript[0].text == "Hello world." + calls_by_stage = {call["stage_name"]: call["config"] for call in llm_client.calls} + assert calls_by_stage["grammar:proposal"].api_key == "primary-key" + assert calls_by_stage["grammar:grammar_only_guard"].api_key == "" + assert calls_by_stage["grammar:grammar_only_guard"].base_url == "http://localhost:9000/v1" def test_process_transcript_result_preserves_partial_progress_when_later_module_fails(tmp_path):