Implemented optional concurrency for the LLM backend

This commit is contained in:
2026-04-28 22:14:25 -05:00
parent f834ffad97
commit bbbef37d9a
10 changed files with 376 additions and 33 deletions

View File

@@ -1,3 +1,4 @@
from concurrent.futures import ThreadPoolExecutor
import sys
import types
@@ -16,6 +17,7 @@ def _config(**overrides):
base = AuditaConfig.from_sources(env={})
data = {
"api_key": "test-key",
"llm_concurrency": base.llm_concurrency,
"module_keys": base.module_keys,
"model": base.model,
"base_url": base.base_url,
@@ -172,3 +174,24 @@ def test_missing_api_key_error_is_provider_neutral():
response_model=DummyResponseModel,
config=_config(api_key=None),
)
def test_client_initialization_is_safe_under_concurrent_calls(monkeypatch):
_, openai_inits = _install_fake_llm_modules(monkeypatch)
client = OpenAICompatibleStructuredLLMClient()
config = _config(api_key="key-1", base_url="http://localhost:8000/v1")
with ThreadPoolExecutor(max_workers=4) as executor:
list(
executor.map(
lambda _: client.run_structured(
stage_name="test-stage",
messages=[{"role": "user", "content": "Hello"}],
response_model=DummyResponseModel,
config=config,
),
range(4),
)
)
assert openai_inits == [{"api_key": "key-1", "base_url": "http://localhost:8000/v1"}]