Implemented optional concurrency for the LLM backend
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
import types
|
||||
|
||||
@@ -16,6 +17,7 @@ def _config(**overrides):
|
||||
base = AuditaConfig.from_sources(env={})
|
||||
data = {
|
||||
"api_key": "test-key",
|
||||
"llm_concurrency": base.llm_concurrency,
|
||||
"module_keys": base.module_keys,
|
||||
"model": base.model,
|
||||
"base_url": base.base_url,
|
||||
@@ -172,3 +174,24 @@ def test_missing_api_key_error_is_provider_neutral():
|
||||
response_model=DummyResponseModel,
|
||||
config=_config(api_key=None),
|
||||
)
|
||||
|
||||
|
||||
def test_client_initialization_is_safe_under_concurrent_calls(monkeypatch):
|
||||
_, openai_inits = _install_fake_llm_modules(monkeypatch)
|
||||
client = OpenAICompatibleStructuredLLMClient()
|
||||
config = _config(api_key="key-1", base_url="http://localhost:8000/v1")
|
||||
|
||||
with ThreadPoolExecutor(max_workers=4) as executor:
|
||||
list(
|
||||
executor.map(
|
||||
lambda _: client.run_structured(
|
||||
stage_name="test-stage",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
response_model=DummyResponseModel,
|
||||
config=config,
|
||||
),
|
||||
range(4),
|
||||
)
|
||||
)
|
||||
|
||||
assert openai_inits == [{"api_key": "key-1", "base_url": "http://localhost:8000/v1"}]
|
||||
|
||||
Reference in New Issue
Block a user