Added a configurable LLM timeout
This commit is contained in:
@@ -18,6 +18,7 @@ def _config(**overrides):
|
||||
data = {
|
||||
"api_key": "test-key",
|
||||
"llm_concurrency": base.llm_concurrency,
|
||||
"llm_timeout_seconds": base.llm_timeout_seconds,
|
||||
"module_keys": base.module_keys,
|
||||
"model": base.model,
|
||||
"base_url": base.base_url,
|
||||
@@ -51,8 +52,8 @@ def _install_fake_llm_modules(monkeypatch):
|
||||
return {"ok": True}
|
||||
|
||||
class FakeOpenAI:
|
||||
def __init__(self, *, api_key, base_url):
|
||||
openai_inits.append({"api_key": api_key, "base_url": base_url})
|
||||
def __init__(self, *, api_key, base_url, timeout):
|
||||
openai_inits.append({"api_key": api_key, "base_url": base_url, "timeout": timeout})
|
||||
|
||||
fake_instructor = types.SimpleNamespace(
|
||||
Mode=types.SimpleNamespace(TOOLS="TOOLS"),
|
||||
@@ -159,8 +160,33 @@ def test_client_cache_identity_uses_api_key_and_base_url(monkeypatch):
|
||||
)
|
||||
|
||||
assert openai_inits == [
|
||||
{"api_key": "key-1", "base_url": "http://localhost:8000/v1"},
|
||||
{"api_key": "key-1", "base_url": "https://api.openai.com/v1"},
|
||||
{"api_key": "key-1", "base_url": "http://localhost:8000/v1", "timeout": 600},
|
||||
{"api_key": "key-1", "base_url": "https://api.openai.com/v1", "timeout": 600},
|
||||
]
|
||||
|
||||
|
||||
def test_client_cache_identity_uses_timeout(monkeypatch):
|
||||
_, openai_inits = _install_fake_llm_modules(monkeypatch)
|
||||
client = OpenAICompatibleStructuredLLMClient()
|
||||
first = _config(api_key="key-1", base_url="http://localhost:8000/v1", llm_timeout_seconds=600)
|
||||
second = _config(api_key="key-1", base_url="http://localhost:8000/v1", llm_timeout_seconds=1200)
|
||||
|
||||
client.run_structured(
|
||||
stage_name="one",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
response_model=DummyResponseModel,
|
||||
config=first,
|
||||
)
|
||||
client.run_structured(
|
||||
stage_name="two",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
response_model=DummyResponseModel,
|
||||
config=second,
|
||||
)
|
||||
|
||||
assert openai_inits == [
|
||||
{"api_key": "key-1", "base_url": "http://localhost:8000/v1", "timeout": 600},
|
||||
{"api_key": "key-1", "base_url": "http://localhost:8000/v1", "timeout": 1200},
|
||||
]
|
||||
|
||||
|
||||
@@ -194,4 +220,4 @@ def test_client_initialization_is_safe_under_concurrent_calls(monkeypatch):
|
||||
)
|
||||
)
|
||||
|
||||
assert openai_inits == [{"api_key": "key-1", "base_url": "http://localhost:8000/v1"}]
|
||||
assert openai_inits == [{"api_key": "key-1", "base_url": "http://localhost:8000/v1", "timeout": 600}]
|
||||
|
||||
Reference in New Issue
Block a user