Added a configurable LLM timeout

This commit is contained in:
2026-04-29 18:08:39 -05:00
parent d8bc84934e
commit f841f7eb71
7 changed files with 143 additions and 8 deletions

View File

@@ -18,6 +18,7 @@ def _config(**overrides):
data = {
"api_key": "test-key",
"llm_concurrency": base.llm_concurrency,
"llm_timeout_seconds": base.llm_timeout_seconds,
"module_keys": base.module_keys,
"model": base.model,
"base_url": base.base_url,
@@ -51,8 +52,8 @@ def _install_fake_llm_modules(monkeypatch):
return {"ok": True}
class FakeOpenAI:
def __init__(self, *, api_key, base_url):
openai_inits.append({"api_key": api_key, "base_url": base_url})
def __init__(self, *, api_key, base_url, timeout):
openai_inits.append({"api_key": api_key, "base_url": base_url, "timeout": timeout})
fake_instructor = types.SimpleNamespace(
Mode=types.SimpleNamespace(TOOLS="TOOLS"),
@@ -159,8 +160,33 @@ def test_client_cache_identity_uses_api_key_and_base_url(monkeypatch):
)
assert openai_inits == [
{"api_key": "key-1", "base_url": "http://localhost:8000/v1"},
{"api_key": "key-1", "base_url": "https://api.openai.com/v1"},
{"api_key": "key-1", "base_url": "http://localhost:8000/v1", "timeout": 600},
{"api_key": "key-1", "base_url": "https://api.openai.com/v1", "timeout": 600},
]
def test_client_cache_identity_uses_timeout(monkeypatch):
_, openai_inits = _install_fake_llm_modules(monkeypatch)
client = OpenAICompatibleStructuredLLMClient()
first = _config(api_key="key-1", base_url="http://localhost:8000/v1", llm_timeout_seconds=600)
second = _config(api_key="key-1", base_url="http://localhost:8000/v1", llm_timeout_seconds=1200)
client.run_structured(
stage_name="one",
messages=[{"role": "user", "content": "Hello"}],
response_model=DummyResponseModel,
config=first,
)
client.run_structured(
stage_name="two",
messages=[{"role": "user", "content": "Hello"}],
response_model=DummyResponseModel,
config=second,
)
assert openai_inits == [
{"api_key": "key-1", "base_url": "http://localhost:8000/v1", "timeout": 600},
{"api_key": "key-1", "base_url": "http://localhost:8000/v1", "timeout": 1200},
]
@@ -194,4 +220,4 @@ def test_client_initialization_is_safe_under_concurrent_calls(monkeypatch):
)
)
assert openai_inits == [{"api_key": "key-1", "base_url": "http://localhost:8000/v1"}]
assert openai_inits == [{"api_key": "key-1", "base_url": "http://localhost:8000/v1", "timeout": 600}]