Added a configuration flag to set the maximum number of tokens in validation prompts
This commit is contained in:
@@ -1092,7 +1092,10 @@ def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path):
|
||||
assert [decision.approved for decision in result.decisions] == [True, True, True]
|
||||
assert len(client.calls) == 2
|
||||
assert len(chunk_calls) == 1
|
||||
assert chunk_calls[0]["max_tokens"] == AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}).max_section_tokens
|
||||
assert (
|
||||
chunk_calls[0]["max_tokens"]
|
||||
== AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}).validation_max_prompt_tokens
|
||||
)
|
||||
assert all("corrected_segment_text" in payload for payload in chunk_calls[0]["payloads"])
|
||||
|
||||
|
||||
@@ -1272,6 +1275,74 @@ def test_validator_uses_validation_llm_config(tmp_path):
|
||||
assert validation_config.llm_concurrency == 3
|
||||
|
||||
|
||||
def test_validator_batches_use_validation_max_prompt_tokens(tmp_path, monkeypatch):
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
[
|
||||
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
|
||||
]
|
||||
"""
|
||||
)
|
||||
proposals = [
|
||||
CorrectionProposal(
|
||||
proposal_index=0,
|
||||
module_instance="homophones",
|
||||
module_key="homophones",
|
||||
id=1,
|
||||
original_text="gestures",
|
||||
corrected_text="Jesters",
|
||||
confidence=0.95,
|
||||
)
|
||||
]
|
||||
client = FakeStructuredLLMClient(
|
||||
[
|
||||
{
|
||||
"validations": [
|
||||
{
|
||||
"correction_index": 0,
|
||||
"approved": True,
|
||||
"confidence": 0.97,
|
||||
"reason": "ok",
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
)
|
||||
chunk_calls = []
|
||||
|
||||
def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message):
|
||||
chunk_calls.append(
|
||||
{
|
||||
"count": len(items),
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
)
|
||||
return [TokenBatch(batch_index=0, items=list(items), token_count=1)]
|
||||
|
||||
monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items)
|
||||
config = AuditaConfig.from_sources(
|
||||
env={"OPENROUTER_API_KEY": "test-key"},
|
||||
overrides=ConfigOverrides(validation_max_prompt_tokens=1024),
|
||||
)
|
||||
context = ValidationContext(
|
||||
proposals=proposals,
|
||||
transcript=transcript,
|
||||
glossary=_glossary(),
|
||||
config=config.validation_llm_config(),
|
||||
run_spec=ModuleRunSpec(
|
||||
instance_name="homophones",
|
||||
module_key="homophones",
|
||||
module=_Module("require_unique"),
|
||||
),
|
||||
run_dir=tmp_path,
|
||||
llm_client=client,
|
||||
)
|
||||
|
||||
SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(context)
|
||||
|
||||
assert chunk_calls[0]["max_tokens"] == 1024
|
||||
|
||||
|
||||
def test_validator_uses_validation_llm_concurrency_override(tmp_path, monkeypatch):
|
||||
transcript = parse_transcript_json(
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user