Added a configuration flag to set the maximum number of tokens in validation prompts

This commit is contained in:
2026-04-29 22:33:45 -05:00
parent 54ce3d08c9
commit 0d5bdd1c84
7 changed files with 121 additions and 2 deletions

View File

@@ -1092,7 +1092,10 @@ def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path):
assert [decision.approved for decision in result.decisions] == [True, True, True]
assert len(client.calls) == 2
assert len(chunk_calls) == 1
assert chunk_calls[0]["max_tokens"] == AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}).max_section_tokens
assert (
chunk_calls[0]["max_tokens"]
== AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}).validation_max_prompt_tokens
)
assert all("corrected_segment_text" in payload for payload in chunk_calls[0]["payloads"])
@@ -1272,6 +1275,74 @@ def test_validator_uses_validation_llm_config(tmp_path):
assert validation_config.llm_concurrency == 3
def test_validator_batches_use_validation_max_prompt_tokens(tmp_path, monkeypatch):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
)
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.97,
"reason": "ok",
}
]
}
]
)
chunk_calls = []
def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message):
chunk_calls.append(
{
"count": len(items),
"max_tokens": max_tokens,
}
)
return [TokenBatch(batch_index=0, items=list(items), token_count=1)]
monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items)
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "test-key"},
overrides=ConfigOverrides(validation_max_prompt_tokens=1024),
)
context = ValidationContext(
proposals=proposals,
transcript=transcript,
glossary=_glossary(),
config=config.validation_llm_config(),
run_spec=ModuleRunSpec(
instance_name="homophones",
module_key="homophones",
module=_Module("require_unique"),
),
run_dir=tmp_path,
llm_client=client,
)
SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(context)
assert chunk_calls[0]["max_tokens"] == 1024
def test_validator_uses_validation_llm_concurrency_override(tmp_path, monkeypatch):
transcript = parse_transcript_json(
"""