eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Added a configuration flag to set the maximum number of tokens in validation prompts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-29 22:33:45 -05:00
parent 54ce3d08c9
commit 0d5bdd1c84
7 changed files with 121 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
tests/test_llm_validators.py
View File

@@ -1092,7 +1092,10 @@ def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path):
assert [decision.approved for decision in result.decisions] == [True, True, True]
assert len(client.calls) == 2
assert len(chunk_calls) == 1
assert chunk_calls[0]["max_tokens"] == AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}).max_section_tokens
assert (
chunk_calls[0]["max_tokens"]
== AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}).validation_max_prompt_tokens
)
assert all("corrected_segment_text" in payload for payload in chunk_calls[0]["payloads"])
@@ -1272,6 +1275,74 @@ def test_validator_uses_validation_llm_config(tmp_path):
assert validation_config.llm_concurrency == 3
def test_validator_batches_use_validation_max_prompt_tokens(tmp_path, monkeypatch):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
)
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.97,
"reason": "ok",
}
]
}
]
)
chunk_calls = []
def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message):
chunk_calls.append(
{
"count": len(items),
"max_tokens": max_tokens,
}
)
return [TokenBatch(batch_index=0, items=list(items), token_count=1)]
monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items)
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "test-key"},
overrides=ConfigOverrides(validation_max_prompt_tokens=1024),
)
context = ValidationContext(
proposals=proposals,
transcript=transcript,
glossary=_glossary(),
config=config.validation_llm_config(),
run_spec=ModuleRunSpec(
instance_name="homophones",
module_key="homophones",
module=_Module("require_unique"),
),
run_dir=tmp_path,
llm_client=client,
)
SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(context)
assert chunk_calls[0]["max_tokens"] == 1024
def test_validator_uses_validation_llm_concurrency_override(tmp_path, monkeypatch):
transcript = parse_transcript_json(
"""

5
tests/test_new_cli.py
View File

@@ -29,6 +29,7 @@ def test_process_help_exposes_framework_flags(capsys):
assert "--validation-model" in output
assert "--validation-base-url" in output
assert "--validation-max-retries" in output
assert "--validation-max-prompt-tokens" in output
assert "--target-sections" in output
assert "--modules" in output
assert "--model" in output
@@ -351,6 +352,7 @@ def test_cli_process_passes_validation_llm_overrides_to_config(monkeypatch, tmp_
captured["validation_model"] = overrides.validation_model
captured["validation_base_url"] = overrides.validation_base_url
captured["validation_max_retries"] = overrides.validation_max_retries
captured["validation_max_prompt_tokens"] = overrides.validation_max_prompt_tokens
return object()
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
@@ -376,6 +378,8 @@ def test_cli_process_passes_validation_llm_overrides_to_config(monkeypatch, tmp_
"http://localhost:9000/v1",
"--validation-max-retries",
"2",
"--validation-max-prompt-tokens",
"1024",
]
)
@@ -387,6 +391,7 @@ def test_cli_process_passes_validation_llm_overrides_to_config(monkeypatch, tmp_
"validation_model": "validator-model",
"validation_base_url": "http://localhost:9000/v1",
"validation_max_retries": 2,
"validation_max_prompt_tokens": 1024,
}

24
tests/test_new_config.py
View File

@@ -12,6 +12,7 @@ from audita.core.config import (
DEFAULT_MIN_SECTION_TOKENS,
DEFAULT_NORMALIZE_MAX_SEGMENT_GAP,
DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD,
DEFAULT_VALIDATION_MAX_PROMPT_TOKENS,
DEFAULT_WORK_DIR_RETENTION,
)
from audita.core.errors import AuditaConfigError
@@ -30,6 +31,7 @@ def test_default_config_allows_missing_api_key():
assert config.validation_model is None
assert config.validation_base_url is None
assert config.validation_max_retries is None
assert config.validation_max_prompt_tokens == DEFAULT_VALIDATION_MAX_PROMPT_TOKENS
assert config.target_sections is None
assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS
assert config.min_section_tokens == DEFAULT_MIN_SECTION_TOKENS
@@ -138,6 +140,21 @@ def test_validation_max_retries_env_is_parsed():
assert config.validation_max_retries == 7
def test_validation_max_prompt_tokens_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_VALIDATION_MAX_PROMPT_TOKENS": "1024"},
overrides=ConfigOverrides(validation_max_prompt_tokens=4096),
)
assert config.validation_max_prompt_tokens == 4096
def test_validation_max_prompt_tokens_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_PROMPT_TOKENS": "1024"})
assert config.validation_max_prompt_tokens == 1024
def test_target_sections_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_TARGET_SECTIONS": "2"},
@@ -296,6 +313,7 @@ def test_report_dict_includes_effective_validation_llm_config():
report = config.to_report_dict()
assert report["validation_model"] == "validation-model"
assert report["validation_max_prompt_tokens"] == DEFAULT_VALIDATION_MAX_PROMPT_TOKENS
assert report["effective_validation_llm"]["api_key_configured"] is True
assert report["effective_validation_llm"]["model"] == "validation-model"
@@ -379,6 +397,12 @@ def test_invalid_validation_max_retries_is_rejected(value):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_RETRIES": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_validation_max_prompt_tokens_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_MAX_PROMPT_TOKENS"):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_PROMPT_TOKENS": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_target_sections_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_TARGET_SECTIONS"):