eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Added validation-specific LLM configuration options

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-29 20:45:51 -05:00
parent 4c899122d9
commit 54ce3d08c9
13 changed files with 714 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@@ -85,11 +85,17 @@ Useful configuration can be supplied by CLI flag or environment variable. CLI fl
| --- | --- | --- | --- |
| `AUDITA_MODULES` | `--modules` | `glossary,homophones,glossary,spoken_word,grammar` | Comma-separated logical module keys to run; CLI overrides the environment value |
| `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; CLI overrides both environment-key variants |
| `AUDITA_VALIDATION_LLM_API_KEY` | `--validation-llm-api-key` | unset | Validation-phase LLM API credential; defaults to the primary LLM API key |
| `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model name sent to the configured OpenAI-compatible endpoint |
| `AUDITA_VALIDATION_MODEL` | `--validation-model` | unset | Validation-phase LLM model; defaults to `AUDITA_MODEL` |
| `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL |
| `AUDITA_VALIDATION_BASE_URL` | `--validation-base-url` | unset | Validation-phase OpenAI-compatible API base URL; defaults to `AUDITA_BASE_URL` |
| `AUDITA_LLM_TIMEOUT_SECONDS` | `--llm-timeout-seconds` | `600` | Per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint |
| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | `--validation-llm-timeout-seconds` | unset | Validation-phase per-request timeout in seconds; defaults to `AUDITA_LLM_TIMEOUT_SECONDS` |
| `AUDITA_TARGET_SECTIONS` | `--target-sections` | unset | Exact number of contiguous proposal-stage transcript sections; errors if min/max token bounds cannot be satisfied |
| `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses |
| `AUDITA_VALIDATION_MAX_RETRIES` | `--validation-max-retries` | unset | Validation-phase structured-output retries; defaults to `AUDITA_MAX_RETRIES` |
| `AUDITA_VALIDATION_LLM_CONCURRENCY` | `--validation-llm-concurrency` | unset | Validation-phase LLM concurrency; defaults to `AUDITA_LLM_CONCURRENCY` |
| `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `8192` | Maximum estimated tokens per proposal-stage transcript section |
| `AUDITA_MIN_SECTION_TOKENS` | `--min-section-tokens` | `2048` | Minimum estimated tokens per proposal-stage transcript section when balancing for concurrency |
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation |
@@ -105,6 +111,8 @@ Useful configuration can be supplied by CLI flag or environment variable. CLI fl
Set `AUDITA_MODULES=grammar` to run only the grammar module by default, or override it per command with `--modules`.
Validation-phase LLM settings inherit from the primary `AUDITA_*` LLM settings by default. Set any of the `AUDITA_VALIDATION_*` values only when you want LLM-backed validators to use a different model, endpoint, credential, timeout, retry budget, or concurrency level.
OpenRouter remains the default out of the box:
```sh

30
src/audita/cli.py
View File

@@ -34,6 +34,30 @@ def _build_parser() -> argparse.ArgumentParser:
type=float,
help="per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint",
)
process.add_argument(
"--validation-llm-api-key",
help="LLM API key for validation phases; defaults to the primary configured OpenAI-compatible endpoint key",
)
process.add_argument(
"--validation-llm-concurrency",
type=int,
help="maximum concurrent LLM calls within validation phases; defaults to --llm-concurrency",
)
process.add_argument(
"--validation-llm-timeout-seconds",
type=float,
help="per-request timeout in seconds for validation-phase LLM calls; defaults to --llm-timeout-seconds",
)
process.add_argument("--validation-model", help="LLM model name for validation phases; defaults to --model")
process.add_argument(
"--validation-base-url",
help="OpenAI-compatible API base URL for validation phases; defaults to --base-url",
)
process.add_argument(
"--validation-max-retries",
type=int,
help="maximum structured-output retries for validation phases; defaults to --max-retries",
)
process.add_argument(
"--target-sections",
type=int,
@@ -108,6 +132,12 @@ def _process(args: argparse.Namespace) -> int:
llm_api_key=args.llm_api_key,
llm_concurrency=args.llm_concurrency,
llm_timeout_seconds=args.llm_timeout_seconds,
validation_llm_api_key=args.validation_llm_api_key,
validation_llm_concurrency=args.validation_llm_concurrency,
validation_llm_timeout_seconds=args.validation_llm_timeout_seconds,
validation_model=args.validation_model,
validation_base_url=args.validation_base_url,
validation_max_retries=args.validation_max_retries,
target_sections=args.target_sections,
module_keys=args.modules,
model=args.model,

132
src/audita/core/config.py
View File

@@ -1,6 +1,6 @@
import math
import os
from dataclasses import dataclass
from dataclasses import dataclass, replace
from pathlib import Path
from typing import Mapping, Optional, Sequence, Tuple, Union
@@ -33,6 +33,12 @@ class ConfigOverrides:
llm_api_key: Optional[str] = None
llm_concurrency: Optional[int] = None
llm_timeout_seconds: Optional[float] = None
validation_llm_api_key: Optional[str] = None
validation_llm_concurrency: Optional[int] = None
validation_llm_timeout_seconds: Optional[float] = None
validation_model: Optional[str] = None
validation_base_url: Optional[str] = None
validation_max_retries: Optional[int] = None
target_sections: Optional[int] = None
module_keys: Optional[Union[str, Sequence[str]]] = None
model: Optional[str] = None
@@ -57,6 +63,12 @@ class AuditaConfig:
api_key: Optional[str] = None
llm_concurrency: int = DEFAULT_LLM_CONCURRENCY
llm_timeout_seconds: float = DEFAULT_LLM_TIMEOUT_SECONDS
validation_llm_api_key: Optional[str] = None
validation_llm_concurrency: Optional[int] = None
validation_llm_timeout_seconds: Optional[float] = None
validation_model: Optional[str] = None
validation_base_url: Optional[str] = None
validation_max_retries: Optional[int] = None
target_sections: Optional[int] = None
module_keys: Tuple[str, ...] = DEFAULT_MODULE_KEYS
model: str = DEFAULT_MODEL
@@ -101,6 +113,33 @@ class AuditaConfig:
DEFAULT_LLM_TIMEOUT_SECONDS,
"AUDITA_LLM_TIMEOUT_SECONDS",
),
validation_llm_api_key=_select_optional_string_override(
selected.validation_llm_api_key,
source.get("AUDITA_VALIDATION_LLM_API_KEY"),
),
validation_llm_concurrency=_select_optional_int(
selected.validation_llm_concurrency,
source.get("AUDITA_VALIDATION_LLM_CONCURRENCY"),
"AUDITA_VALIDATION_LLM_CONCURRENCY",
),
validation_llm_timeout_seconds=_select_optional_float(
selected.validation_llm_timeout_seconds,
source.get("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"),
"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS",
),
validation_model=_select_optional_string_override(
selected.validation_model,
source.get("AUDITA_VALIDATION_MODEL"),
),
validation_base_url=_select_optional_string_override(
selected.validation_base_url,
source.get("AUDITA_VALIDATION_BASE_URL"),
),
validation_max_retries=_select_optional_int(
selected.validation_max_retries,
source.get("AUDITA_VALIDATION_MAX_RETRIES"),
"AUDITA_VALIDATION_MAX_RETRIES",
),
target_sections=_select_optional_int(
selected.target_sections,
source.get("AUDITA_TARGET_SECTIONS"),
@@ -200,12 +239,24 @@ class AuditaConfig:
raise AuditaConfigError("AUDITA_LLM_TIMEOUT_SECONDS must be finite.")
if self.llm_timeout_seconds <= 0:
raise AuditaConfigError("AUDITA_LLM_TIMEOUT_SECONDS must be greater than zero.")
if self.validation_llm_concurrency is not None and self.validation_llm_concurrency <= 0:
raise AuditaConfigError("AUDITA_VALIDATION_LLM_CONCURRENCY must be greater than zero.")
if self.validation_llm_timeout_seconds is not None and not math.isfinite(self.validation_llm_timeout_seconds):
raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be finite.")
if self.validation_llm_timeout_seconds is not None and self.validation_llm_timeout_seconds <= 0:
raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be greater than zero.")
if self.validation_max_retries is not None and self.validation_max_retries < 0:
raise AuditaConfigError("AUDITA_VALIDATION_MAX_RETRIES must be greater than or equal to zero.")
if self.target_sections is not None and self.target_sections <= 0:
raise AuditaConfigError("AUDITA_TARGET_SECTIONS must be greater than zero.")
if not self.model.strip():
raise AuditaConfigError("AUDITA_MODEL must not be empty.")
if not self.base_url.strip():
raise AuditaConfigError("AUDITA_BASE_URL must not be empty.")
if self.validation_model is not None and not self.validation_model.strip():
raise AuditaConfigError("AUDITA_VALIDATION_MODEL must not be empty.")
if self.validation_base_url is not None and not self.validation_base_url.strip():
raise AuditaConfigError("AUDITA_VALIDATION_BASE_URL must not be empty.")
if self.max_retries < 0:
raise AuditaConfigError("AUDITA_MAX_RETRIES must be greater than or equal to zero.")
if self.max_section_tokens <= 0:
@@ -244,17 +295,79 @@ class AuditaConfig:
raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS must be greater than zero.")
if self.work_dir_retention not in ("auto", "always", "never"):
raise AuditaConfigError("AUDITA_WORK_DIR_RETENTION must be one of auto, always, or never.")
validation_config = self.validation_llm_config()
if validation_config.llm_concurrency <= 0:
raise AuditaConfigError("AUDITA_VALIDATION_LLM_CONCURRENCY must be greater than zero.")
if not math.isfinite(validation_config.llm_timeout_seconds):
raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be finite.")
if validation_config.llm_timeout_seconds <= 0:
raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be greater than zero.")
if not validation_config.model.strip():
raise AuditaConfigError("AUDITA_VALIDATION_MODEL must not be empty.")
if not validation_config.base_url.strip():
raise AuditaConfigError("AUDITA_VALIDATION_BASE_URL must not be empty.")
if validation_config.max_retries < 0:
raise AuditaConfigError("AUDITA_VALIDATION_MAX_RETRIES must be greater than or equal to zero.")
def proposal_llm_config(self) -> "AuditaConfig":
return self
def effective_validation_llm_api_key(self) -> Optional[str]:
return self.validation_llm_api_key if self.validation_llm_api_key is not None else self.api_key
def effective_validation_llm_concurrency(self) -> int:
return self.validation_llm_concurrency if self.validation_llm_concurrency is not None else self.llm_concurrency
def effective_validation_llm_timeout_seconds(self) -> float:
if self.validation_llm_timeout_seconds is not None:
return self.validation_llm_timeout_seconds
return self.llm_timeout_seconds
def effective_validation_model(self) -> str:
return self.validation_model if self.validation_model is not None else self.model
def effective_validation_base_url(self) -> str:
return self.validation_base_url if self.validation_base_url is not None else self.base_url
def effective_validation_max_retries(self) -> int:
return self.validation_max_retries if self.validation_max_retries is not None else self.max_retries
def validation_llm_config(self) -> "AuditaConfig":
return replace(
self,
api_key=self.effective_validation_llm_api_key(),
llm_concurrency=self.effective_validation_llm_concurrency(),
llm_timeout_seconds=self.effective_validation_llm_timeout_seconds(),
model=self.effective_validation_model(),
base_url=self.effective_validation_base_url(),
max_retries=self.effective_validation_max_retries(),
)
def to_report_dict(self) -> dict:
effective_validation_config = self.validation_llm_config()
return {
"api_key_configured": bool(self.api_key),
"llm_concurrency": self.llm_concurrency,
"llm_timeout_seconds": self.llm_timeout_seconds,
"validation_llm_api_key_configured": bool(self.validation_llm_api_key),
"validation_llm_concurrency": self.validation_llm_concurrency,
"validation_llm_timeout_seconds": self.validation_llm_timeout_seconds,
"validation_model": self.validation_model,
"validation_base_url": self.validation_base_url,
"validation_max_retries": self.validation_max_retries,
"target_sections": self.target_sections,
"module_keys": list(self.module_keys),
"model": self.model,
"base_url": self.base_url,
"max_retries": self.max_retries,
"effective_validation_llm": {
"api_key_configured": bool(effective_validation_config.api_key),
"llm_concurrency": effective_validation_config.llm_concurrency,
"llm_timeout_seconds": effective_validation_config.llm_timeout_seconds,
"model": effective_validation_config.model,
"base_url": effective_validation_config.base_url,
"max_retries": effective_validation_config.max_retries,
},
"max_section_tokens": self.max_section_tokens,
"min_section_tokens": self.min_section_tokens,
"glossary_confidence_threshold": self.glossary_confidence_threshold,
@@ -276,6 +389,12 @@ def _select_optional_string(value: Optional[str]) -> Optional[str]:
return stripped or None
def _select_optional_string_override(cli_value: Optional[str], env_value: Optional[str]) -> Optional[str]:
if cli_value is not None:
return _select_optional_string(cli_value)
return _select_optional_string(env_value)
def _select_api_key(
cli_value: Optional[str],
generic_env_value: Optional[str],
@@ -311,6 +430,17 @@ def _select_optional_int(cli_value: Optional[int], env_value: Optional[str], nam
raise AuditaConfigError(f"{name} must be an integer.") from exc
def _select_optional_float(cli_value: Optional[float], env_value: Optional[str], name: str) -> Optional[float]:
if cli_value is not None:
return cli_value
if env_value is None:
return None
try:
return float(env_value)
except ValueError as exc:
raise AuditaConfigError(f"{name} must be a number.") from exc
def _select_float(cli_value: Optional[float], env_value: Optional[str], default: float, name: str) -> float:
if cli_value is not None:
return cli_value

3
src/audita/framework/llm.py
View File

@@ -24,7 +24,8 @@ class OpenAICompatibleStructuredLLMClient:
if not config.api_key:
raise AuditaLLMError(
f"Structured LLM stage '{stage_name}' requires LLM API credentials to be configured "
"via --llm-api-key, AUDITA_LLM_API_KEY, or OPENROUTER_API_KEY."
"via --llm-api-key, --validation-llm-api-key, AUDITA_LLM_API_KEY, "
"AUDITA_VALIDATION_LLM_API_KEY, or OPENROUTER_API_KEY."
)
client = self._get_client(config)

1
src/audita/framework/models.py
View File

@@ -50,6 +50,7 @@ class ModuleContext:
run_dir: Path
llm_client: Optional["StructuredLLMClient"] = None
llm_scheduler: Optional[ModuleLLMScheduler] = None
validation_llm_scheduler: Optional[ModuleLLMScheduler] = None
class StructuredLLMClient(Protocol):

4
src/audita/framework/proposal_generation.py
View File

@@ -69,7 +69,7 @@ def generate_llm_correction_proposals(
stage_name=f"{context.run_spec.instance_name}:proposal",
messages=messages,
response_model=StructuredCorrectionSet,
config=context.config,
config=context.config.proposal_llm_config(),
)
)
else:
@@ -77,7 +77,7 @@ def generate_llm_correction_proposals(
stage_name=f"{context.run_spec.instance_name}:proposal",
messages=messages,
response_model=StructuredCorrectionSet,
config=context.config,
config=context.config.proposal_llm_config(),
)
response_path = context.run_dir / f"corrections-{section.section_index:04d}.json"
response_path.write_text(response.model_dump_json(indent=2) + "\n", encoding="utf-8")

8
src/audita/framework/runner.py
View File

@@ -87,6 +87,7 @@ class PipelineRunner:
run_dir=module_dir,
llm_client=llm_client,
llm_scheduler=ModuleLLMScheduler(config.llm_concurrency),
validation_llm_scheduler=ModuleLLMScheduler(config.effective_validation_llm_concurrency()),
)
sections = chunk_transcript(
working,
@@ -296,11 +297,11 @@ def _build_validation_context(
proposals=proposals,
transcript=working,
glossary=context.glossary,
config=context.config,
config=context.config.validation_llm_config(),
run_spec=context.run_spec,
run_dir=context.run_dir,
llm_client=llm_client,
llm_scheduler=context.llm_scheduler,
llm_scheduler=context.validation_llm_scheduler,
)
@@ -358,7 +359,8 @@ def _run_parallel_llm_validator_group(
context=context,
llm_client=llm_client,
)
max_workers = context.llm_scheduler.max_concurrency if context.llm_scheduler is not None else context.config.llm_concurrency
scheduler = context.validation_llm_scheduler
max_workers = scheduler.max_concurrency if scheduler is not None else context.config.effective_validation_llm_concurrency()
if max_workers == 1 or len(validators) <= 1:
results = [validator.validate(validation_context) for validator in validators]
else:

8
tests/test_framework_llm.py
View File

@@ -19,6 +19,12 @@ def _config(**overrides):
"api_key": "test-key",
"llm_concurrency": base.llm_concurrency,
"llm_timeout_seconds": base.llm_timeout_seconds,
"validation_llm_api_key": base.validation_llm_api_key,
"validation_llm_concurrency": base.validation_llm_concurrency,
"validation_llm_timeout_seconds": base.validation_llm_timeout_seconds,
"validation_model": base.validation_model,
"validation_base_url": base.validation_base_url,
"validation_max_retries": base.validation_max_retries,
"module_keys": base.module_keys,
"model": base.model,
"base_url": base.base_url,
@@ -193,7 +199,7 @@ def test_client_cache_identity_uses_timeout(monkeypatch):
def test_missing_api_key_error_is_provider_neutral():
client = OpenAICompatibleStructuredLLMClient()
with pytest.raises(AuditaLLMError, match="AUDITA_LLM_API_KEY"):
with pytest.raises(AuditaLLMError, match="AUDITA_VALIDATION_LLM_API_KEY"):
client.run_structured(
stage_name="test-stage",
messages=[{"role": "user", "content": "Hello"}],

169
tests/test_llm_validators.py
View File

@@ -40,6 +40,7 @@ class FakeStructuredLLMClient:
"stage_name": stage_name,
"messages": list(messages),
"response_model": response_model,
"config": config,
}
)
if not self._responses:
@@ -62,6 +63,7 @@ class CoordinatedStructuredLLMClient:
"stage_name": stage_name,
"messages": list(messages),
"response_model": response_model,
"config": config,
}
)
if not self._responses:
@@ -97,15 +99,20 @@ def _context(
tmp_path,
replacement_policy="require_unique",
llm_concurrency=1,
validation_llm_concurrency=None,
):
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "test-key"},
overrides=ConfigOverrides(
llm_concurrency=llm_concurrency,
validation_llm_concurrency=validation_llm_concurrency,
),
)
return ValidationContext(
proposals=proposals,
transcript=transcript,
glossary=_glossary(),
config=AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "test-key"},
overrides=ConfigOverrides(llm_concurrency=llm_concurrency),
),
config=config.validation_llm_config(),
run_spec=ModuleRunSpec(
instance_name="homophones",
module_key="homophones",
@@ -1190,3 +1197,157 @@ def test_llm_validators_process_batches_concurrently_and_preserve_proposal_order
(1, True),
]
assert len(client.calls) == 2
def test_validator_uses_validation_llm_config(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
)
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.97,
"reason": "ok",
}
]
}
]
)
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "primary-key"},
overrides=ConfigOverrides(
model="primary-model",
base_url="http://localhost:8000/v1",
max_retries=7,
llm_timeout_seconds=120,
validation_llm_api_key="validation-key",
validation_model="validation-model",
validation_base_url="http://localhost:9000/v1",
validation_max_retries=2,
validation_llm_timeout_seconds=240,
validation_llm_concurrency=3,
),
)
context = ValidationContext(
proposals=proposals,
transcript=transcript,
glossary=_glossary(),
config=config.validation_llm_config(),
run_spec=ModuleRunSpec(
instance_name="homophones",
module_key="homophones",
module=_Module("require_unique"),
),
run_dir=tmp_path,
llm_client=client,
)
SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(context)
validation_config = client.calls[0]["config"]
assert validation_config.api_key == "validation-key"
assert validation_config.model == "validation-model"
assert validation_config.base_url == "http://localhost:9000/v1"
assert validation_config.max_retries == 2
assert validation_config.llm_timeout_seconds == 240
assert validation_config.llm_concurrency == 3
def test_validator_uses_validation_llm_concurrency_override(tmp_path, monkeypatch):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "ChatGPT still can't do that with a dam."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="homophones",
module_key="homophones",
id=2,
original_text="dam",
corrected_text="damn",
confidence=0.95,
),
]
client = CoordinatedStructuredLLMClient(
[
lambda **kwargs: {
"validations": [
{
"correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1,
"approved": True,
"confidence": 0.97,
"reason": "ok",
}
]
},
lambda **kwargs: {
"validations": [
{
"correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1,
"approved": True,
"confidence": 0.94,
"reason": "ok",
}
]
},
],
threading.Barrier(2, timeout=1.0),
)
def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message):
return [
TokenBatch(batch_index=0, items=list(items[:1]), token_count=1),
TokenBatch(batch_index=1, items=list(items[1:]), token_count=1),
]
monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items)
result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
_context(
proposals=proposals,
transcript=transcript,
llm_client=client,
tmp_path=tmp_path,
llm_concurrency=1,
validation_llm_concurrency=2,
)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
(0, True),
(1, True),
]
assert len(client.calls) == 2

58
tests/test_module_proposals.py
View File

@@ -31,6 +31,7 @@ class FakeStructuredLLMClient:
"stage_name": stage_name,
"messages": list(messages),
"response_model": response_model,
"config": config,
}
)
payload = _pop_llm_response(self._responses, stage_name)
@@ -115,6 +116,63 @@ def test_glossary_module_propose_writes_diagnostics_and_returns_proposals_withou
assert "gestures" in prompt_text
def test_module_propose_uses_primary_llm_config(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
]
"""
)
section = chunk_transcript(transcript, max_section_tokens=1000)[0]
module = GlossaryModule()
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "gestures",
"corrected_text": "Jesters",
"confidence": 0.95,
}
]
}
]
)
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "primary-key"},
overrides=ConfigOverrides(
model="primary-model",
base_url="http://localhost:8000/v1",
max_retries=7,
llm_timeout_seconds=120,
validation_llm_api_key="validation-key",
validation_model="validation-model",
validation_base_url="http://localhost:9000/v1",
validation_max_retries=2,
validation_llm_timeout_seconds=240,
validation_llm_concurrency=3,
),
)
context = ModuleContext(
run_spec=ModuleRunSpec(instance_name="glossary_primary", module_key="glossary", module=module),
glossary=_glossary(),
config=config,
run_dir=tmp_path,
llm_client=client,
)
module.propose(section, context)
proposal_config = client.calls[0]["config"]
assert proposal_config.api_key == "primary-key"
assert proposal_config.model == "primary-model"
assert proposal_config.base_url == "http://localhost:8000/v1"
assert proposal_config.max_retries == 7
assert proposal_config.llm_timeout_seconds == 120
def test_homophones_prompt_is_explicitly_scoped_to_spoken_form_corrections():
transcript = parse_transcript_json(
"""

82
tests/test_new_cli.py
View File

@@ -23,6 +23,12 @@ def test_process_help_exposes_framework_flags(capsys):
assert "--llm-api-key" in output
assert "--llm-concurrency" in output
assert "--llm-timeout-seconds" in output
assert "--validation-llm-api-key" in output
assert "--validation-llm-concurrency" in output
assert "--validation-llm-timeout-seconds" in output
assert "--validation-model" in output
assert "--validation-base-url" in output
assert "--validation-max-retries" in output
assert "--target-sections" in output
assert "--modules" in output
assert "--model" in output
@@ -308,6 +314,82 @@ def test_cli_process_passes_llm_timeout_seconds_override_to_config(monkeypatch,
assert captured["llm_timeout_seconds"] == 900.0
def test_cli_process_passes_validation_llm_overrides_to_config(monkeypatch, tmp_path):
captured = {}
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
]
"""
)
report = RunReport(
status="success",
config={"model": "m", "base_url": "b"},
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
pipeline=["grammar"],
modules=[],
applied_changes=[],
skipped_corrections=[],
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
work_dir_retention="auto",
work_dir_retained=False,
work_dir=None,
error=None,
)
result = ProcessResult(
transcript=transcript,
report=report,
run_dir=tmp_path / "run",
work_dir_retained=False,
)
def _fake_from_sources(*, overrides=None):
captured["validation_llm_api_key"] = overrides.validation_llm_api_key
captured["validation_llm_concurrency"] = overrides.validation_llm_concurrency
captured["validation_llm_timeout_seconds"] = overrides.validation_llm_timeout_seconds
captured["validation_model"] = overrides.validation_model
captured["validation_base_url"] = overrides.validation_base_url
captured["validation_max_retries"] = overrides.validation_max_retries
return object()
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--validation-llm-api-key",
"validator-key",
"--validation-llm-concurrency",
"4",
"--validation-llm-timeout-seconds",
"180",
"--validation-model",
"validator-model",
"--validation-base-url",
"http://localhost:9000/v1",
"--validation-max-retries",
"2",
]
)
assert exit_code == 0
assert captured == {
"validation_llm_api_key": "validator-key",
"validation_llm_concurrency": 4,
"validation_llm_timeout_seconds": 180.0,
"validation_model": "validator-model",
"validation_base_url": "http://localhost:9000/v1",
"validation_max_retries": 2,
}
def test_cli_process_passes_target_sections_override_to_config(monkeypatch, tmp_path):
captured = {}
transcript = parse_transcript_json(

143
tests/test_new_config.py
View File

@@ -24,6 +24,12 @@ def test_default_config_allows_missing_api_key():
assert config.api_key is None
assert config.llm_concurrency == DEFAULT_LLM_CONCURRENCY
assert config.llm_timeout_seconds == DEFAULT_LLM_TIMEOUT_SECONDS
assert config.validation_llm_api_key is None
assert config.validation_llm_concurrency is None
assert config.validation_llm_timeout_seconds is None
assert config.validation_model is None
assert config.validation_base_url is None
assert config.validation_max_retries is None
assert config.target_sections is None
assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS
assert config.min_section_tokens == DEFAULT_MIN_SECTION_TOKENS
@@ -84,6 +90,54 @@ def test_llm_timeout_seconds_env_is_parsed():
assert config.llm_timeout_seconds == 120.5
def test_validation_llm_concurrency_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_VALIDATION_LLM_CONCURRENCY": "2"},
overrides=ConfigOverrides(validation_llm_concurrency=4),
)
assert config.validation_llm_concurrency == 4
def test_validation_llm_concurrency_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_CONCURRENCY": "3"})
assert config.validation_llm_concurrency == 3
def test_validation_llm_timeout_seconds_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "120"},
overrides=ConfigOverrides(validation_llm_timeout_seconds=900.0),
)
assert config.validation_llm_timeout_seconds == 900.0
def test_validation_llm_timeout_seconds_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "120.5"})
assert config.validation_llm_timeout_seconds == 120.5
def test_validation_model_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MODEL": "validator-model"})
assert config.validation_model == "validator-model"
def test_validation_base_url_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_BASE_URL": "http://localhost:9000/v1"})
assert config.validation_base_url == "http://localhost:9000/v1"
def test_validation_max_retries_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_RETRIES": "7"})
assert config.validation_max_retries == 7
def test_target_sections_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_TARGET_SECTIONS": "2"},
@@ -143,6 +197,62 @@ def test_blank_llm_api_key_override_resolves_to_none():
assert config.api_key is None
def test_validation_llm_api_key_env_is_read():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_API_KEY": "validation-key"})
assert config.validation_llm_api_key == "validation-key"
def test_effective_validation_fields_fall_back_to_primary_settings():
config = AuditaConfig.from_sources(
env={
"AUDITA_LLM_API_KEY": "primary-key",
"AUDITA_MODEL": "primary-model",
"AUDITA_BASE_URL": "http://localhost:8000/v1",
"AUDITA_LLM_TIMEOUT_SECONDS": "120",
"AUDITA_LLM_CONCURRENCY": "5",
"AUDITA_MAX_RETRIES": "9",
}
)
validation = config.validation_llm_config()
assert validation.api_key == "primary-key"
assert validation.model == "primary-model"
assert validation.base_url == "http://localhost:8000/v1"
assert validation.llm_timeout_seconds == 120.0
assert validation.llm_concurrency == 5
assert validation.max_retries == 9
def test_effective_validation_fields_use_overrides_when_set():
config = AuditaConfig.from_sources(
env={
"AUDITA_LLM_API_KEY": "primary-key",
"AUDITA_MODEL": "primary-model",
"AUDITA_BASE_URL": "http://localhost:8000/v1",
"AUDITA_LLM_TIMEOUT_SECONDS": "120",
"AUDITA_LLM_CONCURRENCY": "5",
"AUDITA_MAX_RETRIES": "9",
"AUDITA_VALIDATION_LLM_API_KEY": "validation-key",
"AUDITA_VALIDATION_MODEL": "validation-model",
"AUDITA_VALIDATION_BASE_URL": "http://localhost:9000/v1",
"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "240",
"AUDITA_VALIDATION_LLM_CONCURRENCY": "3",
"AUDITA_VALIDATION_MAX_RETRIES": "2",
}
)
validation = config.validation_llm_config()
assert validation.api_key == "validation-key"
assert validation.model == "validation-model"
assert validation.base_url == "http://localhost:9000/v1"
assert validation.llm_timeout_seconds == 240.0
assert validation.llm_concurrency == 3
assert validation.max_retries == 2
def test_module_key_overrides_take_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_MODULES": "grammar"},
@@ -175,6 +285,21 @@ def test_report_dict_includes_target_sections():
assert config.to_report_dict()["target_sections"] == 7
def test_report_dict_includes_effective_validation_llm_config():
config = AuditaConfig.from_sources(
env={
"AUDITA_LLM_API_KEY": "primary-key",
"AUDITA_VALIDATION_MODEL": "validation-model",
}
)
report = config.to_report_dict()
assert report["validation_model"] == "validation-model"
assert report["effective_validation_llm"]["api_key_configured"] is True
assert report["effective_validation_llm"]["model"] == "validation-model"
def test_threshold_overrides_take_precedence():
config = AuditaConfig.from_sources(
env={
@@ -236,6 +361,24 @@ def test_invalid_llm_timeout_seconds_is_rejected(value):
AuditaConfig.from_sources(env={"AUDITA_LLM_TIMEOUT_SECONDS": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_validation_llm_concurrency_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_LLM_CONCURRENCY"):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_CONCURRENCY": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_validation_llm_timeout_seconds_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": value})
@pytest.mark.parametrize("value", ["-1", "many"])
def test_invalid_validation_max_retries_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_MAX_RETRIES"):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_RETRIES": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_target_sections_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_TARGET_SECTIONS"):

80
tests/test_new_pipeline.py
View File

@@ -24,6 +24,7 @@ class FakeStructuredLLMClient:
"stage_name": stage_name,
"messages": list(messages),
"response_model": response_model,
"config": config,
}
)
response = _pop_llm_response(self._responses, stage_name)
@@ -103,6 +104,85 @@ def test_process_transcript_runs_noop_framework(tmp_path):
]
def test_process_transcript_result_can_use_different_validation_llm_settings(tmp_path):
llm_client = FakeStructuredLLMClient(
{
"grammar:proposal": {
"corrections": [
{
"id": 1,
"original_text": "hello world",
"corrected_text": "Hello world.",
"confidence": 0.95,
}
]
},
"grammar:grammar_only_guard": {
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.98,
"reason": "ok",
}
]
},
"grammar:meaning_reversal_review": {
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.99,
"reason": "ok",
}
]
},
}
)
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "primary-key"},
overrides=ConfigOverrides(
model="primary-model",
base_url="http://localhost:8000/v1",
max_retries=7,
llm_timeout_seconds=120,
validation_llm_api_key="validation-key",
validation_model="validation-model",
validation_base_url="http://localhost:9000/v1",
validation_max_retries=2,
validation_llm_timeout_seconds=240,
validation_llm_concurrency=3,
work_dir=tmp_path / "work",
work_dir_retention="always",
),
)
result = process_transcript_result(
parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
]
"""
),
_glossary(),
config,
module_keys=["grammar"],
llm_client=llm_client,
)
assert result.transcript[0].text == "Hello world."
calls_by_stage = {call["stage_name"]: call["config"] for call in llm_client.calls}
assert calls_by_stage["grammar:proposal"].model == "primary-model"
assert calls_by_stage["grammar:proposal"].base_url == "http://localhost:8000/v1"
assert calls_by_stage["grammar:proposal"].api_key == "primary-key"
assert calls_by_stage["grammar:grammar_only_guard"].model == "validation-model"
assert calls_by_stage["grammar:grammar_only_guard"].base_url == "http://localhost:9000/v1"
assert calls_by_stage["grammar:grammar_only_guard"].api_key == "validation-key"
assert calls_by_stage["grammar:grammar_only_guard"].max_retries == 2
assert calls_by_stage["grammar:grammar_only_guard"].llm_timeout_seconds == 240
def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(tmp_path):
config = AuditaConfig.from_sources(
env={},