Added validation-specific LLM configuration options

This commit is contained in:
2026-04-29 20:45:51 -05:00
parent 4c899122d9
commit 54ce3d08c9
13 changed files with 714 additions and 12 deletions

View File

@@ -85,11 +85,17 @@ Useful configuration can be supplied by CLI flag or environment variable. CLI fl
| --- | --- | --- | --- | | --- | --- | --- | --- |
| `AUDITA_MODULES` | `--modules` | `glossary,homophones,glossary,spoken_word,grammar` | Comma-separated logical module keys to run; CLI overrides the environment value | | `AUDITA_MODULES` | `--modules` | `glossary,homophones,glossary,spoken_word,grammar` | Comma-separated logical module keys to run; CLI overrides the environment value |
| `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; CLI overrides both environment-key variants | | `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; CLI overrides both environment-key variants |
| `AUDITA_VALIDATION_LLM_API_KEY` | `--validation-llm-api-key` | unset | Validation-phase LLM API credential; defaults to the primary LLM API key |
| `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model name sent to the configured OpenAI-compatible endpoint | | `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model name sent to the configured OpenAI-compatible endpoint |
| `AUDITA_VALIDATION_MODEL` | `--validation-model` | unset | Validation-phase LLM model; defaults to `AUDITA_MODEL` |
| `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL | | `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL |
| `AUDITA_VALIDATION_BASE_URL` | `--validation-base-url` | unset | Validation-phase OpenAI-compatible API base URL; defaults to `AUDITA_BASE_URL` |
| `AUDITA_LLM_TIMEOUT_SECONDS` | `--llm-timeout-seconds` | `600` | Per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint | | `AUDITA_LLM_TIMEOUT_SECONDS` | `--llm-timeout-seconds` | `600` | Per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint |
| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | `--validation-llm-timeout-seconds` | unset | Validation-phase per-request timeout in seconds; defaults to `AUDITA_LLM_TIMEOUT_SECONDS` |
| `AUDITA_TARGET_SECTIONS` | `--target-sections` | unset | Exact number of contiguous proposal-stage transcript sections; errors if min/max token bounds cannot be satisfied | | `AUDITA_TARGET_SECTIONS` | `--target-sections` | unset | Exact number of contiguous proposal-stage transcript sections; errors if min/max token bounds cannot be satisfied |
| `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses | | `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses |
| `AUDITA_VALIDATION_MAX_RETRIES` | `--validation-max-retries` | unset | Validation-phase structured-output retries; defaults to `AUDITA_MAX_RETRIES` |
| `AUDITA_VALIDATION_LLM_CONCURRENCY` | `--validation-llm-concurrency` | unset | Validation-phase LLM concurrency; defaults to `AUDITA_LLM_CONCURRENCY` |
| `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `8192` | Maximum estimated tokens per proposal-stage transcript section | | `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `8192` | Maximum estimated tokens per proposal-stage transcript section |
| `AUDITA_MIN_SECTION_TOKENS` | `--min-section-tokens` | `2048` | Minimum estimated tokens per proposal-stage transcript section when balancing for concurrency | | `AUDITA_MIN_SECTION_TOKENS` | `--min-section-tokens` | `2048` | Minimum estimated tokens per proposal-stage transcript section when balancing for concurrency |
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation | | `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation |
@@ -105,6 +111,8 @@ Useful configuration can be supplied by CLI flag or environment variable. CLI fl
Set `AUDITA_MODULES=grammar` to run only the grammar module by default, or override it per command with `--modules`. Set `AUDITA_MODULES=grammar` to run only the grammar module by default, or override it per command with `--modules`.
Validation-phase LLM settings inherit from the primary `AUDITA_*` LLM settings by default. Set any of the `AUDITA_VALIDATION_*` values only when you want LLM-backed validators to use a different model, endpoint, credential, timeout, retry budget, or concurrency level.
OpenRouter remains the default out of the box: OpenRouter remains the default out of the box:
```sh ```sh

View File

@@ -34,6 +34,30 @@ def _build_parser() -> argparse.ArgumentParser:
type=float, type=float,
help="per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint", help="per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint",
) )
process.add_argument(
"--validation-llm-api-key",
help="LLM API key for validation phases; defaults to the primary configured OpenAI-compatible endpoint key",
)
process.add_argument(
"--validation-llm-concurrency",
type=int,
help="maximum concurrent LLM calls within validation phases; defaults to --llm-concurrency",
)
process.add_argument(
"--validation-llm-timeout-seconds",
type=float,
help="per-request timeout in seconds for validation-phase LLM calls; defaults to --llm-timeout-seconds",
)
process.add_argument("--validation-model", help="LLM model name for validation phases; defaults to --model")
process.add_argument(
"--validation-base-url",
help="OpenAI-compatible API base URL for validation phases; defaults to --base-url",
)
process.add_argument(
"--validation-max-retries",
type=int,
help="maximum structured-output retries for validation phases; defaults to --max-retries",
)
process.add_argument( process.add_argument(
"--target-sections", "--target-sections",
type=int, type=int,
@@ -108,6 +132,12 @@ def _process(args: argparse.Namespace) -> int:
llm_api_key=args.llm_api_key, llm_api_key=args.llm_api_key,
llm_concurrency=args.llm_concurrency, llm_concurrency=args.llm_concurrency,
llm_timeout_seconds=args.llm_timeout_seconds, llm_timeout_seconds=args.llm_timeout_seconds,
validation_llm_api_key=args.validation_llm_api_key,
validation_llm_concurrency=args.validation_llm_concurrency,
validation_llm_timeout_seconds=args.validation_llm_timeout_seconds,
validation_model=args.validation_model,
validation_base_url=args.validation_base_url,
validation_max_retries=args.validation_max_retries,
target_sections=args.target_sections, target_sections=args.target_sections,
module_keys=args.modules, module_keys=args.modules,
model=args.model, model=args.model,

View File

@@ -1,6 +1,6 @@
import math import math
import os import os
from dataclasses import dataclass from dataclasses import dataclass, replace
from pathlib import Path from pathlib import Path
from typing import Mapping, Optional, Sequence, Tuple, Union from typing import Mapping, Optional, Sequence, Tuple, Union
@@ -33,6 +33,12 @@ class ConfigOverrides:
llm_api_key: Optional[str] = None llm_api_key: Optional[str] = None
llm_concurrency: Optional[int] = None llm_concurrency: Optional[int] = None
llm_timeout_seconds: Optional[float] = None llm_timeout_seconds: Optional[float] = None
validation_llm_api_key: Optional[str] = None
validation_llm_concurrency: Optional[int] = None
validation_llm_timeout_seconds: Optional[float] = None
validation_model: Optional[str] = None
validation_base_url: Optional[str] = None
validation_max_retries: Optional[int] = None
target_sections: Optional[int] = None target_sections: Optional[int] = None
module_keys: Optional[Union[str, Sequence[str]]] = None module_keys: Optional[Union[str, Sequence[str]]] = None
model: Optional[str] = None model: Optional[str] = None
@@ -57,6 +63,12 @@ class AuditaConfig:
api_key: Optional[str] = None api_key: Optional[str] = None
llm_concurrency: int = DEFAULT_LLM_CONCURRENCY llm_concurrency: int = DEFAULT_LLM_CONCURRENCY
llm_timeout_seconds: float = DEFAULT_LLM_TIMEOUT_SECONDS llm_timeout_seconds: float = DEFAULT_LLM_TIMEOUT_SECONDS
validation_llm_api_key: Optional[str] = None
validation_llm_concurrency: Optional[int] = None
validation_llm_timeout_seconds: Optional[float] = None
validation_model: Optional[str] = None
validation_base_url: Optional[str] = None
validation_max_retries: Optional[int] = None
target_sections: Optional[int] = None target_sections: Optional[int] = None
module_keys: Tuple[str, ...] = DEFAULT_MODULE_KEYS module_keys: Tuple[str, ...] = DEFAULT_MODULE_KEYS
model: str = DEFAULT_MODEL model: str = DEFAULT_MODEL
@@ -101,6 +113,33 @@ class AuditaConfig:
DEFAULT_LLM_TIMEOUT_SECONDS, DEFAULT_LLM_TIMEOUT_SECONDS,
"AUDITA_LLM_TIMEOUT_SECONDS", "AUDITA_LLM_TIMEOUT_SECONDS",
), ),
validation_llm_api_key=_select_optional_string_override(
selected.validation_llm_api_key,
source.get("AUDITA_VALIDATION_LLM_API_KEY"),
),
validation_llm_concurrency=_select_optional_int(
selected.validation_llm_concurrency,
source.get("AUDITA_VALIDATION_LLM_CONCURRENCY"),
"AUDITA_VALIDATION_LLM_CONCURRENCY",
),
validation_llm_timeout_seconds=_select_optional_float(
selected.validation_llm_timeout_seconds,
source.get("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"),
"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS",
),
validation_model=_select_optional_string_override(
selected.validation_model,
source.get("AUDITA_VALIDATION_MODEL"),
),
validation_base_url=_select_optional_string_override(
selected.validation_base_url,
source.get("AUDITA_VALIDATION_BASE_URL"),
),
validation_max_retries=_select_optional_int(
selected.validation_max_retries,
source.get("AUDITA_VALIDATION_MAX_RETRIES"),
"AUDITA_VALIDATION_MAX_RETRIES",
),
target_sections=_select_optional_int( target_sections=_select_optional_int(
selected.target_sections, selected.target_sections,
source.get("AUDITA_TARGET_SECTIONS"), source.get("AUDITA_TARGET_SECTIONS"),
@@ -200,12 +239,24 @@ class AuditaConfig:
raise AuditaConfigError("AUDITA_LLM_TIMEOUT_SECONDS must be finite.") raise AuditaConfigError("AUDITA_LLM_TIMEOUT_SECONDS must be finite.")
if self.llm_timeout_seconds <= 0: if self.llm_timeout_seconds <= 0:
raise AuditaConfigError("AUDITA_LLM_TIMEOUT_SECONDS must be greater than zero.") raise AuditaConfigError("AUDITA_LLM_TIMEOUT_SECONDS must be greater than zero.")
if self.validation_llm_concurrency is not None and self.validation_llm_concurrency <= 0:
raise AuditaConfigError("AUDITA_VALIDATION_LLM_CONCURRENCY must be greater than zero.")
if self.validation_llm_timeout_seconds is not None and not math.isfinite(self.validation_llm_timeout_seconds):
raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be finite.")
if self.validation_llm_timeout_seconds is not None and self.validation_llm_timeout_seconds <= 0:
raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be greater than zero.")
if self.validation_max_retries is not None and self.validation_max_retries < 0:
raise AuditaConfigError("AUDITA_VALIDATION_MAX_RETRIES must be greater than or equal to zero.")
if self.target_sections is not None and self.target_sections <= 0: if self.target_sections is not None and self.target_sections <= 0:
raise AuditaConfigError("AUDITA_TARGET_SECTIONS must be greater than zero.") raise AuditaConfigError("AUDITA_TARGET_SECTIONS must be greater than zero.")
if not self.model.strip(): if not self.model.strip():
raise AuditaConfigError("AUDITA_MODEL must not be empty.") raise AuditaConfigError("AUDITA_MODEL must not be empty.")
if not self.base_url.strip(): if not self.base_url.strip():
raise AuditaConfigError("AUDITA_BASE_URL must not be empty.") raise AuditaConfigError("AUDITA_BASE_URL must not be empty.")
if self.validation_model is not None and not self.validation_model.strip():
raise AuditaConfigError("AUDITA_VALIDATION_MODEL must not be empty.")
if self.validation_base_url is not None and not self.validation_base_url.strip():
raise AuditaConfigError("AUDITA_VALIDATION_BASE_URL must not be empty.")
if self.max_retries < 0: if self.max_retries < 0:
raise AuditaConfigError("AUDITA_MAX_RETRIES must be greater than or equal to zero.") raise AuditaConfigError("AUDITA_MAX_RETRIES must be greater than or equal to zero.")
if self.max_section_tokens <= 0: if self.max_section_tokens <= 0:
@@ -244,17 +295,79 @@ class AuditaConfig:
raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS must be greater than zero.") raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS must be greater than zero.")
if self.work_dir_retention not in ("auto", "always", "never"): if self.work_dir_retention not in ("auto", "always", "never"):
raise AuditaConfigError("AUDITA_WORK_DIR_RETENTION must be one of auto, always, or never.") raise AuditaConfigError("AUDITA_WORK_DIR_RETENTION must be one of auto, always, or never.")
validation_config = self.validation_llm_config()
if validation_config.llm_concurrency <= 0:
raise AuditaConfigError("AUDITA_VALIDATION_LLM_CONCURRENCY must be greater than zero.")
if not math.isfinite(validation_config.llm_timeout_seconds):
raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be finite.")
if validation_config.llm_timeout_seconds <= 0:
raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be greater than zero.")
if not validation_config.model.strip():
raise AuditaConfigError("AUDITA_VALIDATION_MODEL must not be empty.")
if not validation_config.base_url.strip():
raise AuditaConfigError("AUDITA_VALIDATION_BASE_URL must not be empty.")
if validation_config.max_retries < 0:
raise AuditaConfigError("AUDITA_VALIDATION_MAX_RETRIES must be greater than or equal to zero.")
def proposal_llm_config(self) -> "AuditaConfig":
return self
def effective_validation_llm_api_key(self) -> Optional[str]:
return self.validation_llm_api_key if self.validation_llm_api_key is not None else self.api_key
def effective_validation_llm_concurrency(self) -> int:
return self.validation_llm_concurrency if self.validation_llm_concurrency is not None else self.llm_concurrency
def effective_validation_llm_timeout_seconds(self) -> float:
if self.validation_llm_timeout_seconds is not None:
return self.validation_llm_timeout_seconds
return self.llm_timeout_seconds
def effective_validation_model(self) -> str:
return self.validation_model if self.validation_model is not None else self.model
def effective_validation_base_url(self) -> str:
return self.validation_base_url if self.validation_base_url is not None else self.base_url
def effective_validation_max_retries(self) -> int:
return self.validation_max_retries if self.validation_max_retries is not None else self.max_retries
def validation_llm_config(self) -> "AuditaConfig":
return replace(
self,
api_key=self.effective_validation_llm_api_key(),
llm_concurrency=self.effective_validation_llm_concurrency(),
llm_timeout_seconds=self.effective_validation_llm_timeout_seconds(),
model=self.effective_validation_model(),
base_url=self.effective_validation_base_url(),
max_retries=self.effective_validation_max_retries(),
)
def to_report_dict(self) -> dict: def to_report_dict(self) -> dict:
effective_validation_config = self.validation_llm_config()
return { return {
"api_key_configured": bool(self.api_key), "api_key_configured": bool(self.api_key),
"llm_concurrency": self.llm_concurrency, "llm_concurrency": self.llm_concurrency,
"llm_timeout_seconds": self.llm_timeout_seconds, "llm_timeout_seconds": self.llm_timeout_seconds,
"validation_llm_api_key_configured": bool(self.validation_llm_api_key),
"validation_llm_concurrency": self.validation_llm_concurrency,
"validation_llm_timeout_seconds": self.validation_llm_timeout_seconds,
"validation_model": self.validation_model,
"validation_base_url": self.validation_base_url,
"validation_max_retries": self.validation_max_retries,
"target_sections": self.target_sections, "target_sections": self.target_sections,
"module_keys": list(self.module_keys), "module_keys": list(self.module_keys),
"model": self.model, "model": self.model,
"base_url": self.base_url, "base_url": self.base_url,
"max_retries": self.max_retries, "max_retries": self.max_retries,
"effective_validation_llm": {
"api_key_configured": bool(effective_validation_config.api_key),
"llm_concurrency": effective_validation_config.llm_concurrency,
"llm_timeout_seconds": effective_validation_config.llm_timeout_seconds,
"model": effective_validation_config.model,
"base_url": effective_validation_config.base_url,
"max_retries": effective_validation_config.max_retries,
},
"max_section_tokens": self.max_section_tokens, "max_section_tokens": self.max_section_tokens,
"min_section_tokens": self.min_section_tokens, "min_section_tokens": self.min_section_tokens,
"glossary_confidence_threshold": self.glossary_confidence_threshold, "glossary_confidence_threshold": self.glossary_confidence_threshold,
@@ -276,6 +389,12 @@ def _select_optional_string(value: Optional[str]) -> Optional[str]:
return stripped or None return stripped or None
def _select_optional_string_override(cli_value: Optional[str], env_value: Optional[str]) -> Optional[str]:
if cli_value is not None:
return _select_optional_string(cli_value)
return _select_optional_string(env_value)
def _select_api_key( def _select_api_key(
cli_value: Optional[str], cli_value: Optional[str],
generic_env_value: Optional[str], generic_env_value: Optional[str],
@@ -311,6 +430,17 @@ def _select_optional_int(cli_value: Optional[int], env_value: Optional[str], nam
raise AuditaConfigError(f"{name} must be an integer.") from exc raise AuditaConfigError(f"{name} must be an integer.") from exc
def _select_optional_float(cli_value: Optional[float], env_value: Optional[str], name: str) -> Optional[float]:
if cli_value is not None:
return cli_value
if env_value is None:
return None
try:
return float(env_value)
except ValueError as exc:
raise AuditaConfigError(f"{name} must be a number.") from exc
def _select_float(cli_value: Optional[float], env_value: Optional[str], default: float, name: str) -> float: def _select_float(cli_value: Optional[float], env_value: Optional[str], default: float, name: str) -> float:
if cli_value is not None: if cli_value is not None:
return cli_value return cli_value

View File

@@ -24,7 +24,8 @@ class OpenAICompatibleStructuredLLMClient:
if not config.api_key: if not config.api_key:
raise AuditaLLMError( raise AuditaLLMError(
f"Structured LLM stage '{stage_name}' requires LLM API credentials to be configured " f"Structured LLM stage '{stage_name}' requires LLM API credentials to be configured "
"via --llm-api-key, AUDITA_LLM_API_KEY, or OPENROUTER_API_KEY." "via --llm-api-key, --validation-llm-api-key, AUDITA_LLM_API_KEY, "
"AUDITA_VALIDATION_LLM_API_KEY, or OPENROUTER_API_KEY."
) )
client = self._get_client(config) client = self._get_client(config)

View File

@@ -50,6 +50,7 @@ class ModuleContext:
run_dir: Path run_dir: Path
llm_client: Optional["StructuredLLMClient"] = None llm_client: Optional["StructuredLLMClient"] = None
llm_scheduler: Optional[ModuleLLMScheduler] = None llm_scheduler: Optional[ModuleLLMScheduler] = None
validation_llm_scheduler: Optional[ModuleLLMScheduler] = None
class StructuredLLMClient(Protocol): class StructuredLLMClient(Protocol):

View File

@@ -69,7 +69,7 @@ def generate_llm_correction_proposals(
stage_name=f"{context.run_spec.instance_name}:proposal", stage_name=f"{context.run_spec.instance_name}:proposal",
messages=messages, messages=messages,
response_model=StructuredCorrectionSet, response_model=StructuredCorrectionSet,
config=context.config, config=context.config.proposal_llm_config(),
) )
) )
else: else:
@@ -77,7 +77,7 @@ def generate_llm_correction_proposals(
stage_name=f"{context.run_spec.instance_name}:proposal", stage_name=f"{context.run_spec.instance_name}:proposal",
messages=messages, messages=messages,
response_model=StructuredCorrectionSet, response_model=StructuredCorrectionSet,
config=context.config, config=context.config.proposal_llm_config(),
) )
response_path = context.run_dir / f"corrections-{section.section_index:04d}.json" response_path = context.run_dir / f"corrections-{section.section_index:04d}.json"
response_path.write_text(response.model_dump_json(indent=2) + "\n", encoding="utf-8") response_path.write_text(response.model_dump_json(indent=2) + "\n", encoding="utf-8")

View File

@@ -87,6 +87,7 @@ class PipelineRunner:
run_dir=module_dir, run_dir=module_dir,
llm_client=llm_client, llm_client=llm_client,
llm_scheduler=ModuleLLMScheduler(config.llm_concurrency), llm_scheduler=ModuleLLMScheduler(config.llm_concurrency),
validation_llm_scheduler=ModuleLLMScheduler(config.effective_validation_llm_concurrency()),
) )
sections = chunk_transcript( sections = chunk_transcript(
working, working,
@@ -296,11 +297,11 @@ def _build_validation_context(
proposals=proposals, proposals=proposals,
transcript=working, transcript=working,
glossary=context.glossary, glossary=context.glossary,
config=context.config, config=context.config.validation_llm_config(),
run_spec=context.run_spec, run_spec=context.run_spec,
run_dir=context.run_dir, run_dir=context.run_dir,
llm_client=llm_client, llm_client=llm_client,
llm_scheduler=context.llm_scheduler, llm_scheduler=context.validation_llm_scheduler,
) )
@@ -358,7 +359,8 @@ def _run_parallel_llm_validator_group(
context=context, context=context,
llm_client=llm_client, llm_client=llm_client,
) )
max_workers = context.llm_scheduler.max_concurrency if context.llm_scheduler is not None else context.config.llm_concurrency scheduler = context.validation_llm_scheduler
max_workers = scheduler.max_concurrency if scheduler is not None else context.config.effective_validation_llm_concurrency()
if max_workers == 1 or len(validators) <= 1: if max_workers == 1 or len(validators) <= 1:
results = [validator.validate(validation_context) for validator in validators] results = [validator.validate(validation_context) for validator in validators]
else: else:

View File

@@ -19,6 +19,12 @@ def _config(**overrides):
"api_key": "test-key", "api_key": "test-key",
"llm_concurrency": base.llm_concurrency, "llm_concurrency": base.llm_concurrency,
"llm_timeout_seconds": base.llm_timeout_seconds, "llm_timeout_seconds": base.llm_timeout_seconds,
"validation_llm_api_key": base.validation_llm_api_key,
"validation_llm_concurrency": base.validation_llm_concurrency,
"validation_llm_timeout_seconds": base.validation_llm_timeout_seconds,
"validation_model": base.validation_model,
"validation_base_url": base.validation_base_url,
"validation_max_retries": base.validation_max_retries,
"module_keys": base.module_keys, "module_keys": base.module_keys,
"model": base.model, "model": base.model,
"base_url": base.base_url, "base_url": base.base_url,
@@ -193,7 +199,7 @@ def test_client_cache_identity_uses_timeout(monkeypatch):
def test_missing_api_key_error_is_provider_neutral(): def test_missing_api_key_error_is_provider_neutral():
client = OpenAICompatibleStructuredLLMClient() client = OpenAICompatibleStructuredLLMClient()
with pytest.raises(AuditaLLMError, match="AUDITA_LLM_API_KEY"): with pytest.raises(AuditaLLMError, match="AUDITA_VALIDATION_LLM_API_KEY"):
client.run_structured( client.run_structured(
stage_name="test-stage", stage_name="test-stage",
messages=[{"role": "user", "content": "Hello"}], messages=[{"role": "user", "content": "Hello"}],

View File

@@ -40,6 +40,7 @@ class FakeStructuredLLMClient:
"stage_name": stage_name, "stage_name": stage_name,
"messages": list(messages), "messages": list(messages),
"response_model": response_model, "response_model": response_model,
"config": config,
} }
) )
if not self._responses: if not self._responses:
@@ -62,6 +63,7 @@ class CoordinatedStructuredLLMClient:
"stage_name": stage_name, "stage_name": stage_name,
"messages": list(messages), "messages": list(messages),
"response_model": response_model, "response_model": response_model,
"config": config,
} }
) )
if not self._responses: if not self._responses:
@@ -97,15 +99,20 @@ def _context(
tmp_path, tmp_path,
replacement_policy="require_unique", replacement_policy="require_unique",
llm_concurrency=1, llm_concurrency=1,
validation_llm_concurrency=None,
): ):
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "test-key"},
overrides=ConfigOverrides(
llm_concurrency=llm_concurrency,
validation_llm_concurrency=validation_llm_concurrency,
),
)
return ValidationContext( return ValidationContext(
proposals=proposals, proposals=proposals,
transcript=transcript, transcript=transcript,
glossary=_glossary(), glossary=_glossary(),
config=AuditaConfig.from_sources( config=config.validation_llm_config(),
env={"OPENROUTER_API_KEY": "test-key"},
overrides=ConfigOverrides(llm_concurrency=llm_concurrency),
),
run_spec=ModuleRunSpec( run_spec=ModuleRunSpec(
instance_name="homophones", instance_name="homophones",
module_key="homophones", module_key="homophones",
@@ -1190,3 +1197,157 @@ def test_llm_validators_process_batches_concurrently_and_preserve_proposal_order
(1, True), (1, True),
] ]
assert len(client.calls) == 2 assert len(client.calls) == 2
def test_validator_uses_validation_llm_config(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
)
]
client = FakeStructuredLLMClient(
[
{
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.97,
"reason": "ok",
}
]
}
]
)
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "primary-key"},
overrides=ConfigOverrides(
model="primary-model",
base_url="http://localhost:8000/v1",
max_retries=7,
llm_timeout_seconds=120,
validation_llm_api_key="validation-key",
validation_model="validation-model",
validation_base_url="http://localhost:9000/v1",
validation_max_retries=2,
validation_llm_timeout_seconds=240,
validation_llm_concurrency=3,
),
)
context = ValidationContext(
proposals=proposals,
transcript=transcript,
glossary=_glossary(),
config=config.validation_llm_config(),
run_spec=ModuleRunSpec(
instance_name="homophones",
module_key="homophones",
module=_Module("require_unique"),
),
run_dir=tmp_path,
llm_client=client,
)
SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(context)
validation_config = client.calls[0]["config"]
assert validation_config.api_key == "validation-key"
assert validation_config.model == "validation-model"
assert validation_config.base_url == "http://localhost:9000/v1"
assert validation_config.max_retries == 2
assert validation_config.llm_timeout_seconds == 240
assert validation_config.llm_concurrency == 3
def test_validator_uses_validation_llm_concurrency_override(tmp_path, monkeypatch):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "ChatGPT still can't do that with a dam."}
]
"""
)
proposals = [
CorrectionProposal(
proposal_index=0,
module_instance="homophones",
module_key="homophones",
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
),
CorrectionProposal(
proposal_index=1,
module_instance="homophones",
module_key="homophones",
id=2,
original_text="dam",
corrected_text="damn",
confidence=0.95,
),
]
client = CoordinatedStructuredLLMClient(
[
lambda **kwargs: {
"validations": [
{
"correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1,
"approved": True,
"confidence": 0.97,
"reason": "ok",
}
]
},
lambda **kwargs: {
"validations": [
{
"correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1,
"approved": True,
"confidence": 0.94,
"reason": "ok",
}
]
},
],
threading.Barrier(2, timeout=1.0),
)
def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message):
return [
TokenBatch(batch_index=0, items=list(items[:1]), token_count=1),
TokenBatch(batch_index=1, items=list(items[1:]), token_count=1),
]
monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items)
result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
_context(
proposals=proposals,
transcript=transcript,
llm_client=client,
tmp_path=tmp_path,
llm_concurrency=1,
validation_llm_concurrency=2,
)
)
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
(0, True),
(1, True),
]
assert len(client.calls) == 2

View File

@@ -31,6 +31,7 @@ class FakeStructuredLLMClient:
"stage_name": stage_name, "stage_name": stage_name,
"messages": list(messages), "messages": list(messages),
"response_model": response_model, "response_model": response_model,
"config": config,
} }
) )
payload = _pop_llm_response(self._responses, stage_name) payload = _pop_llm_response(self._responses, stage_name)
@@ -115,6 +116,63 @@ def test_glossary_module_propose_writes_diagnostics_and_returns_proposals_withou
assert "gestures" in prompt_text assert "gestures" in prompt_text
def test_module_propose_uses_primary_llm_config(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
]
"""
)
section = chunk_transcript(transcript, max_section_tokens=1000)[0]
module = GlossaryModule()
client = FakeStructuredLLMClient(
[
{
"corrections": [
{
"id": 1,
"original_text": "gestures",
"corrected_text": "Jesters",
"confidence": 0.95,
}
]
}
]
)
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "primary-key"},
overrides=ConfigOverrides(
model="primary-model",
base_url="http://localhost:8000/v1",
max_retries=7,
llm_timeout_seconds=120,
validation_llm_api_key="validation-key",
validation_model="validation-model",
validation_base_url="http://localhost:9000/v1",
validation_max_retries=2,
validation_llm_timeout_seconds=240,
validation_llm_concurrency=3,
),
)
context = ModuleContext(
run_spec=ModuleRunSpec(instance_name="glossary_primary", module_key="glossary", module=module),
glossary=_glossary(),
config=config,
run_dir=tmp_path,
llm_client=client,
)
module.propose(section, context)
proposal_config = client.calls[0]["config"]
assert proposal_config.api_key == "primary-key"
assert proposal_config.model == "primary-model"
assert proposal_config.base_url == "http://localhost:8000/v1"
assert proposal_config.max_retries == 7
assert proposal_config.llm_timeout_seconds == 120
def test_homophones_prompt_is_explicitly_scoped_to_spoken_form_corrections(): def test_homophones_prompt_is_explicitly_scoped_to_spoken_form_corrections():
transcript = parse_transcript_json( transcript = parse_transcript_json(
""" """

View File

@@ -23,6 +23,12 @@ def test_process_help_exposes_framework_flags(capsys):
assert "--llm-api-key" in output assert "--llm-api-key" in output
assert "--llm-concurrency" in output assert "--llm-concurrency" in output
assert "--llm-timeout-seconds" in output assert "--llm-timeout-seconds" in output
assert "--validation-llm-api-key" in output
assert "--validation-llm-concurrency" in output
assert "--validation-llm-timeout-seconds" in output
assert "--validation-model" in output
assert "--validation-base-url" in output
assert "--validation-max-retries" in output
assert "--target-sections" in output assert "--target-sections" in output
assert "--modules" in output assert "--modules" in output
assert "--model" in output assert "--model" in output
@@ -308,6 +314,82 @@ def test_cli_process_passes_llm_timeout_seconds_override_to_config(monkeypatch,
assert captured["llm_timeout_seconds"] == 900.0 assert captured["llm_timeout_seconds"] == 900.0
def test_cli_process_passes_validation_llm_overrides_to_config(monkeypatch, tmp_path):
captured = {}
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
]
"""
)
report = RunReport(
status="success",
config={"model": "m", "base_url": "b"},
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
pipeline=["grammar"],
modules=[],
applied_changes=[],
skipped_corrections=[],
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
work_dir_retention="auto",
work_dir_retained=False,
work_dir=None,
error=None,
)
result = ProcessResult(
transcript=transcript,
report=report,
run_dir=tmp_path / "run",
work_dir_retained=False,
)
def _fake_from_sources(*, overrides=None):
captured["validation_llm_api_key"] = overrides.validation_llm_api_key
captured["validation_llm_concurrency"] = overrides.validation_llm_concurrency
captured["validation_llm_timeout_seconds"] = overrides.validation_llm_timeout_seconds
captured["validation_model"] = overrides.validation_model
captured["validation_base_url"] = overrides.validation_base_url
captured["validation_max_retries"] = overrides.validation_max_retries
return object()
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--validation-llm-api-key",
"validator-key",
"--validation-llm-concurrency",
"4",
"--validation-llm-timeout-seconds",
"180",
"--validation-model",
"validator-model",
"--validation-base-url",
"http://localhost:9000/v1",
"--validation-max-retries",
"2",
]
)
assert exit_code == 0
assert captured == {
"validation_llm_api_key": "validator-key",
"validation_llm_concurrency": 4,
"validation_llm_timeout_seconds": 180.0,
"validation_model": "validator-model",
"validation_base_url": "http://localhost:9000/v1",
"validation_max_retries": 2,
}
def test_cli_process_passes_target_sections_override_to_config(monkeypatch, tmp_path): def test_cli_process_passes_target_sections_override_to_config(monkeypatch, tmp_path):
captured = {} captured = {}
transcript = parse_transcript_json( transcript = parse_transcript_json(

View File

@@ -24,6 +24,12 @@ def test_default_config_allows_missing_api_key():
assert config.api_key is None assert config.api_key is None
assert config.llm_concurrency == DEFAULT_LLM_CONCURRENCY assert config.llm_concurrency == DEFAULT_LLM_CONCURRENCY
assert config.llm_timeout_seconds == DEFAULT_LLM_TIMEOUT_SECONDS assert config.llm_timeout_seconds == DEFAULT_LLM_TIMEOUT_SECONDS
assert config.validation_llm_api_key is None
assert config.validation_llm_concurrency is None
assert config.validation_llm_timeout_seconds is None
assert config.validation_model is None
assert config.validation_base_url is None
assert config.validation_max_retries is None
assert config.target_sections is None assert config.target_sections is None
assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS
assert config.min_section_tokens == DEFAULT_MIN_SECTION_TOKENS assert config.min_section_tokens == DEFAULT_MIN_SECTION_TOKENS
@@ -84,6 +90,54 @@ def test_llm_timeout_seconds_env_is_parsed():
assert config.llm_timeout_seconds == 120.5 assert config.llm_timeout_seconds == 120.5
def test_validation_llm_concurrency_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_VALIDATION_LLM_CONCURRENCY": "2"},
overrides=ConfigOverrides(validation_llm_concurrency=4),
)
assert config.validation_llm_concurrency == 4
def test_validation_llm_concurrency_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_CONCURRENCY": "3"})
assert config.validation_llm_concurrency == 3
def test_validation_llm_timeout_seconds_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "120"},
overrides=ConfigOverrides(validation_llm_timeout_seconds=900.0),
)
assert config.validation_llm_timeout_seconds == 900.0
def test_validation_llm_timeout_seconds_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "120.5"})
assert config.validation_llm_timeout_seconds == 120.5
def test_validation_model_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MODEL": "validator-model"})
assert config.validation_model == "validator-model"
def test_validation_base_url_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_BASE_URL": "http://localhost:9000/v1"})
assert config.validation_base_url == "http://localhost:9000/v1"
def test_validation_max_retries_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_RETRIES": "7"})
assert config.validation_max_retries == 7
def test_target_sections_cli_override_takes_precedence(): def test_target_sections_cli_override_takes_precedence():
config = AuditaConfig.from_sources( config = AuditaConfig.from_sources(
env={"AUDITA_TARGET_SECTIONS": "2"}, env={"AUDITA_TARGET_SECTIONS": "2"},
@@ -143,6 +197,62 @@ def test_blank_llm_api_key_override_resolves_to_none():
assert config.api_key is None assert config.api_key is None
def test_validation_llm_api_key_env_is_read():
config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_API_KEY": "validation-key"})
assert config.validation_llm_api_key == "validation-key"
def test_effective_validation_fields_fall_back_to_primary_settings():
config = AuditaConfig.from_sources(
env={
"AUDITA_LLM_API_KEY": "primary-key",
"AUDITA_MODEL": "primary-model",
"AUDITA_BASE_URL": "http://localhost:8000/v1",
"AUDITA_LLM_TIMEOUT_SECONDS": "120",
"AUDITA_LLM_CONCURRENCY": "5",
"AUDITA_MAX_RETRIES": "9",
}
)
validation = config.validation_llm_config()
assert validation.api_key == "primary-key"
assert validation.model == "primary-model"
assert validation.base_url == "http://localhost:8000/v1"
assert validation.llm_timeout_seconds == 120.0
assert validation.llm_concurrency == 5
assert validation.max_retries == 9
def test_effective_validation_fields_use_overrides_when_set():
config = AuditaConfig.from_sources(
env={
"AUDITA_LLM_API_KEY": "primary-key",
"AUDITA_MODEL": "primary-model",
"AUDITA_BASE_URL": "http://localhost:8000/v1",
"AUDITA_LLM_TIMEOUT_SECONDS": "120",
"AUDITA_LLM_CONCURRENCY": "5",
"AUDITA_MAX_RETRIES": "9",
"AUDITA_VALIDATION_LLM_API_KEY": "validation-key",
"AUDITA_VALIDATION_MODEL": "validation-model",
"AUDITA_VALIDATION_BASE_URL": "http://localhost:9000/v1",
"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "240",
"AUDITA_VALIDATION_LLM_CONCURRENCY": "3",
"AUDITA_VALIDATION_MAX_RETRIES": "2",
}
)
validation = config.validation_llm_config()
assert validation.api_key == "validation-key"
assert validation.model == "validation-model"
assert validation.base_url == "http://localhost:9000/v1"
assert validation.llm_timeout_seconds == 240.0
assert validation.llm_concurrency == 3
assert validation.max_retries == 2
def test_module_key_overrides_take_precedence(): def test_module_key_overrides_take_precedence():
config = AuditaConfig.from_sources( config = AuditaConfig.from_sources(
env={"AUDITA_MODULES": "grammar"}, env={"AUDITA_MODULES": "grammar"},
@@ -175,6 +285,21 @@ def test_report_dict_includes_target_sections():
assert config.to_report_dict()["target_sections"] == 7 assert config.to_report_dict()["target_sections"] == 7
def test_report_dict_includes_effective_validation_llm_config():
config = AuditaConfig.from_sources(
env={
"AUDITA_LLM_API_KEY": "primary-key",
"AUDITA_VALIDATION_MODEL": "validation-model",
}
)
report = config.to_report_dict()
assert report["validation_model"] == "validation-model"
assert report["effective_validation_llm"]["api_key_configured"] is True
assert report["effective_validation_llm"]["model"] == "validation-model"
def test_threshold_overrides_take_precedence(): def test_threshold_overrides_take_precedence():
config = AuditaConfig.from_sources( config = AuditaConfig.from_sources(
env={ env={
@@ -236,6 +361,24 @@ def test_invalid_llm_timeout_seconds_is_rejected(value):
AuditaConfig.from_sources(env={"AUDITA_LLM_TIMEOUT_SECONDS": value}) AuditaConfig.from_sources(env={"AUDITA_LLM_TIMEOUT_SECONDS": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_validation_llm_concurrency_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_LLM_CONCURRENCY"):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_CONCURRENCY": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_validation_llm_timeout_seconds_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": value})
@pytest.mark.parametrize("value", ["-1", "many"])
def test_invalid_validation_max_retries_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_MAX_RETRIES"):
AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_RETRIES": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"]) @pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_target_sections_is_rejected(value): def test_invalid_target_sections_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_TARGET_SECTIONS"): with pytest.raises(AuditaConfigError, match="AUDITA_TARGET_SECTIONS"):

View File

@@ -24,6 +24,7 @@ class FakeStructuredLLMClient:
"stage_name": stage_name, "stage_name": stage_name,
"messages": list(messages), "messages": list(messages),
"response_model": response_model, "response_model": response_model,
"config": config,
} }
) )
response = _pop_llm_response(self._responses, stage_name) response = _pop_llm_response(self._responses, stage_name)
@@ -103,6 +104,85 @@ def test_process_transcript_runs_noop_framework(tmp_path):
] ]
def test_process_transcript_result_can_use_different_validation_llm_settings(tmp_path):
llm_client = FakeStructuredLLMClient(
{
"grammar:proposal": {
"corrections": [
{
"id": 1,
"original_text": "hello world",
"corrected_text": "Hello world.",
"confidence": 0.95,
}
]
},
"grammar:grammar_only_guard": {
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.98,
"reason": "ok",
}
]
},
"grammar:meaning_reversal_review": {
"validations": [
{
"correction_index": 0,
"approved": True,
"confidence": 0.99,
"reason": "ok",
}
]
},
}
)
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "primary-key"},
overrides=ConfigOverrides(
model="primary-model",
base_url="http://localhost:8000/v1",
max_retries=7,
llm_timeout_seconds=120,
validation_llm_api_key="validation-key",
validation_model="validation-model",
validation_base_url="http://localhost:9000/v1",
validation_max_retries=2,
validation_llm_timeout_seconds=240,
validation_llm_concurrency=3,
work_dir=tmp_path / "work",
work_dir_retention="always",
),
)
result = process_transcript_result(
parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
]
"""
),
_glossary(),
config,
module_keys=["grammar"],
llm_client=llm_client,
)
assert result.transcript[0].text == "Hello world."
calls_by_stage = {call["stage_name"]: call["config"] for call in llm_client.calls}
assert calls_by_stage["grammar:proposal"].model == "primary-model"
assert calls_by_stage["grammar:proposal"].base_url == "http://localhost:8000/v1"
assert calls_by_stage["grammar:proposal"].api_key == "primary-key"
assert calls_by_stage["grammar:grammar_only_guard"].model == "validation-model"
assert calls_by_stage["grammar:grammar_only_guard"].base_url == "http://localhost:9000/v1"
assert calls_by_stage["grammar:grammar_only_guard"].api_key == "validation-key"
assert calls_by_stage["grammar:grammar_only_guard"].max_retries == 2
assert calls_by_stage["grammar:grammar_only_guard"].llm_timeout_seconds == 240
def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(tmp_path): def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(tmp_path):
config = AuditaConfig.from_sources( config = AuditaConfig.from_sources(
env={}, env={},