diff --git a/README.md b/README.md index a380e7c..0181613 100644 --- a/README.md +++ b/README.md @@ -85,11 +85,17 @@ Useful configuration can be supplied by CLI flag or environment variable. CLI fl | --- | --- | --- | --- | | `AUDITA_MODULES` | `--modules` | `glossary,homophones,glossary,spoken_word,grammar` | Comma-separated logical module keys to run; CLI overrides the environment value | | `AUDITA_LLM_API_KEY` | `--llm-api-key` | unset | Preferred provider-neutral LLM API credential; CLI overrides both environment-key variants | +| `AUDITA_VALIDATION_LLM_API_KEY` | `--validation-llm-api-key` | unset | Validation-phase LLM API credential; defaults to the primary LLM API key | | `AUDITA_MODEL` | `--model` | `openrouter/google/gemma-4-31b-it` | LLM model name sent to the configured OpenAI-compatible endpoint | +| `AUDITA_VALIDATION_MODEL` | `--validation-model` | unset | Validation-phase LLM model; defaults to `AUDITA_MODEL` | | `AUDITA_BASE_URL` | `--base-url` | `https://openrouter.ai/api/v1` | OpenAI-compatible API base URL | +| `AUDITA_VALIDATION_BASE_URL` | `--validation-base-url` | unset | Validation-phase OpenAI-compatible API base URL; defaults to `AUDITA_BASE_URL` | | `AUDITA_LLM_TIMEOUT_SECONDS` | `--llm-timeout-seconds` | `600` | Per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint | +| `AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS` | `--validation-llm-timeout-seconds` | unset | Validation-phase per-request timeout in seconds; defaults to `AUDITA_LLM_TIMEOUT_SECONDS` | | `AUDITA_TARGET_SECTIONS` | `--target-sections` | unset | Exact number of contiguous proposal-stage transcript sections; errors if min/max token bounds cannot be satisfied | | `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses | +| `AUDITA_VALIDATION_MAX_RETRIES` | `--validation-max-retries` | unset | Validation-phase structured-output retries; defaults to `AUDITA_MAX_RETRIES` | +| `AUDITA_VALIDATION_LLM_CONCURRENCY` | `--validation-llm-concurrency` | unset | Validation-phase LLM concurrency; defaults to `AUDITA_LLM_CONCURRENCY` | | `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `8192` | Maximum estimated tokens per proposal-stage transcript section | | `AUDITA_MIN_SECTION_TOKENS` | `--min-section-tokens` | `2048` | Minimum estimated tokens per proposal-stage transcript section when balancing for concurrency | | `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation | @@ -105,6 +111,8 @@ Useful configuration can be supplied by CLI flag or environment variable. CLI fl Set `AUDITA_MODULES=grammar` to run only the grammar module by default, or override it per command with `--modules`. +Validation-phase LLM settings inherit from the primary `AUDITA_*` LLM settings by default. Set any of the `AUDITA_VALIDATION_*` values only when you want LLM-backed validators to use a different model, endpoint, credential, timeout, retry budget, or concurrency level. + OpenRouter remains the default out of the box: ```sh diff --git a/src/audita/cli.py b/src/audita/cli.py index c4b1f3d..a7795cc 100644 --- a/src/audita/cli.py +++ b/src/audita/cli.py @@ -34,6 +34,30 @@ def _build_parser() -> argparse.ArgumentParser: type=float, help="per-request timeout in seconds for LLM calls to the configured OpenAI-compatible endpoint", ) + process.add_argument( + "--validation-llm-api-key", + help="LLM API key for validation phases; defaults to the primary configured OpenAI-compatible endpoint key", + ) + process.add_argument( + "--validation-llm-concurrency", + type=int, + help="maximum concurrent LLM calls within validation phases; defaults to --llm-concurrency", + ) + process.add_argument( + "--validation-llm-timeout-seconds", + type=float, + help="per-request timeout in seconds for validation-phase LLM calls; defaults to --llm-timeout-seconds", + ) + process.add_argument("--validation-model", help="LLM model name for validation phases; defaults to --model") + process.add_argument( + "--validation-base-url", + help="OpenAI-compatible API base URL for validation phases; defaults to --base-url", + ) + process.add_argument( + "--validation-max-retries", + type=int, + help="maximum structured-output retries for validation phases; defaults to --max-retries", + ) process.add_argument( "--target-sections", type=int, @@ -108,6 +132,12 @@ def _process(args: argparse.Namespace) -> int: llm_api_key=args.llm_api_key, llm_concurrency=args.llm_concurrency, llm_timeout_seconds=args.llm_timeout_seconds, + validation_llm_api_key=args.validation_llm_api_key, + validation_llm_concurrency=args.validation_llm_concurrency, + validation_llm_timeout_seconds=args.validation_llm_timeout_seconds, + validation_model=args.validation_model, + validation_base_url=args.validation_base_url, + validation_max_retries=args.validation_max_retries, target_sections=args.target_sections, module_keys=args.modules, model=args.model, diff --git a/src/audita/core/config.py b/src/audita/core/config.py index 9021e48..7dc5219 100644 --- a/src/audita/core/config.py +++ b/src/audita/core/config.py @@ -1,6 +1,6 @@ import math import os -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path from typing import Mapping, Optional, Sequence, Tuple, Union @@ -33,6 +33,12 @@ class ConfigOverrides: llm_api_key: Optional[str] = None llm_concurrency: Optional[int] = None llm_timeout_seconds: Optional[float] = None + validation_llm_api_key: Optional[str] = None + validation_llm_concurrency: Optional[int] = None + validation_llm_timeout_seconds: Optional[float] = None + validation_model: Optional[str] = None + validation_base_url: Optional[str] = None + validation_max_retries: Optional[int] = None target_sections: Optional[int] = None module_keys: Optional[Union[str, Sequence[str]]] = None model: Optional[str] = None @@ -57,6 +63,12 @@ class AuditaConfig: api_key: Optional[str] = None llm_concurrency: int = DEFAULT_LLM_CONCURRENCY llm_timeout_seconds: float = DEFAULT_LLM_TIMEOUT_SECONDS + validation_llm_api_key: Optional[str] = None + validation_llm_concurrency: Optional[int] = None + validation_llm_timeout_seconds: Optional[float] = None + validation_model: Optional[str] = None + validation_base_url: Optional[str] = None + validation_max_retries: Optional[int] = None target_sections: Optional[int] = None module_keys: Tuple[str, ...] = DEFAULT_MODULE_KEYS model: str = DEFAULT_MODEL @@ -101,6 +113,33 @@ class AuditaConfig: DEFAULT_LLM_TIMEOUT_SECONDS, "AUDITA_LLM_TIMEOUT_SECONDS", ), + validation_llm_api_key=_select_optional_string_override( + selected.validation_llm_api_key, + source.get("AUDITA_VALIDATION_LLM_API_KEY"), + ), + validation_llm_concurrency=_select_optional_int( + selected.validation_llm_concurrency, + source.get("AUDITA_VALIDATION_LLM_CONCURRENCY"), + "AUDITA_VALIDATION_LLM_CONCURRENCY", + ), + validation_llm_timeout_seconds=_select_optional_float( + selected.validation_llm_timeout_seconds, + source.get("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"), + "AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS", + ), + validation_model=_select_optional_string_override( + selected.validation_model, + source.get("AUDITA_VALIDATION_MODEL"), + ), + validation_base_url=_select_optional_string_override( + selected.validation_base_url, + source.get("AUDITA_VALIDATION_BASE_URL"), + ), + validation_max_retries=_select_optional_int( + selected.validation_max_retries, + source.get("AUDITA_VALIDATION_MAX_RETRIES"), + "AUDITA_VALIDATION_MAX_RETRIES", + ), target_sections=_select_optional_int( selected.target_sections, source.get("AUDITA_TARGET_SECTIONS"), @@ -200,12 +239,24 @@ class AuditaConfig: raise AuditaConfigError("AUDITA_LLM_TIMEOUT_SECONDS must be finite.") if self.llm_timeout_seconds <= 0: raise AuditaConfigError("AUDITA_LLM_TIMEOUT_SECONDS must be greater than zero.") + if self.validation_llm_concurrency is not None and self.validation_llm_concurrency <= 0: + raise AuditaConfigError("AUDITA_VALIDATION_LLM_CONCURRENCY must be greater than zero.") + if self.validation_llm_timeout_seconds is not None and not math.isfinite(self.validation_llm_timeout_seconds): + raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be finite.") + if self.validation_llm_timeout_seconds is not None and self.validation_llm_timeout_seconds <= 0: + raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be greater than zero.") + if self.validation_max_retries is not None and self.validation_max_retries < 0: + raise AuditaConfigError("AUDITA_VALIDATION_MAX_RETRIES must be greater than or equal to zero.") if self.target_sections is not None and self.target_sections <= 0: raise AuditaConfigError("AUDITA_TARGET_SECTIONS must be greater than zero.") if not self.model.strip(): raise AuditaConfigError("AUDITA_MODEL must not be empty.") if not self.base_url.strip(): raise AuditaConfigError("AUDITA_BASE_URL must not be empty.") + if self.validation_model is not None and not self.validation_model.strip(): + raise AuditaConfigError("AUDITA_VALIDATION_MODEL must not be empty.") + if self.validation_base_url is not None and not self.validation_base_url.strip(): + raise AuditaConfigError("AUDITA_VALIDATION_BASE_URL must not be empty.") if self.max_retries < 0: raise AuditaConfigError("AUDITA_MAX_RETRIES must be greater than or equal to zero.") if self.max_section_tokens <= 0: @@ -244,17 +295,79 @@ class AuditaConfig: raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS must be greater than zero.") if self.work_dir_retention not in ("auto", "always", "never"): raise AuditaConfigError("AUDITA_WORK_DIR_RETENTION must be one of auto, always, or never.") + validation_config = self.validation_llm_config() + if validation_config.llm_concurrency <= 0: + raise AuditaConfigError("AUDITA_VALIDATION_LLM_CONCURRENCY must be greater than zero.") + if not math.isfinite(validation_config.llm_timeout_seconds): + raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be finite.") + if validation_config.llm_timeout_seconds <= 0: + raise AuditaConfigError("AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS must be greater than zero.") + if not validation_config.model.strip(): + raise AuditaConfigError("AUDITA_VALIDATION_MODEL must not be empty.") + if not validation_config.base_url.strip(): + raise AuditaConfigError("AUDITA_VALIDATION_BASE_URL must not be empty.") + if validation_config.max_retries < 0: + raise AuditaConfigError("AUDITA_VALIDATION_MAX_RETRIES must be greater than or equal to zero.") + + def proposal_llm_config(self) -> "AuditaConfig": + return self + + def effective_validation_llm_api_key(self) -> Optional[str]: + return self.validation_llm_api_key if self.validation_llm_api_key is not None else self.api_key + + def effective_validation_llm_concurrency(self) -> int: + return self.validation_llm_concurrency if self.validation_llm_concurrency is not None else self.llm_concurrency + + def effective_validation_llm_timeout_seconds(self) -> float: + if self.validation_llm_timeout_seconds is not None: + return self.validation_llm_timeout_seconds + return self.llm_timeout_seconds + + def effective_validation_model(self) -> str: + return self.validation_model if self.validation_model is not None else self.model + + def effective_validation_base_url(self) -> str: + return self.validation_base_url if self.validation_base_url is not None else self.base_url + + def effective_validation_max_retries(self) -> int: + return self.validation_max_retries if self.validation_max_retries is not None else self.max_retries + + def validation_llm_config(self) -> "AuditaConfig": + return replace( + self, + api_key=self.effective_validation_llm_api_key(), + llm_concurrency=self.effective_validation_llm_concurrency(), + llm_timeout_seconds=self.effective_validation_llm_timeout_seconds(), + model=self.effective_validation_model(), + base_url=self.effective_validation_base_url(), + max_retries=self.effective_validation_max_retries(), + ) def to_report_dict(self) -> dict: + effective_validation_config = self.validation_llm_config() return { "api_key_configured": bool(self.api_key), "llm_concurrency": self.llm_concurrency, "llm_timeout_seconds": self.llm_timeout_seconds, + "validation_llm_api_key_configured": bool(self.validation_llm_api_key), + "validation_llm_concurrency": self.validation_llm_concurrency, + "validation_llm_timeout_seconds": self.validation_llm_timeout_seconds, + "validation_model": self.validation_model, + "validation_base_url": self.validation_base_url, + "validation_max_retries": self.validation_max_retries, "target_sections": self.target_sections, "module_keys": list(self.module_keys), "model": self.model, "base_url": self.base_url, "max_retries": self.max_retries, + "effective_validation_llm": { + "api_key_configured": bool(effective_validation_config.api_key), + "llm_concurrency": effective_validation_config.llm_concurrency, + "llm_timeout_seconds": effective_validation_config.llm_timeout_seconds, + "model": effective_validation_config.model, + "base_url": effective_validation_config.base_url, + "max_retries": effective_validation_config.max_retries, + }, "max_section_tokens": self.max_section_tokens, "min_section_tokens": self.min_section_tokens, "glossary_confidence_threshold": self.glossary_confidence_threshold, @@ -276,6 +389,12 @@ def _select_optional_string(value: Optional[str]) -> Optional[str]: return stripped or None +def _select_optional_string_override(cli_value: Optional[str], env_value: Optional[str]) -> Optional[str]: + if cli_value is not None: + return _select_optional_string(cli_value) + return _select_optional_string(env_value) + + def _select_api_key( cli_value: Optional[str], generic_env_value: Optional[str], @@ -311,6 +430,17 @@ def _select_optional_int(cli_value: Optional[int], env_value: Optional[str], nam raise AuditaConfigError(f"{name} must be an integer.") from exc +def _select_optional_float(cli_value: Optional[float], env_value: Optional[str], name: str) -> Optional[float]: + if cli_value is not None: + return cli_value + if env_value is None: + return None + try: + return float(env_value) + except ValueError as exc: + raise AuditaConfigError(f"{name} must be a number.") from exc + + def _select_float(cli_value: Optional[float], env_value: Optional[str], default: float, name: str) -> float: if cli_value is not None: return cli_value diff --git a/src/audita/framework/llm.py b/src/audita/framework/llm.py index 02b28a8..39632ca 100644 --- a/src/audita/framework/llm.py +++ b/src/audita/framework/llm.py @@ -24,7 +24,8 @@ class OpenAICompatibleStructuredLLMClient: if not config.api_key: raise AuditaLLMError( f"Structured LLM stage '{stage_name}' requires LLM API credentials to be configured " - "via --llm-api-key, AUDITA_LLM_API_KEY, or OPENROUTER_API_KEY." + "via --llm-api-key, --validation-llm-api-key, AUDITA_LLM_API_KEY, " + "AUDITA_VALIDATION_LLM_API_KEY, or OPENROUTER_API_KEY." ) client = self._get_client(config) diff --git a/src/audita/framework/models.py b/src/audita/framework/models.py index ffd9699..c00593b 100644 --- a/src/audita/framework/models.py +++ b/src/audita/framework/models.py @@ -50,6 +50,7 @@ class ModuleContext: run_dir: Path llm_client: Optional["StructuredLLMClient"] = None llm_scheduler: Optional[ModuleLLMScheduler] = None + validation_llm_scheduler: Optional[ModuleLLMScheduler] = None class StructuredLLMClient(Protocol): diff --git a/src/audita/framework/proposal_generation.py b/src/audita/framework/proposal_generation.py index d3b37b6..0eafa43 100644 --- a/src/audita/framework/proposal_generation.py +++ b/src/audita/framework/proposal_generation.py @@ -69,7 +69,7 @@ def generate_llm_correction_proposals( stage_name=f"{context.run_spec.instance_name}:proposal", messages=messages, response_model=StructuredCorrectionSet, - config=context.config, + config=context.config.proposal_llm_config(), ) ) else: @@ -77,7 +77,7 @@ def generate_llm_correction_proposals( stage_name=f"{context.run_spec.instance_name}:proposal", messages=messages, response_model=StructuredCorrectionSet, - config=context.config, + config=context.config.proposal_llm_config(), ) response_path = context.run_dir / f"corrections-{section.section_index:04d}.json" response_path.write_text(response.model_dump_json(indent=2) + "\n", encoding="utf-8") diff --git a/src/audita/framework/runner.py b/src/audita/framework/runner.py index ce4bb7a..78aec3f 100644 --- a/src/audita/framework/runner.py +++ b/src/audita/framework/runner.py @@ -87,6 +87,7 @@ class PipelineRunner: run_dir=module_dir, llm_client=llm_client, llm_scheduler=ModuleLLMScheduler(config.llm_concurrency), + validation_llm_scheduler=ModuleLLMScheduler(config.effective_validation_llm_concurrency()), ) sections = chunk_transcript( working, @@ -296,11 +297,11 @@ def _build_validation_context( proposals=proposals, transcript=working, glossary=context.glossary, - config=context.config, + config=context.config.validation_llm_config(), run_spec=context.run_spec, run_dir=context.run_dir, llm_client=llm_client, - llm_scheduler=context.llm_scheduler, + llm_scheduler=context.validation_llm_scheduler, ) @@ -358,7 +359,8 @@ def _run_parallel_llm_validator_group( context=context, llm_client=llm_client, ) - max_workers = context.llm_scheduler.max_concurrency if context.llm_scheduler is not None else context.config.llm_concurrency + scheduler = context.validation_llm_scheduler + max_workers = scheduler.max_concurrency if scheduler is not None else context.config.effective_validation_llm_concurrency() if max_workers == 1 or len(validators) <= 1: results = [validator.validate(validation_context) for validator in validators] else: diff --git a/tests/test_framework_llm.py b/tests/test_framework_llm.py index 4ad6ec7..870daf2 100644 --- a/tests/test_framework_llm.py +++ b/tests/test_framework_llm.py @@ -19,6 +19,12 @@ def _config(**overrides): "api_key": "test-key", "llm_concurrency": base.llm_concurrency, "llm_timeout_seconds": base.llm_timeout_seconds, + "validation_llm_api_key": base.validation_llm_api_key, + "validation_llm_concurrency": base.validation_llm_concurrency, + "validation_llm_timeout_seconds": base.validation_llm_timeout_seconds, + "validation_model": base.validation_model, + "validation_base_url": base.validation_base_url, + "validation_max_retries": base.validation_max_retries, "module_keys": base.module_keys, "model": base.model, "base_url": base.base_url, @@ -193,7 +199,7 @@ def test_client_cache_identity_uses_timeout(monkeypatch): def test_missing_api_key_error_is_provider_neutral(): client = OpenAICompatibleStructuredLLMClient() - with pytest.raises(AuditaLLMError, match="AUDITA_LLM_API_KEY"): + with pytest.raises(AuditaLLMError, match="AUDITA_VALIDATION_LLM_API_KEY"): client.run_structured( stage_name="test-stage", messages=[{"role": "user", "content": "Hello"}], diff --git a/tests/test_llm_validators.py b/tests/test_llm_validators.py index 3533f24..aeb9eb7 100644 --- a/tests/test_llm_validators.py +++ b/tests/test_llm_validators.py @@ -40,6 +40,7 @@ class FakeStructuredLLMClient: "stage_name": stage_name, "messages": list(messages), "response_model": response_model, + "config": config, } ) if not self._responses: @@ -62,6 +63,7 @@ class CoordinatedStructuredLLMClient: "stage_name": stage_name, "messages": list(messages), "response_model": response_model, + "config": config, } ) if not self._responses: @@ -97,15 +99,20 @@ def _context( tmp_path, replacement_policy="require_unique", llm_concurrency=1, + validation_llm_concurrency=None, ): + config = AuditaConfig.from_sources( + env={"OPENROUTER_API_KEY": "test-key"}, + overrides=ConfigOverrides( + llm_concurrency=llm_concurrency, + validation_llm_concurrency=validation_llm_concurrency, + ), + ) return ValidationContext( proposals=proposals, transcript=transcript, glossary=_glossary(), - config=AuditaConfig.from_sources( - env={"OPENROUTER_API_KEY": "test-key"}, - overrides=ConfigOverrides(llm_concurrency=llm_concurrency), - ), + config=config.validation_llm_config(), run_spec=ModuleRunSpec( instance_name="homophones", module_key="homophones", @@ -1190,3 +1197,157 @@ def test_llm_validators_process_batches_concurrently_and_preserve_proposal_order (1, True), ] assert len(client.calls) == 2 + + +def test_validator_uses_validation_llm_config(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} + ] + """ + ) + proposals = [ + CorrectionProposal( + proposal_index=0, + module_instance="homophones", + module_key="homophones", + id=1, + original_text="gestures", + corrected_text="Jesters", + confidence=0.95, + ) + ] + client = FakeStructuredLLMClient( + [ + { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.97, + "reason": "ok", + } + ] + } + ] + ) + config = AuditaConfig.from_sources( + env={"OPENROUTER_API_KEY": "primary-key"}, + overrides=ConfigOverrides( + model="primary-model", + base_url="http://localhost:8000/v1", + max_retries=7, + llm_timeout_seconds=120, + validation_llm_api_key="validation-key", + validation_model="validation-model", + validation_base_url="http://localhost:9000/v1", + validation_max_retries=2, + validation_llm_timeout_seconds=240, + validation_llm_concurrency=3, + ), + ) + context = ValidationContext( + proposals=proposals, + transcript=transcript, + glossary=_glossary(), + config=config.validation_llm_config(), + run_spec=ModuleRunSpec( + instance_name="homophones", + module_key="homophones", + module=_Module("require_unique"), + ), + run_dir=tmp_path, + llm_client=client, + ) + + SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(context) + + validation_config = client.calls[0]["config"] + assert validation_config.api_key == "validation-key" + assert validation_config.model == "validation-model" + assert validation_config.base_url == "http://localhost:9000/v1" + assert validation_config.max_retries == 2 + assert validation_config.llm_timeout_seconds == 240 + assert validation_config.llm_concurrency == 3 + + +def test_validator_uses_validation_llm_concurrency_override(tmp_path, monkeypatch): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}, + {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "ChatGPT still can't do that with a dam."} + ] + """ + ) + proposals = [ + CorrectionProposal( + proposal_index=0, + module_instance="homophones", + module_key="homophones", + id=1, + original_text="gestures", + corrected_text="Jesters", + confidence=0.95, + ), + CorrectionProposal( + proposal_index=1, + module_instance="homophones", + module_key="homophones", + id=2, + original_text="dam", + corrected_text="damn", + confidence=0.95, + ), + ] + client = CoordinatedStructuredLLMClient( + [ + lambda **kwargs: { + "validations": [ + { + "correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1, + "approved": True, + "confidence": 0.97, + "reason": "ok", + } + ] + }, + lambda **kwargs: { + "validations": [ + { + "correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1, + "approved": True, + "confidence": 0.94, + "reason": "ok", + } + ] + }, + ], + threading.Barrier(2, timeout=1.0), + ) + + def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message): + return [ + TokenBatch(batch_index=0, items=list(items[:1]), token_count=1), + TokenBatch(batch_index=1, items=list(items[1:]), token_count=1), + ] + + monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items) + + result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate( + _context( + proposals=proposals, + transcript=transcript, + llm_client=client, + tmp_path=tmp_path, + llm_concurrency=1, + validation_llm_concurrency=2, + ) + ) + + assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [ + (0, True), + (1, True), + ] + assert len(client.calls) == 2 diff --git a/tests/test_module_proposals.py b/tests/test_module_proposals.py index dfe5d81..a875723 100644 --- a/tests/test_module_proposals.py +++ b/tests/test_module_proposals.py @@ -31,6 +31,7 @@ class FakeStructuredLLMClient: "stage_name": stage_name, "messages": list(messages), "response_model": response_model, + "config": config, } ) payload = _pop_llm_response(self._responses, stage_name) @@ -115,6 +116,63 @@ def test_glossary_module_propose_writes_diagnostics_and_returns_proposals_withou assert "gestures" in prompt_text +def test_module_propose_uses_primary_llm_config(tmp_path): + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."} + ] + """ + ) + section = chunk_transcript(transcript, max_section_tokens=1000)[0] + module = GlossaryModule() + client = FakeStructuredLLMClient( + [ + { + "corrections": [ + { + "id": 1, + "original_text": "gestures", + "corrected_text": "Jesters", + "confidence": 0.95, + } + ] + } + ] + ) + config = AuditaConfig.from_sources( + env={"OPENROUTER_API_KEY": "primary-key"}, + overrides=ConfigOverrides( + model="primary-model", + base_url="http://localhost:8000/v1", + max_retries=7, + llm_timeout_seconds=120, + validation_llm_api_key="validation-key", + validation_model="validation-model", + validation_base_url="http://localhost:9000/v1", + validation_max_retries=2, + validation_llm_timeout_seconds=240, + validation_llm_concurrency=3, + ), + ) + context = ModuleContext( + run_spec=ModuleRunSpec(instance_name="glossary_primary", module_key="glossary", module=module), + glossary=_glossary(), + config=config, + run_dir=tmp_path, + llm_client=client, + ) + + module.propose(section, context) + + proposal_config = client.calls[0]["config"] + assert proposal_config.api_key == "primary-key" + assert proposal_config.model == "primary-model" + assert proposal_config.base_url == "http://localhost:8000/v1" + assert proposal_config.max_retries == 7 + assert proposal_config.llm_timeout_seconds == 120 + + def test_homophones_prompt_is_explicitly_scoped_to_spoken_form_corrections(): transcript = parse_transcript_json( """ diff --git a/tests/test_new_cli.py b/tests/test_new_cli.py index 8cb84e2..0b06900 100644 --- a/tests/test_new_cli.py +++ b/tests/test_new_cli.py @@ -23,6 +23,12 @@ def test_process_help_exposes_framework_flags(capsys): assert "--llm-api-key" in output assert "--llm-concurrency" in output assert "--llm-timeout-seconds" in output + assert "--validation-llm-api-key" in output + assert "--validation-llm-concurrency" in output + assert "--validation-llm-timeout-seconds" in output + assert "--validation-model" in output + assert "--validation-base-url" in output + assert "--validation-max-retries" in output assert "--target-sections" in output assert "--modules" in output assert "--model" in output @@ -308,6 +314,82 @@ def test_cli_process_passes_llm_timeout_seconds_override_to_config(monkeypatch, assert captured["llm_timeout_seconds"] == 900.0 +def test_cli_process_passes_validation_llm_overrides_to_config(monkeypatch, tmp_path): + captured = {} + transcript = parse_transcript_json( + """ + [ + {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."} + ] + """ + ) + report = RunReport( + status="success", + config={"model": "m", "base_url": "b"}, + normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0}, + pipeline=["grammar"], + modules=[], + applied_changes=[], + skipped_corrections=[], + totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0}, + work_dir_retention="auto", + work_dir_retained=False, + work_dir=None, + error=None, + ) + result = ProcessResult( + transcript=transcript, + report=report, + run_dir=tmp_path / "run", + work_dir_retained=False, + ) + + def _fake_from_sources(*, overrides=None): + captured["validation_llm_api_key"] = overrides.validation_llm_api_key + captured["validation_llm_concurrency"] = overrides.validation_llm_concurrency + captured["validation_llm_timeout_seconds"] = overrides.validation_llm_timeout_seconds + captured["validation_model"] = overrides.validation_model + captured["validation_base_url"] = overrides.validation_base_url + captured["validation_max_retries"] = overrides.validation_max_retries + return object() + + monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources) + monkeypatch.setattr("audita.cli.load_transcript", lambda path: []) + monkeypatch.setattr("audita.cli.load_glossary", lambda path: object()) + monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result) + + exit_code = main( + [ + "process", + "transcript.json", + "--glossary", + "glossary.yaml", + "--validation-llm-api-key", + "validator-key", + "--validation-llm-concurrency", + "4", + "--validation-llm-timeout-seconds", + "180", + "--validation-model", + "validator-model", + "--validation-base-url", + "http://localhost:9000/v1", + "--validation-max-retries", + "2", + ] + ) + + assert exit_code == 0 + assert captured == { + "validation_llm_api_key": "validator-key", + "validation_llm_concurrency": 4, + "validation_llm_timeout_seconds": 180.0, + "validation_model": "validator-model", + "validation_base_url": "http://localhost:9000/v1", + "validation_max_retries": 2, + } + + def test_cli_process_passes_target_sections_override_to_config(monkeypatch, tmp_path): captured = {} transcript = parse_transcript_json( diff --git a/tests/test_new_config.py b/tests/test_new_config.py index 74c3f38..c758e2b 100644 --- a/tests/test_new_config.py +++ b/tests/test_new_config.py @@ -24,6 +24,12 @@ def test_default_config_allows_missing_api_key(): assert config.api_key is None assert config.llm_concurrency == DEFAULT_LLM_CONCURRENCY assert config.llm_timeout_seconds == DEFAULT_LLM_TIMEOUT_SECONDS + assert config.validation_llm_api_key is None + assert config.validation_llm_concurrency is None + assert config.validation_llm_timeout_seconds is None + assert config.validation_model is None + assert config.validation_base_url is None + assert config.validation_max_retries is None assert config.target_sections is None assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS assert config.min_section_tokens == DEFAULT_MIN_SECTION_TOKENS @@ -84,6 +90,54 @@ def test_llm_timeout_seconds_env_is_parsed(): assert config.llm_timeout_seconds == 120.5 +def test_validation_llm_concurrency_cli_override_takes_precedence(): + config = AuditaConfig.from_sources( + env={"AUDITA_VALIDATION_LLM_CONCURRENCY": "2"}, + overrides=ConfigOverrides(validation_llm_concurrency=4), + ) + + assert config.validation_llm_concurrency == 4 + + +def test_validation_llm_concurrency_env_is_parsed(): + config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_CONCURRENCY": "3"}) + + assert config.validation_llm_concurrency == 3 + + +def test_validation_llm_timeout_seconds_cli_override_takes_precedence(): + config = AuditaConfig.from_sources( + env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "120"}, + overrides=ConfigOverrides(validation_llm_timeout_seconds=900.0), + ) + + assert config.validation_llm_timeout_seconds == 900.0 + + +def test_validation_llm_timeout_seconds_env_is_parsed(): + config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "120.5"}) + + assert config.validation_llm_timeout_seconds == 120.5 + + +def test_validation_model_env_is_parsed(): + config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MODEL": "validator-model"}) + + assert config.validation_model == "validator-model" + + +def test_validation_base_url_env_is_parsed(): + config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_BASE_URL": "http://localhost:9000/v1"}) + + assert config.validation_base_url == "http://localhost:9000/v1" + + +def test_validation_max_retries_env_is_parsed(): + config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_RETRIES": "7"}) + + assert config.validation_max_retries == 7 + + def test_target_sections_cli_override_takes_precedence(): config = AuditaConfig.from_sources( env={"AUDITA_TARGET_SECTIONS": "2"}, @@ -143,6 +197,62 @@ def test_blank_llm_api_key_override_resolves_to_none(): assert config.api_key is None +def test_validation_llm_api_key_env_is_read(): + config = AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_API_KEY": "validation-key"}) + + assert config.validation_llm_api_key == "validation-key" + + +def test_effective_validation_fields_fall_back_to_primary_settings(): + config = AuditaConfig.from_sources( + env={ + "AUDITA_LLM_API_KEY": "primary-key", + "AUDITA_MODEL": "primary-model", + "AUDITA_BASE_URL": "http://localhost:8000/v1", + "AUDITA_LLM_TIMEOUT_SECONDS": "120", + "AUDITA_LLM_CONCURRENCY": "5", + "AUDITA_MAX_RETRIES": "9", + } + ) + + validation = config.validation_llm_config() + + assert validation.api_key == "primary-key" + assert validation.model == "primary-model" + assert validation.base_url == "http://localhost:8000/v1" + assert validation.llm_timeout_seconds == 120.0 + assert validation.llm_concurrency == 5 + assert validation.max_retries == 9 + + +def test_effective_validation_fields_use_overrides_when_set(): + config = AuditaConfig.from_sources( + env={ + "AUDITA_LLM_API_KEY": "primary-key", + "AUDITA_MODEL": "primary-model", + "AUDITA_BASE_URL": "http://localhost:8000/v1", + "AUDITA_LLM_TIMEOUT_SECONDS": "120", + "AUDITA_LLM_CONCURRENCY": "5", + "AUDITA_MAX_RETRIES": "9", + "AUDITA_VALIDATION_LLM_API_KEY": "validation-key", + "AUDITA_VALIDATION_MODEL": "validation-model", + "AUDITA_VALIDATION_BASE_URL": "http://localhost:9000/v1", + "AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "240", + "AUDITA_VALIDATION_LLM_CONCURRENCY": "3", + "AUDITA_VALIDATION_MAX_RETRIES": "2", + } + ) + + validation = config.validation_llm_config() + + assert validation.api_key == "validation-key" + assert validation.model == "validation-model" + assert validation.base_url == "http://localhost:9000/v1" + assert validation.llm_timeout_seconds == 240.0 + assert validation.llm_concurrency == 3 + assert validation.max_retries == 2 + + def test_module_key_overrides_take_precedence(): config = AuditaConfig.from_sources( env={"AUDITA_MODULES": "grammar"}, @@ -175,6 +285,21 @@ def test_report_dict_includes_target_sections(): assert config.to_report_dict()["target_sections"] == 7 +def test_report_dict_includes_effective_validation_llm_config(): + config = AuditaConfig.from_sources( + env={ + "AUDITA_LLM_API_KEY": "primary-key", + "AUDITA_VALIDATION_MODEL": "validation-model", + } + ) + + report = config.to_report_dict() + + assert report["validation_model"] == "validation-model" + assert report["effective_validation_llm"]["api_key_configured"] is True + assert report["effective_validation_llm"]["model"] == "validation-model" + + def test_threshold_overrides_take_precedence(): config = AuditaConfig.from_sources( env={ @@ -236,6 +361,24 @@ def test_invalid_llm_timeout_seconds_is_rejected(value): AuditaConfig.from_sources(env={"AUDITA_LLM_TIMEOUT_SECONDS": value}) +@pytest.mark.parametrize("value", ["0", "-1", "many"]) +def test_invalid_validation_llm_concurrency_is_rejected(value): + with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_LLM_CONCURRENCY"): + AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_CONCURRENCY": value}) + + +@pytest.mark.parametrize("value", ["0", "-1", "many"]) +def test_invalid_validation_llm_timeout_seconds_is_rejected(value): + with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS"): + AuditaConfig.from_sources(env={"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": value}) + + +@pytest.mark.parametrize("value", ["-1", "many"]) +def test_invalid_validation_max_retries_is_rejected(value): + with pytest.raises(AuditaConfigError, match="AUDITA_VALIDATION_MAX_RETRIES"): + AuditaConfig.from_sources(env={"AUDITA_VALIDATION_MAX_RETRIES": value}) + + @pytest.mark.parametrize("value", ["0", "-1", "many"]) def test_invalid_target_sections_is_rejected(value): with pytest.raises(AuditaConfigError, match="AUDITA_TARGET_SECTIONS"): diff --git a/tests/test_new_pipeline.py b/tests/test_new_pipeline.py index 5d20777..c56ac9a 100644 --- a/tests/test_new_pipeline.py +++ b/tests/test_new_pipeline.py @@ -24,6 +24,7 @@ class FakeStructuredLLMClient: "stage_name": stage_name, "messages": list(messages), "response_model": response_model, + "config": config, } ) response = _pop_llm_response(self._responses, stage_name) @@ -103,6 +104,85 @@ def test_process_transcript_runs_noop_framework(tmp_path): ] +def test_process_transcript_result_can_use_different_validation_llm_settings(tmp_path): + llm_client = FakeStructuredLLMClient( + { + "grammar:proposal": { + "corrections": [ + { + "id": 1, + "original_text": "hello world", + "corrected_text": "Hello world.", + "confidence": 0.95, + } + ] + }, + "grammar:grammar_only_guard": { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.98, + "reason": "ok", + } + ] + }, + "grammar:meaning_reversal_review": { + "validations": [ + { + "correction_index": 0, + "approved": True, + "confidence": 0.99, + "reason": "ok", + } + ] + }, + } + ) + config = AuditaConfig.from_sources( + env={"OPENROUTER_API_KEY": "primary-key"}, + overrides=ConfigOverrides( + model="primary-model", + base_url="http://localhost:8000/v1", + max_retries=7, + llm_timeout_seconds=120, + validation_llm_api_key="validation-key", + validation_model="validation-model", + validation_base_url="http://localhost:9000/v1", + validation_max_retries=2, + validation_llm_timeout_seconds=240, + validation_llm_concurrency=3, + work_dir=tmp_path / "work", + work_dir_retention="always", + ), + ) + + result = process_transcript_result( + parse_source_transcript_json( + """ + [ + {"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"} + ] + """ + ), + _glossary(), + config, + module_keys=["grammar"], + llm_client=llm_client, + ) + + assert result.transcript[0].text == "Hello world." + calls_by_stage = {call["stage_name"]: call["config"] for call in llm_client.calls} + assert calls_by_stage["grammar:proposal"].model == "primary-model" + assert calls_by_stage["grammar:proposal"].base_url == "http://localhost:8000/v1" + assert calls_by_stage["grammar:proposal"].api_key == "primary-key" + assert calls_by_stage["grammar:grammar_only_guard"].model == "validation-model" + assert calls_by_stage["grammar:grammar_only_guard"].base_url == "http://localhost:9000/v1" + assert calls_by_stage["grammar:grammar_only_guard"].api_key == "validation-key" + assert calls_by_stage["grammar:grammar_only_guard"].max_retries == 2 + assert calls_by_stage["grammar:grammar_only_guard"].llm_timeout_seconds == 240 + + def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(tmp_path): config = AuditaConfig.from_sources( env={},