Implemented a second LLM stage for grammatical review

This commit is contained in:
2026-04-21 15:42:09 -05:00
parent ca01e46d77
commit 445329de81
13 changed files with 620 additions and 129 deletions

View File

@@ -3,18 +3,29 @@ from pathlib import Path
import pytest
from audita.config import AuditaConfig, ConfigOverrides
from audita.config import DEFAULT_CONFIDENCE_THRESHOLD, DEFAULT_GLOSSARY_MAX_LLM_PASSES, DEFAULT_MAX_RETRIES, DEFAULT_MAX_SECTION_TOKENS, DEFAULT_WORK_DIR
from audita.config import (
DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD,
DEFAULT_GLOSSARY_MAX_LLM_PASSES,
DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD,
DEFAULT_GRAMMAR_MAX_LLM_PASSES,
DEFAULT_MAX_RETRIES,
DEFAULT_MAX_SECTION_TOKENS,
DEFAULT_WORK_DIR,
)
from audita.errors import AuditaConfigError
def test_config_uses_defaults_with_api_key():
config = AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "key"})
assert config.confidence_threshold == DEFAULT_CONFIDENCE_THRESHOLD
assert config.confidence_threshold == 0.6
assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
assert config.glossary_confidence_threshold == 0.6
assert config.grammar_confidence_threshold == DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD
assert config.grammar_confidence_threshold == 0.6
assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS
assert config.max_retries == DEFAULT_MAX_RETRIES
assert config.glossary_max_llm_passes == DEFAULT_GLOSSARY_MAX_LLM_PASSES
assert config.grammar_max_llm_passes == DEFAULT_GRAMMAR_MAX_LLM_PASSES
assert config.work_dir == Path(DEFAULT_WORK_DIR)
@@ -23,17 +34,21 @@ def test_config_env_overrides_defaults():
env={
"OPENROUTER_API_KEY": "key",
"AUDITA_MAX_SECTION_TOKENS": "42",
"AUDITA_CONFIDENCE_THRESHOLD": "0.9",
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.9",
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.7",
"AUDITA_MAX_RETRIES": "5",
"AUDITA_GLOSSARY_MAX_LLM_PASSES": "7",
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "4",
"AUDITA_WORK_DIR": "/tmp/custom-audita",
}
)
assert config.max_section_tokens == 42
assert config.confidence_threshold == 0.9
assert config.glossary_confidence_threshold == 0.9
assert config.grammar_confidence_threshold == 0.7
assert config.max_retries == 5
assert config.glossary_max_llm_passes == 7
assert config.grammar_max_llm_passes == 4
assert config.work_dir == Path("/tmp/custom-audita")
@@ -44,21 +59,26 @@ def test_config_cli_overrides_env():
"AUDITA_MAX_SECTION_TOKENS": "42",
"AUDITA_MAX_RETRIES": "5",
"AUDITA_GLOSSARY_MAX_LLM_PASSES": "7",
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "6",
"AUDITA_WORK_DIR": "/tmp/env-audita",
},
overrides=ConfigOverrides(
max_section_tokens=100,
confidence_threshold=0.7,
glossary_confidence_threshold=0.7,
grammar_confidence_threshold=0.65,
max_retries=3,
glossary_max_llm_passes=2,
grammar_max_llm_passes=3,
work_dir=Path("/tmp/cli-audita"),
),
)
assert config.max_section_tokens == 100
assert config.confidence_threshold == 0.7
assert config.glossary_confidence_threshold == 0.7
assert config.grammar_confidence_threshold == 0.65
assert config.max_retries == 3
assert config.glossary_max_llm_passes == 2
assert config.grammar_max_llm_passes == 3
assert config.work_dir == Path("/tmp/cli-audita")
@@ -79,3 +99,30 @@ def test_config_rejects_invalid_glossary_pass_count():
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GLOSSARY_MAX_LLM_PASSES": "0"}
)
def test_config_rejects_invalid_grammar_pass_count():
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GRAMMAR_MAX_LLM_PASSES": "0"}
)
def test_config_rejects_invalid_stage_thresholds():
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "1.1"}
)
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "-0.1"}
)
def test_legacy_confidence_threshold_env_is_ignored():
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_CONFIDENCE_THRESHOLD": "0.9"}
)
assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
assert config.grammar_confidence_threshold == DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD