Replace legacy configuration loading with ConfigKit schema two
This commit is contained in:
@@ -1,705 +1,145 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
)
|
||||
|
||||
func TestDefaultConfigValues(t *testing.T) {
|
||||
cfg := Default()
|
||||
const base = "version: 2\nbackends:\n local:\n endpoint: https://local.invalid/v1\nllm:\n proposal_profile: proposal\n"
|
||||
|
||||
if got, want := strings.Join(cfg.Modules, ","), DefaultModulesCSV; got != want {
|
||||
t.Fatalf("modules mismatch: got %q want %q", got, want)
|
||||
}
|
||||
if cfg.OutputSchema != DefaultOutputSchema {
|
||||
t.Fatalf("unexpected default output schema: %q", cfg.OutputSchema)
|
||||
}
|
||||
if cfg.PrimaryLLM.Model != DefaultPrimaryModel {
|
||||
t.Fatalf("unexpected default primary model: %q", cfg.PrimaryLLM.Model)
|
||||
}
|
||||
if cfg.PrimaryLLM.BaseURL != DefaultPrimaryBaseURL {
|
||||
t.Fatalf("unexpected default primary base url: %q", cfg.PrimaryLLM.BaseURL)
|
||||
}
|
||||
if cfg.PrimaryLLM.TimeoutSeconds != DefaultPrimaryLLMTimeoutSeconds {
|
||||
t.Fatalf("unexpected default timeout seconds: %d", cfg.PrimaryLLM.TimeoutSeconds)
|
||||
}
|
||||
if cfg.PrimaryLLM.MaxRetries != DefaultMaxRetries {
|
||||
t.Fatalf("unexpected default max retries: %d", cfg.PrimaryLLM.MaxRetries)
|
||||
}
|
||||
if cfg.TotalLLMConcurrency != DefaultLLMConcurrency {
|
||||
t.Fatalf("unexpected default total llm concurrency: %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != DefaultLLMConcurrency {
|
||||
t.Fatalf("unexpected default proposal llm concurrency: %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
if cfg.ValidationLLMConcurrency != nil {
|
||||
t.Fatalf("expected validation llm concurrency to be unset by default")
|
||||
}
|
||||
if cfg.PrimaryLLM.Concurrency != cfg.TotalLLMConcurrency {
|
||||
t.Fatalf("expected primary llm concurrency alias to mirror total, got primary=%d total=%d", cfg.PrimaryLLM.Concurrency, cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ValidationLLM.TimeoutSeconds != nil {
|
||||
t.Fatalf("expected validation timeout to be unset by default")
|
||||
}
|
||||
if cfg.ValidationLLM.MaxRetries != nil {
|
||||
t.Fatalf("expected validation max retries to be unset by default")
|
||||
}
|
||||
if cfg.ValidationLLM.Concurrency != nil {
|
||||
t.Fatalf("expected legacy validation llm concurrency alias to be unset by default")
|
||||
}
|
||||
if cfg.TargetSections != nil {
|
||||
t.Fatalf("expected target sections to be unset by default")
|
||||
}
|
||||
if cfg.WorkDir != DefaultWorkDir {
|
||||
t.Fatalf("unexpected default work dir: %q", cfg.WorkDir)
|
||||
}
|
||||
if cfg.TranscriptDescription != "" {
|
||||
t.Fatalf("expected default transcript description to be empty, got %q", cfg.TranscriptDescription)
|
||||
}
|
||||
if cfg.WorkDirRetention != DefaultWorkDirRetention {
|
||||
t.Fatalf("unexpected default work dir retention: %q", cfg.WorkDirRetention)
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("default config should validate: %v", err)
|
||||
}
|
||||
func loadText(text, overlay string, overrides ...ck.Override) (*ck.Result[Config], error) {
|
||||
return Load(ck.Bytes("config.yml", []byte(text)), overlay, overrides...)
|
||||
}
|
||||
|
||||
func TestLoadFromEnvOverridesAndFallback(t *testing.T) {
|
||||
env := map[string]string{
|
||||
"AUDITA_MODEL": "openai/gpt-4.1-mini",
|
||||
"AUDITA_BASE_URL": "https://api.openai.com/v1",
|
||||
"AUDITA_LLM_TIMEOUT_SECONDS": "120",
|
||||
"AUDITA_MAX_RETRIES": "7",
|
||||
"AUDITA_TOTAL_LLM_CONCURRENCY": "6",
|
||||
"AUDITA_PROPOSAL_LLM_CONCURRENCY": "4",
|
||||
"AUDITA_VALIDATION_LLM_CONCURRENCY": "2",
|
||||
"AUDITA_VALIDATION_MAX_PROMPT_TOKENS": "4096",
|
||||
"AUDITA_MAX_SECTION_TOKENS": "9000",
|
||||
"AUDITA_MIN_SECTION_TOKENS": "3000",
|
||||
"AUDITA_TARGET_SECTIONS": "5",
|
||||
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.9",
|
||||
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.7",
|
||||
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.6",
|
||||
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.5",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "2.5",
|
||||
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "2.0",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "30.0",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "1024",
|
||||
"AUDITA_WORK_DIR": "/var/tmp/audita",
|
||||
"AUDITA_WORK_DIR_RETENTION": "always",
|
||||
"OPENROUTER_API_KEY": "fallback-key",
|
||||
}
|
||||
|
||||
cfg, err := loadFromLookup(mapLookup(env))
|
||||
func TestDefaultsAndDomainSettings(t *testing.T) {
|
||||
loaded, err := loadText(base, "")
|
||||
if err != nil {
|
||||
t.Fatalf("loadFromLookup returned error: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if cfg.PrimaryLLM.APIKey != "fallback-key" {
|
||||
t.Fatalf("expected OPENROUTER_API_KEY fallback, got %q", cfg.PrimaryLLM.APIKey)
|
||||
c := loaded.Config
|
||||
if c.LLM.ValidationProfile != "proposal" || c.Backends["local"].Concurrency != 1 || c.Backends["local"].QueueCapacity != 1024 {
|
||||
t.Fatalf("derived settings: %+v %+v", c.LLM, c.Backends)
|
||||
}
|
||||
if cfg.PrimaryLLM.Model != env["AUDITA_MODEL"] {
|
||||
t.Fatalf("unexpected model: %q", cfg.PrimaryLLM.Model)
|
||||
if !reflect.DeepEqual(c.Pipeline.Modules, []string{"glossary", "homophones", "glossary", "spoken_word", "grammar"}) {
|
||||
t.Fatalf("module order: %v", c.Pipeline.Modules)
|
||||
}
|
||||
if cfg.PrimaryLLM.BaseURL != env["AUDITA_BASE_URL"] {
|
||||
t.Fatalf("unexpected base url: %q", cfg.PrimaryLLM.BaseURL)
|
||||
if explanation, ok := loaded.Explain(ck.Keys("llm", "validation_profile")); !ok || len(explanation.History) == 0 {
|
||||
t.Fatal("validation inheritance has no recorded derivation")
|
||||
}
|
||||
if cfg.TargetSections == nil || *cfg.TargetSections != 5 {
|
||||
t.Fatalf("unexpected target sections: %#v", cfg.TargetSections)
|
||||
if _, err := loadText(base+"pipeline:\n modules: []\n", ""); err != nil {
|
||||
t.Fatalf("empty pipeline: %v", err)
|
||||
}
|
||||
if cfg.TotalLLMConcurrency != 6 {
|
||||
t.Fatalf("unexpected total llm concurrency: %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 4 {
|
||||
t.Fatalf("unexpected proposal llm concurrency: %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
if cfg.ValidationLLMConcurrency == nil || *cfg.ValidationLLMConcurrency != 2 {
|
||||
t.Fatalf("unexpected validation llm concurrency: %#v", cfg.ValidationLLMConcurrency)
|
||||
}
|
||||
if cfg.PrimaryLLM.Concurrency != 6 {
|
||||
t.Fatalf("expected primary alias concurrency 6, got %d", cfg.PrimaryLLM.Concurrency)
|
||||
}
|
||||
if cfg.ValidationLLM.Concurrency == nil || *cfg.ValidationLLM.Concurrency != 2 {
|
||||
t.Fatalf("expected validation alias concurrency 2, got %#v", cfg.ValidationLLM.Concurrency)
|
||||
}
|
||||
if cfg.WorkDirRetention != WorkDirRetentionAlways {
|
||||
t.Fatalf("unexpected work dir retention: %q", cfg.WorkDirRetention)
|
||||
if _, err := loadText(base+"pipeline:\n modules: [grammar, glossary, grammar]\n", ""); err != nil {
|
||||
t.Fatalf("repeated modules: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvLegacyLLMConcurrencyAliasForTotalAndProposal(t *testing.T) {
|
||||
env := map[string]string{
|
||||
"AUDITA_LLM_CONCURRENCY": "5",
|
||||
}
|
||||
|
||||
cfg, err := loadFromLookup(mapLookup(env))
|
||||
if err != nil {
|
||||
t.Fatalf("loadFromLookup returned error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 5 {
|
||||
t.Fatalf("expected total concurrency from legacy alias, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 5 {
|
||||
t.Fatalf("expected proposal concurrency to inherit legacy total, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvCanonicalTotalWinsLegacyAlias(t *testing.T) {
|
||||
env := map[string]string{
|
||||
"AUDITA_TOTAL_LLM_CONCURRENCY": "4",
|
||||
"AUDITA_LLM_CONCURRENCY": "9",
|
||||
}
|
||||
|
||||
cfg, err := loadFromLookup(mapLookup(env))
|
||||
if err != nil {
|
||||
t.Fatalf("loadFromLookup returned error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected canonical total to win over legacy alias, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected proposal to inherit canonical total when unset, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvUsesAuditaLLMAPIKeyOverFallback(t *testing.T) {
|
||||
env := map[string]string{
|
||||
"AUDITA_LLM_API_KEY": "primary-key",
|
||||
"OPENROUTER_API_KEY": "fallback-key",
|
||||
}
|
||||
|
||||
cfg, err := loadFromLookup(mapLookup(env))
|
||||
if err != nil {
|
||||
t.Fatalf("loadFromLookup returned error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.PrimaryLLM.APIKey != "primary-key" {
|
||||
t.Fatalf("expected AUDITA_LLM_API_KEY to win, got %q", cfg.PrimaryLLM.APIKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCLIOverridesPrecedence(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.PrimaryLLM.Model = "env-model"
|
||||
cfg.WorkDir = "/env/work"
|
||||
|
||||
model := "cli-model"
|
||||
workDir := "/cli/work"
|
||||
modules := "grammar"
|
||||
outputSchema := "audita-v1"
|
||||
totalLLMConcurrency := 5
|
||||
proposalLLMConcurrency := 3
|
||||
overrides := CLIOverrides{
|
||||
PrimaryModel: &model,
|
||||
WorkDir: &workDir,
|
||||
ModulesCSV: &modules,
|
||||
OutputSchema: &outputSchema,
|
||||
TotalLLMConcurrency: &totalLLMConcurrency,
|
||||
ProposalLLMConcurrency: &proposalLLMConcurrency,
|
||||
}
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(overrides); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
|
||||
if cfg.PrimaryLLM.Model != "cli-model" {
|
||||
t.Fatalf("expected CLI model override, got %q", cfg.PrimaryLLM.Model)
|
||||
}
|
||||
if cfg.WorkDir != "/cli/work" {
|
||||
t.Fatalf("expected CLI work dir override, got %q", cfg.WorkDir)
|
||||
}
|
||||
if !reflect.DeepEqual(cfg.Modules, []string{"grammar"}) {
|
||||
t.Fatalf("unexpected modules: %#v", cfg.Modules)
|
||||
}
|
||||
if cfg.OutputSchema != "audita-v1" {
|
||||
t.Fatalf("expected CLI output schema override, got %q", cfg.OutputSchema)
|
||||
}
|
||||
if cfg.TotalLLMConcurrency != 5 {
|
||||
t.Fatalf("expected CLI total concurrency override, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 3 {
|
||||
t.Fatalf("expected CLI proposal concurrency override, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCLIOverridesTrimsTranscriptDescription(t *testing.T) {
|
||||
cfg := Default()
|
||||
description := " background context about speakers "
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{TranscriptDescription: &description}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
if cfg.TranscriptDescription != "background context about speakers" {
|
||||
t.Fatalf("unexpected transcript description trim result: %q", cfg.TranscriptDescription)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigSourcesApplySharedEffectiveFieldsConsistently(t *testing.T) {
|
||||
fileCfg := mustParseFileConfigYAML(t, `
|
||||
version: 1
|
||||
output:
|
||||
schema: " audita-v1 "
|
||||
llm:
|
||||
proposal:
|
||||
base_url: https://proposal.example.test/v1
|
||||
model: provider/proposal
|
||||
timeout: 101
|
||||
max_retries: 5
|
||||
validation:
|
||||
base_url: https://validation.example.test/v1
|
||||
model: provider/validation
|
||||
timeout: 202
|
||||
max_retries: 6
|
||||
chunking:
|
||||
target_sections: 7
|
||||
max_section_tokens: 9000
|
||||
min_section_tokens: 1000
|
||||
thresholds:
|
||||
glossary: 0.91
|
||||
grammar: 0.92
|
||||
homophones: 0.93
|
||||
spoken_word: 0.94
|
||||
normalization:
|
||||
max_segment_gap: 1.2
|
||||
ellipsis_gap: 2.3
|
||||
max_segment_duration: 45.6
|
||||
max_segment_tokens: 321
|
||||
context:
|
||||
description: " shared context "
|
||||
diagnostics:
|
||||
work_dir: /tmp/audita-shared
|
||||
retention: always
|
||||
`)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
apply func(*Config) error
|
||||
}{
|
||||
{
|
||||
name: "file",
|
||||
apply: func(cfg *Config) error {
|
||||
return cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{}))
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "env",
|
||||
apply: func(cfg *Config) error {
|
||||
return cfg.applyEnvOverrides(mapLookup(map[string]string{
|
||||
"AUDITA_MODEL": "provider/proposal",
|
||||
"AUDITA_BASE_URL": "https://proposal.example.test/v1",
|
||||
"AUDITA_LLM_TIMEOUT_SECONDS": "101",
|
||||
"AUDITA_MAX_RETRIES": "5",
|
||||
"AUDITA_VALIDATION_MODEL": "provider/validation",
|
||||
"AUDITA_VALIDATION_BASE_URL": "https://validation.example.test/v1",
|
||||
"AUDITA_VALIDATION_LLM_TIMEOUT_SECONDS": "202",
|
||||
"AUDITA_VALIDATION_MAX_RETRIES": "6",
|
||||
"AUDITA_TARGET_SECTIONS": "7",
|
||||
"AUDITA_MAX_SECTION_TOKENS": "9000",
|
||||
"AUDITA_MIN_SECTION_TOKENS": "1000",
|
||||
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.91",
|
||||
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.92",
|
||||
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.93",
|
||||
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.94",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "1.2",
|
||||
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "2.3",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.6",
|
||||
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "321",
|
||||
"AUDITA_WORK_DIR": "/tmp/audita-shared",
|
||||
"AUDITA_WORK_DIR_RETENTION": "always",
|
||||
}))
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "cli",
|
||||
apply: func(cfg *Config) error {
|
||||
outputSchema := " audita-v1 "
|
||||
proposalModel := "provider/proposal"
|
||||
proposalBaseURL := "https://proposal.example.test/v1"
|
||||
proposalTimeout := 101
|
||||
proposalMaxRetries := 5
|
||||
validationModel := "provider/validation"
|
||||
validationBaseURL := "https://validation.example.test/v1"
|
||||
validationTimeout := 202
|
||||
validationMaxRetries := 6
|
||||
targetSections := 7
|
||||
maxSectionTokens := 9000
|
||||
minSectionTokens := 1000
|
||||
glossaryThreshold := 0.91
|
||||
grammarThreshold := 0.92
|
||||
homophonesThreshold := 0.93
|
||||
spokenWordThreshold := 0.94
|
||||
normalizeMaxSegmentGap := 1.2
|
||||
normalizeEllipsisGap := 2.3
|
||||
normalizeMaxSegmentDuration := 45.6
|
||||
normalizeMaxSegmentTokens := 321
|
||||
description := " shared context "
|
||||
workDir := "/tmp/audita-shared"
|
||||
workDirRetention := "always"
|
||||
return cfg.ApplyCLIOverrides(CLIOverrides{
|
||||
OutputSchema: &outputSchema,
|
||||
PrimaryModel: &proposalModel,
|
||||
PrimaryBaseURL: &proposalBaseURL,
|
||||
PrimaryLLMTimeoutSeconds: &proposalTimeout,
|
||||
MaxRetries: &proposalMaxRetries,
|
||||
ValidationModel: &validationModel,
|
||||
ValidationBaseURL: &validationBaseURL,
|
||||
ValidationLLMTimeoutSeconds: &validationTimeout,
|
||||
ValidationMaxRetries: &validationMaxRetries,
|
||||
TargetSections: &targetSections,
|
||||
MaxSectionTokens: &maxSectionTokens,
|
||||
MinSectionTokens: &minSectionTokens,
|
||||
GlossaryConfidenceThreshold: &glossaryThreshold,
|
||||
GrammarConfidenceThreshold: &grammarThreshold,
|
||||
HomophonesConfidenceThreshold: &homophonesThreshold,
|
||||
SpokenWordConfidenceThreshold: &spokenWordThreshold,
|
||||
NormalizeMaxSegmentGap: &normalizeMaxSegmentGap,
|
||||
NormalizeEllipsisGap: &normalizeEllipsisGap,
|
||||
NormalizeMaxSegmentDuration: &normalizeMaxSegmentDuration,
|
||||
NormalizeMaxSegmentTokens: &normalizeMaxSegmentTokens,
|
||||
TranscriptDescription: &description,
|
||||
WorkDir: &workDir,
|
||||
WorkDirRetention: &workDirRetention,
|
||||
})
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
func TestStrictInputAndConstraints(t *testing.T) {
|
||||
for _, tc := range []struct{ name, text string }{
|
||||
{"old version", strings.Replace(base, "version: 2", "version: 1", 1)},
|
||||
{"missing version", strings.Replace(base, "version: 2\n", "", 1)},
|
||||
{"null version", strings.Replace(base, "version: 2", "version: null", 1)},
|
||||
{"quoted version", strings.Replace(base, "version: 2", "version: '2'", 1)},
|
||||
{"unknown field", base + "mystery: true\n"},
|
||||
{"empty validation profile", base + " validation_profile: ''\n"},
|
||||
{"null validation profile", base + " validation_profile: null\n"},
|
||||
{"null proposal profile", strings.Replace(base, "proposal_profile: proposal", "proposal_profile: null", 1)},
|
||||
{"zero backend concurrency", base + " concurrency: 0\n"},
|
||||
{"invalid target", base + "chunking:\n target_sections: 0\n"},
|
||||
{"invalid section relation", base + "chunking:\n min_section_tokens: 10000\n"},
|
||||
{"string normalization", base + "normalization:\n max_segment_gap: 4s\n"},
|
||||
{"invalid threshold", base + "thresholds:\n glossary: 1.1\n"},
|
||||
{"invalid retention", base + "diagnostics:\n retention: sometimes\n"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg := Default()
|
||||
if err := tc.apply(&cfg); err != nil {
|
||||
t.Fatalf("apply config source: %v", err)
|
||||
if _, err := loadText(tc.text, ""); err == nil {
|
||||
t.Fatal("invalid config accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
_, err := loadText(strings.Replace(base, "version: 2", "version: 1", 1), "")
|
||||
var loadErr *ck.LoadError
|
||||
if !errors.As(err, &loadErr) || len(loadErr.Diagnostics()) == 0 || !strings.Contains(loadErr.Diagnostics()[0].Message, "docs/migrations/v2.md") {
|
||||
t.Fatalf("migration guidance: %v", err)
|
||||
}
|
||||
for _, tc := range []struct{ name, text string }{
|
||||
{"optional target null", base + "chunking:\n target_sections: null\n"},
|
||||
{"zero queue", strings.Replace(base, "llm:\n", " queue_capacity: 0\nllm:\n", 1)},
|
||||
{"zero retries", base + " transient_retries: 0\n repair_attempts: 0\n"},
|
||||
{"zero threshold", base + "thresholds:\n grammar: 0\n"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if _, err := loadText(tc.text, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertSharedEffectiveFields(t, cfg, sharedEffectiveFieldOptions{
|
||||
wantOutputSchemaOverride: tc.name != "env",
|
||||
wantTranscriptDescriptionPatch: tc.name != "env",
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLIAPIKeyOverrideIsDirectValue(t *testing.T) {
|
||||
cfg := Default()
|
||||
apiKey := "NOT_AN_ENV_VAR_NAME"
|
||||
validationAPIKey := "also direct"
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{PrimaryLLMAPIKey: &apiKey, ValidationLLMAPIKey: &validationAPIKey}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
func TestOverlayOverrideAndInheritance(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(dir, "overlays"), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.PrimaryLLM.APIKey != apiKey {
|
||||
t.Fatalf("expected direct primary api key, got %q", cfg.PrimaryLLM.APIKey)
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.ValidationLLM.APIKey != validationAPIKey {
|
||||
t.Fatalf("expected direct validation api key, got %q", cfg.ValidationLLM.APIKey)
|
||||
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("llm:\n proposal_profile: testing\nchunking:\n max_section_tokens: 1000\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigTotalConcurrencyDoesNotChangeProposalWhenProposalUnset(t *testing.T) {
|
||||
fileCfg := mustParseFileConfigYAML(t, `
|
||||
version: 1
|
||||
concurrency:
|
||||
total_llm: 4
|
||||
`)
|
||||
cfg := Default()
|
||||
|
||||
if err := cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{})); err != nil {
|
||||
t.Fatalf("applyFileConfigWithLookup failed: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected file total concurrency 4, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != DefaultLLMConcurrency {
|
||||
t.Fatalf("expected file config to preserve proposal concurrency when unset, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationRejectsOverlyLongTranscriptDescription(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.TranscriptDescription = strings.Repeat("a", DefaultTranscriptDescriptionMaxChars+1)
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expected transcript description length validation error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "transcript description must be 500 characters or fewer") {
|
||||
t.Fatalf("unexpected validation error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCLIOverridesLegacyLLMConcurrencyAlias(t *testing.T) {
|
||||
cfg := Default()
|
||||
aliasConcurrency := 6
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{PrimaryLLMConcurrency: &aliasConcurrency}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 6 {
|
||||
t.Fatalf("expected legacy --llm-concurrency alias to set total, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 6 {
|
||||
t.Fatalf("expected proposal to inherit aliased total when unset, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyCLIOverridesCanonicalTotalWinsLegacyAlias(t *testing.T) {
|
||||
cfg := Default()
|
||||
canonicalTotal := 4
|
||||
legacyAlias := 9
|
||||
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{TotalLLMConcurrency: &canonicalTotal, PrimaryLLMConcurrency: &legacyAlias}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
|
||||
if cfg.TotalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected canonical total concurrency to win, got %d", cfg.TotalLLMConcurrency)
|
||||
}
|
||||
if cfg.ProposalLLMConcurrency != 4 {
|
||||
t.Fatalf("expected proposal to inherit canonical total when proposal is unset, got %d", cfg.ProposalLLMConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationFailures(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.OutputSchema = "unknown-schema"
|
||||
cfg.PrimaryLLM.TimeoutSeconds = -1
|
||||
cfg.TotalLLMConcurrency = 0
|
||||
cfg.ProposalLLMConcurrency = 0
|
||||
validationConcurrency := 5
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
cfg.ValidationMaxPromptTokens = 0
|
||||
cfg.MaxSectionTokens = 100
|
||||
cfg.MinSectionTokens = 200
|
||||
cfg.Thresholds.Grammar = 1.5
|
||||
cfg.WorkDirRetention = WorkDirRetention("sometimes")
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expected validation error")
|
||||
}
|
||||
|
||||
message := err.Error()
|
||||
for _, expected := range []string{
|
||||
"primary llm timeout seconds",
|
||||
"total llm concurrency",
|
||||
"proposal llm concurrency",
|
||||
"validation llm concurrency must be less than or equal to total llm concurrency",
|
||||
"validation max prompt tokens",
|
||||
"min section tokens",
|
||||
"grammar confidence threshold",
|
||||
"work dir retention",
|
||||
"unsupported output schema",
|
||||
} {
|
||||
if !strings.Contains(message, expected) {
|
||||
t.Fatalf("expected error to contain %q, got %q", expected, message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationRejectsUnsupportedModuleKey(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Modules = []string{modulecatalog.KeyGlossary, "made_up"}
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatalf("expected validation error for unsupported module key")
|
||||
}
|
||||
if !strings.Contains(err.Error(), `unsupported module key "made_up"`) {
|
||||
t.Fatalf("expected unsupported module key error, got %q", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationAllowsRepeatedSupportedModuleKeys(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Modules = []string{modulecatalog.KeyGlossary, modulecatalog.KeyGlossary, modulecatalog.KeyGrammar}
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("expected repeated supported module keys to validate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationAcceptsAllSupportedOutputSchemas(t *testing.T) {
|
||||
for _, schemaKey := range outputschema.SupportedKeys() {
|
||||
cfg := Default()
|
||||
cfg.OutputSchema = schemaKey
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("expected output schema %q to validate, got %v", schemaKey, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveValidationLLMInheritance(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.PrimaryLLM.APIKey = "primary-key"
|
||||
cfg.PrimaryLLM.Model = "primary-model"
|
||||
cfg.PrimaryLLM.BaseURL = "https://primary.example/v1"
|
||||
cfg.PrimaryLLM.TimeoutSeconds = 111
|
||||
cfg.PrimaryLLM.MaxRetries = 2
|
||||
cfg.TotalLLMConcurrency = 7
|
||||
cfg.syncLegacyConcurrencyAliases()
|
||||
|
||||
effective := cfg.EffectiveValidationLLMConfig()
|
||||
if effective.APIKey != "primary-key" || effective.Model != "primary-model" || effective.BaseURL != "https://primary.example/v1" || effective.TimeoutSeconds != 111 || effective.MaxRetries != 2 || effective.Concurrency != 7 {
|
||||
t.Fatalf("unexpected inherited config: %#v", effective)
|
||||
}
|
||||
|
||||
validationTimeout := 222
|
||||
validationRetries := 9
|
||||
cfg.ValidationLLM.APIKey = "validation-key"
|
||||
cfg.ValidationLLM.Model = "validation-model"
|
||||
cfg.ValidationLLM.BaseURL = "https://validation.example/v1"
|
||||
cfg.ValidationLLM.TimeoutSeconds = &validationTimeout
|
||||
cfg.ValidationLLM.MaxRetries = &validationRetries
|
||||
validationConcurrency := 4
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
cfg.syncLegacyConcurrencyAliases()
|
||||
|
||||
effective = cfg.EffectiveValidationLLMConfig()
|
||||
if effective.APIKey != "validation-key" || effective.Model != "validation-model" || effective.BaseURL != "https://validation.example/v1" || effective.TimeoutSeconds != 222 || effective.MaxRetries != 9 || effective.Concurrency != 4 {
|
||||
t.Fatalf("unexpected overridden validation config: %#v", effective)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationLLMConcurrencyCannotExceedTotal(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.TotalLLMConcurrency = 2
|
||||
cfg.ProposalLLMConcurrency = 2
|
||||
validationConcurrency := 3
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Fatal("expected validation error when validation llm concurrency exceeds total")
|
||||
}
|
||||
|
||||
validationConcurrency = 2
|
||||
cfg.ValidationLLMConcurrency = &validationConcurrency
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("expected equal concurrency to validate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProposalLLMConcurrencyCannotExceedTotal(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.TotalLLMConcurrency = 2
|
||||
cfg.ProposalLLMConcurrency = 3
|
||||
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Fatal("expected validation error when proposal llm concurrency exceeds total")
|
||||
}
|
||||
|
||||
cfg.ProposalLLMConcurrency = 2
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("expected equal concurrency to validate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCLITotalLLMConcurrencyOverrideDrivesEffectiveValidationConcurrencyWhenValidationUnset(t *testing.T) {
|
||||
cfg := Default()
|
||||
totalLLMConcurrency := 6
|
||||
if err := cfg.ApplyCLIOverrides(CLIOverrides{TotalLLMConcurrency: &totalLLMConcurrency}); err != nil {
|
||||
t.Fatalf("ApplyCLIOverrides failed: %v", err)
|
||||
}
|
||||
if cfg.ValidationLLMConcurrency != nil {
|
||||
t.Fatalf("expected validation concurrency to remain unset, got %#v", cfg.ValidationLLMConcurrency)
|
||||
}
|
||||
if cfg.EffectiveValidationLLMConcurrency() != 6 {
|
||||
t.Fatalf("expected inherited validation concurrency 6, got %d", cfg.EffectiveValidationLLMConcurrency())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactedConfig(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.PrimaryLLM.APIKey = "secret-primary"
|
||||
cfg.ValidationLLM.APIKey = "secret-validation"
|
||||
|
||||
redacted := cfg.Redacted()
|
||||
|
||||
if redacted.PrimaryLLM.APIKey != redactedSecret {
|
||||
t.Fatalf("expected primary api key to be redacted, got %q", redacted.PrimaryLLM.APIKey)
|
||||
}
|
||||
if redacted.ValidationLLM.APIKey != redactedSecret {
|
||||
t.Fatalf("expected validation api key to be redacted, got %q", redacted.ValidationLLM.APIKey)
|
||||
}
|
||||
if cfg.PrimaryLLM.APIKey != "secret-primary" {
|
||||
t.Fatalf("redaction should not mutate original config")
|
||||
}
|
||||
}
|
||||
|
||||
func mapLookup(values map[string]string) func(string) (string, bool) {
|
||||
return func(key string) (string, bool) {
|
||||
value, ok := values[key]
|
||||
return value, ok
|
||||
}
|
||||
}
|
||||
|
||||
func mustParseFileConfigYAML(t *testing.T, raw string) FileConfig {
|
||||
t.Helper()
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(raw))
|
||||
loaded, err := Load(ck.File(main), "testing", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "explicit", Flag: "--proposal-profile"}, ck.Override{Path: ck.Keys("chunking", "min_section_tokens"), Value: 500, Flag: "--min-section-tokens"})
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML failed: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
if loaded.Config.LLM.ProposalProfile != "explicit" || loaded.Config.LLM.ValidationProfile != "explicit" || loaded.Config.Chunking.MaxSectionTokens != 1000 || loaded.Config.Chunking.MinSectionTokens != 500 {
|
||||
t.Fatalf("resolved: %+v", loaded.Config)
|
||||
}
|
||||
if got, _ := loaded.Metadata().Lookup(ck.Keys("llm", "proposal_profile")); got.Origin.Layer != ck.LayerCLI {
|
||||
t.Fatalf("override origin: %+v", got)
|
||||
}
|
||||
if got, _ := loaded.Metadata().Lookup(ck.Keys("chunking", "max_section_tokens")); got.Origin.Layer != ck.LayerOverlay {
|
||||
t.Fatalf("overlay origin: %+v", got)
|
||||
}
|
||||
return fileCfg
|
||||
}
|
||||
|
||||
type sharedEffectiveFieldOptions struct {
|
||||
wantOutputSchemaOverride bool
|
||||
wantTranscriptDescriptionPatch bool
|
||||
}
|
||||
|
||||
func assertSharedEffectiveFields(t *testing.T, cfg Config, opts sharedEffectiveFieldOptions) {
|
||||
t.Helper()
|
||||
wantOutputSchema := DefaultOutputSchema
|
||||
if opts.wantOutputSchemaOverride {
|
||||
wantOutputSchema = "audita-v1"
|
||||
func TestSurvivingCredentials(t *testing.T) {
|
||||
t.Setenv("BASE_SECRET", "")
|
||||
t.Setenv("NEW_SECRET", "synthetic-secret")
|
||||
withSecret := strings.Replace(base, "llm:\n", " other:\n endpoint: https://other.invalid/v1\n credential: BASE_SECRET\nllm:\n", 1)
|
||||
if _, err := loadText(withSecret, ""); err == nil {
|
||||
t.Fatal("missing unselected backend secret accepted")
|
||||
}
|
||||
if cfg.OutputSchema != wantOutputSchema {
|
||||
t.Fatalf("unexpected output schema: %q", cfg.OutputSchema)
|
||||
replacement := ck.Override{Path: ck.Keys("backends", "other", "credential"), Value: "NEW_SECRET", Flag: "--credential"}
|
||||
loaded, err := loadText(withSecret, "", replacement)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.PrimaryLLM.Model != "provider/proposal" ||
|
||||
cfg.PrimaryLLM.BaseURL != "https://proposal.example.test/v1" ||
|
||||
cfg.PrimaryLLM.TimeoutSeconds != 101 ||
|
||||
cfg.PrimaryLLM.MaxRetries != 5 {
|
||||
t.Fatalf("unexpected primary llm config: %+v", cfg.PrimaryLLM)
|
||||
if got := loaded.Config.Backends["other"].Credential.Value(); got != "synthetic-secret" {
|
||||
t.Fatalf("resolved secret: %q", got)
|
||||
}
|
||||
if cfg.ValidationLLM.Model != "provider/validation" ||
|
||||
cfg.ValidationLLM.BaseURL != "https://validation.example.test/v1" ||
|
||||
cfg.ValidationLLM.TimeoutSeconds == nil ||
|
||||
*cfg.ValidationLLM.TimeoutSeconds != 202 ||
|
||||
cfg.ValidationLLM.MaxRetries == nil ||
|
||||
*cfg.ValidationLLM.MaxRetries != 6 {
|
||||
t.Fatalf("unexpected validation llm config: %+v", cfg.ValidationLLM)
|
||||
var effective bytes.Buffer
|
||||
if err := loaded.WriteEffective(&effective); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.TargetSections == nil || *cfg.TargetSections != 7 ||
|
||||
cfg.MaxSectionTokens != 9000 ||
|
||||
cfg.MinSectionTokens != 1000 {
|
||||
t.Fatalf("unexpected chunking config: target=%v max=%d min=%d", cfg.TargetSections, cfg.MaxSectionTokens, cfg.MinSectionTokens)
|
||||
if strings.Contains(effective.String(), "synthetic-secret") {
|
||||
t.Fatal("effective output leaked a secret")
|
||||
}
|
||||
if cfg.Thresholds.Glossary != 0.91 ||
|
||||
cfg.Thresholds.Grammar != 0.92 ||
|
||||
cfg.Thresholds.Homophones != 0.93 ||
|
||||
cfg.Thresholds.SpokenWord != 0.94 {
|
||||
t.Fatalf("unexpected thresholds: %+v", cfg.Thresholds)
|
||||
null := ck.Override{Path: ck.Keys("backends", "other", "credential"), Value: nil, Flag: "--credential"}
|
||||
loaded, err = loadText(withSecret, "", null)
|
||||
if err != nil || loaded.Config.Backends["other"].Credential != nil {
|
||||
t.Fatalf("null credential: %+v %v", loaded, err)
|
||||
}
|
||||
if cfg.Normalization.MaxSegmentGap != 1.2 ||
|
||||
cfg.Normalization.EllipsisGap != 2.3 ||
|
||||
cfg.Normalization.MaxSegmentDuration != 45.6 ||
|
||||
cfg.Normalization.MaxSegmentTokens != 321 {
|
||||
t.Fatalf("unexpected normalization: %+v", cfg.Normalization)
|
||||
}
|
||||
wantDescription := ""
|
||||
if opts.wantTranscriptDescriptionPatch {
|
||||
wantDescription = "shared context"
|
||||
}
|
||||
if cfg.TranscriptDescription != wantDescription {
|
||||
t.Fatalf("unexpected transcript description: %q", cfg.TranscriptDescription)
|
||||
}
|
||||
if cfg.WorkDir != "/tmp/audita-shared" ||
|
||||
cfg.WorkDirRetention != WorkDirRetentionAlways {
|
||||
t.Fatalf("unexpected diagnostics config: work_dir=%q retention=%q", cfg.WorkDir, cfg.WorkDirRetention)
|
||||
t.Setenv("AUDITA_MODEL", "legacy-model")
|
||||
loaded, err = loadText(base, "")
|
||||
if err != nil || loaded.Config.LLM.ProposalProfile != "proposal" {
|
||||
t.Fatalf("legacy environment affected loading: %+v %v", loaded, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user