Replace legacy configuration loading with ConfigKit schema two
This commit is contained in:
@@ -3,197 +3,187 @@ package config
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/outputschema"
|
||||
)
|
||||
|
||||
type WorkDirRetention string
|
||||
|
||||
const (
|
||||
WorkDirRetentionAuto WorkDirRetention = "auto"
|
||||
WorkDirRetentionAlways WorkDirRetention = "always"
|
||||
WorkDirRetentionNever WorkDirRetention = "never"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultModulesCSV = modulecatalog.KeyGlossary + "," + modulecatalog.KeyHomophones + "," + modulecatalog.KeyGlossary + "," + modulecatalog.KeySpokenWord + "," + modulecatalog.KeyGrammar
|
||||
DefaultOutputSchema = "bare-segments"
|
||||
DefaultPrimaryModel = "openrouter/google/gemma-4-31b-it"
|
||||
DefaultPrimaryBaseURL = "https://openrouter.ai/api/v1"
|
||||
DefaultPrimaryLLMTimeoutSeconds = 600
|
||||
DefaultMaxRetries = 3
|
||||
DefaultLLMConcurrency = 1
|
||||
DefaultValidationMaxPromptTokens = 2048
|
||||
DefaultMaxSectionTokens = 8192
|
||||
DefaultMinSectionTokens = 2048
|
||||
DefaultConfidenceThreshold = 0.8
|
||||
DefaultNormalizeMaxSegmentGap = 4.0
|
||||
DefaultNormalizeEllipsisGap = 3.5
|
||||
DefaultNormalizeMaxSegmentDuration = 60.0
|
||||
DefaultNormalizeMaxSegmentTokens = 2048
|
||||
WorkDirRetentionAuto WorkDirRetention = "auto"
|
||||
WorkDirRetentionAlways WorkDirRetention = "always"
|
||||
WorkDirRetentionNever WorkDirRetention = "never"
|
||||
DefaultModulesCSV = "glossary,homophones,glossary,spoken_word,grammar"
|
||||
DefaultTranscriptDescriptionMaxChars = 500
|
||||
DefaultWorkDir = "/tmp/audita"
|
||||
DefaultWorkDirRetention WorkDirRetention = WorkDirRetentionAuto
|
||||
)
|
||||
|
||||
// Config is the version-2 application configuration. Invocation paths are not settings.
|
||||
type Config struct {
|
||||
Modules []string
|
||||
OutputSchema string
|
||||
PrimaryLLM LLMConfig
|
||||
ValidationLLM ValidationLLMConfig
|
||||
TotalLLMConcurrency int
|
||||
ProposalLLMConcurrency int
|
||||
ValidationLLMConcurrency *int
|
||||
ValidationMaxPromptTokens int
|
||||
MaxSectionTokens int
|
||||
MinSectionTokens int
|
||||
TargetSections *int
|
||||
Thresholds ConfidenceThresholds
|
||||
Normalization NormalizationConfig
|
||||
TranscriptDescription string
|
||||
WorkDir string
|
||||
WorkDirRetention WorkDirRetention
|
||||
Version int `yaml:"version"`
|
||||
ProfilesDir string `yaml:"profiles_dir"`
|
||||
Backends map[string]Backend `yaml:"backends"`
|
||||
LLM LLM `yaml:"llm"`
|
||||
Pipeline Pipeline `yaml:"pipeline"`
|
||||
Output Output `yaml:"output"`
|
||||
Validation Validation `yaml:"validation"`
|
||||
Chunking Chunking `yaml:"chunking"`
|
||||
Normalization Normalization `yaml:"normalization"`
|
||||
Thresholds Thresholds `yaml:"thresholds"`
|
||||
Context Context `yaml:"context"`
|
||||
Diagnostics Diagnostics `yaml:"diagnostics"`
|
||||
}
|
||||
|
||||
type LLMConfig struct {
|
||||
APIKey string
|
||||
Model string
|
||||
BaseURL string
|
||||
TimeoutSeconds int
|
||||
MaxRetries int
|
||||
// Concurrency is retained as a backward-compatible alias for
|
||||
// TotalLLMConcurrency.
|
||||
Concurrency int
|
||||
type Backend struct {
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Concurrency int `yaml:"concurrency"`
|
||||
QueueCapacity int `yaml:"queue_capacity"`
|
||||
Credential *ck.Secret `yaml:"credential"`
|
||||
}
|
||||
type LLM struct {
|
||||
ProposalProfile string `yaml:"proposal_profile"`
|
||||
ValidationProfile string `yaml:"validation_profile"`
|
||||
TransientRetries int `yaml:"transient_retries"`
|
||||
RepairAttempts int `yaml:"repair_attempts"`
|
||||
OperationTimeout time.Duration `yaml:"operation_timeout"`
|
||||
}
|
||||
type Pipeline struct {
|
||||
Modules []string `yaml:"modules"`
|
||||
}
|
||||
type Output struct {
|
||||
Schema string `yaml:"schema"`
|
||||
}
|
||||
type Validation struct {
|
||||
MaxPromptTokens int `yaml:"max_prompt_tokens"`
|
||||
}
|
||||
type Chunking struct {
|
||||
TargetSections *int `yaml:"target_sections"`
|
||||
MaxSectionTokens int `yaml:"max_section_tokens"`
|
||||
MinSectionTokens int `yaml:"min_section_tokens"`
|
||||
}
|
||||
type Normalization struct {
|
||||
MaxSegmentGap float64 `yaml:"max_segment_gap"`
|
||||
EllipsisGap float64 `yaml:"ellipsis_gap"`
|
||||
MaxSegmentDuration float64 `yaml:"max_segment_duration"`
|
||||
MaxSegmentTokens int `yaml:"max_segment_tokens"`
|
||||
}
|
||||
type Thresholds struct {
|
||||
Glossary float64 `yaml:"glossary"`
|
||||
Homophones float64 `yaml:"homophones"`
|
||||
SpokenWord float64 `yaml:"spoken_word"`
|
||||
Grammar float64 `yaml:"grammar"`
|
||||
}
|
||||
type Context struct {
|
||||
Description string `yaml:"description"`
|
||||
}
|
||||
type Diagnostics struct {
|
||||
WorkDir string `yaml:"work_dir"`
|
||||
Retention WorkDirRetention `yaml:"retention"`
|
||||
}
|
||||
|
||||
type ValidationLLMConfig struct {
|
||||
APIKey string
|
||||
Model string
|
||||
BaseURL string
|
||||
TimeoutSeconds *int
|
||||
MaxRetries *int
|
||||
// Concurrency is retained as a backward-compatible alias for
|
||||
// ValidationLLMConcurrency.
|
||||
Concurrency *int
|
||||
}
|
||||
|
||||
type ConfidenceThresholds struct {
|
||||
Glossary float64
|
||||
Grammar float64
|
||||
Homophones float64
|
||||
SpokenWord float64
|
||||
}
|
||||
|
||||
type NormalizationConfig struct {
|
||||
MaxSegmentGap float64
|
||||
EllipsisGap float64
|
||||
MaxSegmentDuration float64
|
||||
MaxSegmentTokens int
|
||||
}
|
||||
|
||||
func Default() Config {
|
||||
modules, _ := ParseModulesCSV(DefaultModulesCSV)
|
||||
|
||||
func defaults() Config {
|
||||
return Config{
|
||||
Modules: modules,
|
||||
OutputSchema: DefaultOutputSchema,
|
||||
PrimaryLLM: LLMConfig{
|
||||
Model: DefaultPrimaryModel,
|
||||
BaseURL: DefaultPrimaryBaseURL,
|
||||
TimeoutSeconds: DefaultPrimaryLLMTimeoutSeconds,
|
||||
MaxRetries: DefaultMaxRetries,
|
||||
Concurrency: DefaultLLMConcurrency,
|
||||
},
|
||||
ValidationLLM: ValidationLLMConfig{},
|
||||
TotalLLMConcurrency: DefaultLLMConcurrency,
|
||||
ProposalLLMConcurrency: DefaultLLMConcurrency,
|
||||
ValidationLLMConcurrency: nil,
|
||||
ValidationMaxPromptTokens: DefaultValidationMaxPromptTokens,
|
||||
MaxSectionTokens: DefaultMaxSectionTokens,
|
||||
MinSectionTokens: DefaultMinSectionTokens,
|
||||
TargetSections: nil,
|
||||
Thresholds: ConfidenceThresholds{
|
||||
Glossary: DefaultConfidenceThreshold,
|
||||
Grammar: DefaultConfidenceThreshold,
|
||||
Homophones: DefaultConfidenceThreshold,
|
||||
SpokenWord: DefaultConfidenceThreshold,
|
||||
},
|
||||
Normalization: NormalizationConfig{
|
||||
MaxSegmentGap: DefaultNormalizeMaxSegmentGap,
|
||||
EllipsisGap: DefaultNormalizeEllipsisGap,
|
||||
MaxSegmentDuration: DefaultNormalizeMaxSegmentDuration,
|
||||
MaxSegmentTokens: DefaultNormalizeMaxSegmentTokens,
|
||||
},
|
||||
WorkDir: DefaultWorkDir,
|
||||
WorkDirRetention: DefaultWorkDirRetention,
|
||||
ProfilesDir: "profiles",
|
||||
LLM: LLM{TransientRetries: 2, RepairAttempts: 2, OperationTimeout: 15 * time.Minute},
|
||||
Pipeline: Pipeline{Modules: []string{"glossary", "homophones", "glossary", "spoken_word", "grammar"}},
|
||||
Output: Output{Schema: "bare-segments"},
|
||||
Validation: Validation{MaxPromptTokens: 2048},
|
||||
Chunking: Chunking{MaxSectionTokens: 8192, MinSectionTokens: 2048},
|
||||
Normalization: Normalization{MaxSegmentGap: 4, EllipsisGap: 3.5, MaxSegmentDuration: 60, MaxSegmentTokens: 2048},
|
||||
Thresholds: Thresholds{Glossary: .8, Homophones: .8, SpokenWord: .8, Grammar: .8},
|
||||
Diagnostics: Diagnostics{WorkDir: "/tmp/audita", Retention: WorkDirRetentionAuto},
|
||||
}
|
||||
}
|
||||
|
||||
// Load resolves the selected main file, optional overlay, and explicit setting overrides.
|
||||
// Its result retains safe effective output and provenance for CLI composition.
|
||||
func Load(source ck.Source, overlay string, overrides ...ck.Override) (*ck.Result[Config], error) {
|
||||
return ck.Load(source, ck.Options[Config]{
|
||||
Overlay: overlay, Overrides: overrides,
|
||||
Definition: ck.Definition[Config]{
|
||||
Defaults: defaults,
|
||||
Version: &ck.VersionRule{Path: ck.Keys("version"), Check: func(version ck.Version, v *ck.Validation) error {
|
||||
if !version.Present || version.Null || version.Text != "2" {
|
||||
v.Error(ck.Keys("version"), "version 2 is required; see docs/migrations/v2.md")
|
||||
}
|
||||
return nil
|
||||
}},
|
||||
Resolve: func(c *Config, r *ck.Resolution) error {
|
||||
ck.Default(r, ck.Keys("llm", "validation_profile"), &c.LLM.ValidationProfile, c.LLM.ProposalProfile, "validation uses the final proposal profile when omitted", ck.Keys("llm", "proposal_profile"))
|
||||
for id, backend := range c.Backends {
|
||||
changed := false
|
||||
if !r.IsExplicit(ck.Keys("backends", id, "concurrency")) {
|
||||
backend.Concurrency = 1
|
||||
changed = true
|
||||
}
|
||||
if !r.IsExplicit(ck.Keys("backends", id, "queue_capacity")) {
|
||||
backend.QueueCapacity = 1024
|
||||
changed = true
|
||||
}
|
||||
if changed {
|
||||
c.Backends[id] = backend
|
||||
r.Record(ck.Keys("backends", id), ck.ChangeDerivedDefault, "finite backend defaults")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Validate: validate,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func validate(c *Config, v *ck.Validation) {
|
||||
check := func(ok bool, path ck.Path, message string) {
|
||||
if !ok {
|
||||
v.Error(path, message)
|
||||
}
|
||||
}
|
||||
check(strings.TrimSpace(c.ProfilesDir) != "", ck.Keys("profiles_dir"), "profile directory is required")
|
||||
check(strings.TrimSpace(c.LLM.ProposalProfile) != "", ck.Keys("llm", "proposal_profile"), "proposal profile is required")
|
||||
check(strings.TrimSpace(c.LLM.ValidationProfile) != "", ck.Keys("llm", "validation_profile"), "validation profile is required")
|
||||
check(c.LLM.TransientRetries >= 0, ck.Keys("llm", "transient_retries"), "retry budget must be non-negative")
|
||||
check(c.LLM.RepairAttempts >= 0 && c.LLM.RepairAttempts <= 3, ck.Keys("llm", "repair_attempts"), "repair attempts must be between zero and three")
|
||||
check(c.LLM.OperationTimeout > 0, ck.Keys("llm", "operation_timeout"), "operation timeout must be positive")
|
||||
check(len(c.Backends) > 0, ck.Keys("backends"), "at least one backend is required")
|
||||
for id, b := range c.Backends {
|
||||
p := ck.Keys("backends", id)
|
||||
check(strings.TrimSpace(b.Endpoint) != "", p.Key("endpoint"), "backend endpoint is required")
|
||||
check(b.Concurrency > 0, p.Key("concurrency"), "backend concurrency must be positive")
|
||||
check(b.QueueCapacity >= 0, p.Key("queue_capacity"), "queue capacity must be non-negative")
|
||||
}
|
||||
for i, module := range c.Pipeline.Modules {
|
||||
check(modulecatalog.IsSupported(module), ck.Keys("pipeline", "modules").Index(i), "unsupported module key")
|
||||
}
|
||||
check(outputschema.IsSupported(c.Output.Schema), ck.Keys("output", "schema"), "unsupported output schema")
|
||||
check(c.Validation.MaxPromptTokens > 0, ck.Keys("validation", "max_prompt_tokens"), "validation prompt budget must be positive")
|
||||
check(c.Chunking.MaxSectionTokens > 0, ck.Keys("chunking", "max_section_tokens"), "maximum section tokens must be positive")
|
||||
check(c.Chunking.MinSectionTokens > 0 && c.Chunking.MinSectionTokens <= c.Chunking.MaxSectionTokens, ck.Keys("chunking", "min_section_tokens"), "minimum section tokens must be positive and no greater than maximum")
|
||||
check(c.Chunking.TargetSections == nil || *c.Chunking.TargetSections > 0, ck.Keys("chunking", "target_sections"), "target sections must be positive when set")
|
||||
check(c.Normalization.MaxSegmentGap >= 0, ck.Keys("normalization", "max_segment_gap"), "maximum segment gap must be non-negative")
|
||||
check(c.Normalization.EllipsisGap >= 0, ck.Keys("normalization", "ellipsis_gap"), "ellipsis gap must be non-negative")
|
||||
check(c.Normalization.MaxSegmentDuration > 0, ck.Keys("normalization", "max_segment_duration"), "maximum segment duration must be positive")
|
||||
check(c.Normalization.MaxSegmentTokens > 0, ck.Keys("normalization", "max_segment_tokens"), "maximum segment tokens must be positive")
|
||||
for name, threshold := range map[string]float64{"glossary": c.Thresholds.Glossary, "homophones": c.Thresholds.Homophones, "spoken_word": c.Thresholds.SpokenWord, "grammar": c.Thresholds.Grammar} {
|
||||
check(threshold >= 0 && threshold <= 1, ck.Keys("thresholds", name), "confidence threshold must be between zero and one")
|
||||
}
|
||||
check(len(strings.TrimSpace(c.Context.Description)) <= DefaultTranscriptDescriptionMaxChars, ck.Keys("context", "description"), fmt.Sprintf("description must be at most %d bytes after trimming", DefaultTranscriptDescriptionMaxChars))
|
||||
check(strings.TrimSpace(c.Diagnostics.WorkDir) != "", ck.Keys("diagnostics", "work_dir"), "work directory is required")
|
||||
check(c.Diagnostics.Retention == WorkDirRetentionAuto || c.Diagnostics.Retention == WorkDirRetentionAlways || c.Diagnostics.Retention == WorkDirRetentionNever, ck.Keys("diagnostics", "retention"), "retention must be auto, always, or never")
|
||||
}
|
||||
|
||||
// ParseModulesCSV is used only for an explicitly supplied --modules override.
|
||||
func ParseModulesCSV(raw string) ([]string, error) {
|
||||
if raw == "" {
|
||||
return []string{}, nil
|
||||
}
|
||||
parts := strings.Split(raw, ",")
|
||||
modules := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
trimmed := strings.TrimSpace(part)
|
||||
if trimmed == "" {
|
||||
for i := range parts {
|
||||
parts[i] = strings.TrimSpace(parts[i])
|
||||
if parts[i] == "" {
|
||||
return nil, fmt.Errorf("modules list contains an empty value")
|
||||
}
|
||||
modules = append(modules, trimmed)
|
||||
}
|
||||
if len(modules) == 0 {
|
||||
return nil, fmt.Errorf("modules list must not be empty")
|
||||
}
|
||||
return modules, nil
|
||||
}
|
||||
|
||||
func (c Config) EffectiveValidationLLMConfig() LLMConfig {
|
||||
effective := c.PrimaryLLM
|
||||
|
||||
if c.ValidationLLM.APIKey != "" {
|
||||
effective.APIKey = c.ValidationLLM.APIKey
|
||||
}
|
||||
if c.ValidationLLM.Model != "" {
|
||||
effective.Model = c.ValidationLLM.Model
|
||||
}
|
||||
if c.ValidationLLM.BaseURL != "" {
|
||||
effective.BaseURL = c.ValidationLLM.BaseURL
|
||||
}
|
||||
if c.ValidationLLM.TimeoutSeconds != nil {
|
||||
effective.TimeoutSeconds = *c.ValidationLLM.TimeoutSeconds
|
||||
}
|
||||
if c.ValidationLLM.MaxRetries != nil {
|
||||
effective.MaxRetries = *c.ValidationLLM.MaxRetries
|
||||
}
|
||||
effective.Concurrency = c.EffectiveValidationLLMConcurrency()
|
||||
|
||||
return effective
|
||||
}
|
||||
|
||||
func (c Config) EffectiveValidationLLMConcurrency() int {
|
||||
if c.ValidationLLMConcurrency != nil {
|
||||
return *c.ValidationLLMConcurrency
|
||||
}
|
||||
return c.TotalLLMConcurrency
|
||||
}
|
||||
|
||||
func (c Config) EffectiveProposalLLMConcurrency() int {
|
||||
if c.ProposalLLMConcurrency > 0 {
|
||||
return c.ProposalLLMConcurrency
|
||||
}
|
||||
return c.TotalLLMConcurrency
|
||||
}
|
||||
|
||||
func (c *Config) syncLegacyConcurrencyAliases() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.PrimaryLLM.Concurrency = c.TotalLLMConcurrency
|
||||
c.ValidationLLM.Concurrency = intPtr(c.ValidationLLMConcurrency)
|
||||
}
|
||||
|
||||
func intPtr(v *int) *int {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
x := *v
|
||||
return &x
|
||||
return parts, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user