Add versioned Audita config support
This commit is contained in:
334
internal/core/config/file_config.go
Normal file
334
internal/core/config/file_config.go
Normal file
@@ -0,0 +1,334 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const SupportedFileConfigVersion = 1
|
||||
|
||||
var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
type FileConfig struct {
|
||||
Version int `yaml:"version"`
|
||||
Pipeline *FileConfigPipeline `yaml:"pipeline,omitempty"`
|
||||
LLM *FileConfigLLM `yaml:"llm,omitempty"`
|
||||
Concurrency *FileConfigConcurrency `yaml:"concurrency,omitempty"`
|
||||
Chunking *FileConfigChunking `yaml:"chunking,omitempty"`
|
||||
Normalization *FileConfigNormalization `yaml:"normalization,omitempty"`
|
||||
Thresholds *FileConfigThresholds `yaml:"thresholds,omitempty"`
|
||||
Context *FileConfigContext `yaml:"context,omitempty"`
|
||||
Diagnostics *FileConfigDiagnostics `yaml:"diagnostics,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigPipeline struct {
|
||||
Modules []string `yaml:"modules,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigLLM struct {
|
||||
Proposal *FileConfigLLMTarget `yaml:"proposal,omitempty"`
|
||||
Validation *FileConfigLLMTarget `yaml:"validation,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigLLMTarget struct {
|
||||
BaseURL *string `yaml:"base_url,omitempty"`
|
||||
Model *string `yaml:"model,omitempty"`
|
||||
APIKeyEnv *string `yaml:"api_key_env,omitempty"`
|
||||
Timeout *fileConfigDurationOrInt `yaml:"timeout,omitempty"`
|
||||
MaxRetries *int `yaml:"max_retries,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigConcurrency struct {
|
||||
TotalLLM *int `yaml:"total_llm,omitempty"`
|
||||
ProposalLLM *int `yaml:"proposal_llm,omitempty"`
|
||||
ValidationLLM *int `yaml:"validation_llm,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigChunking struct {
|
||||
TargetSections *int `yaml:"target_sections,omitempty"`
|
||||
MaxSectionTokens *int `yaml:"max_section_tokens,omitempty"`
|
||||
MinSectionTokens *int `yaml:"min_section_tokens,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigNormalization struct {
|
||||
MaxSegmentGap *fileConfigDurationOrFloat `yaml:"max_segment_gap,omitempty"`
|
||||
EllipsisGap *fileConfigDurationOrFloat `yaml:"ellipsis_gap,omitempty"`
|
||||
MaxSegmentDuration *fileConfigDurationOrFloat `yaml:"max_segment_duration,omitempty"`
|
||||
MaxSegmentTokens *int `yaml:"max_segment_tokens,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigThresholds struct {
|
||||
Glossary *float64 `yaml:"glossary,omitempty"`
|
||||
Homophones *float64 `yaml:"homophones,omitempty"`
|
||||
SpokenWord *float64 `yaml:"spoken_word,omitempty"`
|
||||
Grammar *float64 `yaml:"grammar,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigContext struct {
|
||||
Description *string `yaml:"description,omitempty"`
|
||||
}
|
||||
|
||||
type FileConfigDiagnostics struct {
|
||||
WorkDir *string `yaml:"work_dir,omitempty"`
|
||||
Retention *string `yaml:"retention,omitempty"`
|
||||
}
|
||||
|
||||
type fileConfigDurationOrInt struct {
|
||||
seconds int
|
||||
}
|
||||
|
||||
func (v *fileConfigDurationOrInt) UnmarshalYAML(node *yaml.Node) error {
|
||||
switch node.Kind {
|
||||
case yaml.ScalarNode:
|
||||
if node.Tag == "!!int" {
|
||||
var n int
|
||||
if err := node.Decode(&n); err != nil {
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
v.seconds = n
|
||||
return nil
|
||||
}
|
||||
|
||||
var s string
|
||||
if err := node.Decode(&s); err != nil {
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
d, err := time.ParseDuration(strings.TrimSpace(s))
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid duration %q", s)
|
||||
}
|
||||
if d <= 0 {
|
||||
v.seconds = int(d / time.Second)
|
||||
return nil
|
||||
}
|
||||
if d%time.Second != 0 {
|
||||
return fmt.Errorf("duration %q must resolve to whole seconds", s)
|
||||
}
|
||||
v.seconds = int(d / time.Second)
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
}
|
||||
|
||||
func (v fileConfigDurationOrInt) Seconds() int { return v.seconds }
|
||||
|
||||
type fileConfigDurationOrFloat struct {
|
||||
seconds float64
|
||||
}
|
||||
|
||||
func (v *fileConfigDurationOrFloat) UnmarshalYAML(node *yaml.Node) error {
|
||||
switch node.Kind {
|
||||
case yaml.ScalarNode:
|
||||
if node.Tag == "!!int" || node.Tag == "!!float" {
|
||||
var f float64
|
||||
if err := node.Decode(&f); err != nil {
|
||||
return fmt.Errorf("must be a numeric seconds value or duration string")
|
||||
}
|
||||
v.seconds = f
|
||||
return nil
|
||||
}
|
||||
var s string
|
||||
if err := node.Decode(&s); err != nil {
|
||||
return fmt.Errorf("must be a numeric seconds value or duration string")
|
||||
}
|
||||
d, err := time.ParseDuration(strings.TrimSpace(s))
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid duration %q", s)
|
||||
}
|
||||
v.seconds = d.Seconds()
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("must be a numeric seconds value or duration string")
|
||||
}
|
||||
}
|
||||
|
||||
func (v fileConfigDurationOrFloat) Seconds() float64 { return v.seconds }
|
||||
|
||||
func LoadFileConfig(path string) (FileConfig, error) {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return FileConfig{}, fmt.Errorf("read config file %q: %w", path, err)
|
||||
}
|
||||
cfg, err := ParseFileConfigYAML(b)
|
||||
if err != nil {
|
||||
return FileConfig{}, fmt.Errorf("parse config file %q: %w", path, err)
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func ParseFileConfigYAML(data []byte) (FileConfig, error) {
|
||||
var fileCfg FileConfig
|
||||
dec := yaml.NewDecoder(strings.NewReader(string(data)))
|
||||
dec.KnownFields(true)
|
||||
if err := dec.Decode(&fileCfg); err != nil {
|
||||
return FileConfig{}, fmt.Errorf("decode yaml: %w", err)
|
||||
}
|
||||
if fileCfg.Version == 0 {
|
||||
return FileConfig{}, fmt.Errorf("config version is required")
|
||||
}
|
||||
if fileCfg.Version != SupportedFileConfigVersion {
|
||||
return FileConfig{}, fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
||||
}
|
||||
return fileCfg, nil
|
||||
}
|
||||
|
||||
func (c *Config) ApplyFileConfig(fileCfg FileConfig) error {
|
||||
return c.applyFileConfigWithLookup(fileCfg, os.LookupEnv)
|
||||
}
|
||||
|
||||
func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
|
||||
if c == nil {
|
||||
return fmt.Errorf("config must not be nil")
|
||||
}
|
||||
if fileCfg.Version != SupportedFileConfigVersion {
|
||||
return fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
||||
}
|
||||
|
||||
if fileCfg.Pipeline != nil && len(fileCfg.Pipeline.Modules) > 0 {
|
||||
c.Modules = append([]string(nil), fileCfg.Pipeline.Modules...)
|
||||
}
|
||||
|
||||
if fileCfg.LLM != nil {
|
||||
if fileCfg.LLM.Proposal != nil {
|
||||
if fileCfg.LLM.Proposal.BaseURL != nil {
|
||||
c.PrimaryLLM.BaseURL = *fileCfg.LLM.Proposal.BaseURL
|
||||
}
|
||||
if fileCfg.LLM.Proposal.Model != nil {
|
||||
c.PrimaryLLM.Model = *fileCfg.LLM.Proposal.Model
|
||||
}
|
||||
if fileCfg.LLM.Proposal.Timeout != nil {
|
||||
c.PrimaryLLM.TimeoutSeconds = fileCfg.LLM.Proposal.Timeout.Seconds()
|
||||
}
|
||||
if fileCfg.LLM.Proposal.MaxRetries != nil {
|
||||
c.PrimaryLLM.MaxRetries = *fileCfg.LLM.Proposal.MaxRetries
|
||||
}
|
||||
if fileCfg.LLM.Proposal.APIKeyEnv != nil {
|
||||
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Proposal.APIKeyEnv, lookup)
|
||||
if err != nil {
|
||||
return fmt.Errorf("llm.proposal.api_key_env: %w", err)
|
||||
}
|
||||
c.PrimaryLLM.APIKey = apiKey
|
||||
}
|
||||
}
|
||||
if fileCfg.LLM.Validation != nil {
|
||||
if fileCfg.LLM.Validation.BaseURL != nil {
|
||||
c.ValidationLLM.BaseURL = *fileCfg.LLM.Validation.BaseURL
|
||||
}
|
||||
if fileCfg.LLM.Validation.Model != nil {
|
||||
c.ValidationLLM.Model = *fileCfg.LLM.Validation.Model
|
||||
}
|
||||
if fileCfg.LLM.Validation.Timeout != nil {
|
||||
v := fileCfg.LLM.Validation.Timeout.Seconds()
|
||||
c.ValidationLLM.TimeoutSeconds = &v
|
||||
}
|
||||
if fileCfg.LLM.Validation.MaxRetries != nil {
|
||||
v := *fileCfg.LLM.Validation.MaxRetries
|
||||
c.ValidationLLM.MaxRetries = &v
|
||||
}
|
||||
if fileCfg.LLM.Validation.APIKeyEnv != nil {
|
||||
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Validation.APIKeyEnv, lookup)
|
||||
if err != nil {
|
||||
return fmt.Errorf("llm.validation.api_key_env: %w", err)
|
||||
}
|
||||
c.ValidationLLM.APIKey = apiKey
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if fileCfg.Concurrency != nil {
|
||||
if fileCfg.Concurrency.TotalLLM != nil {
|
||||
c.TotalLLMConcurrency = *fileCfg.Concurrency.TotalLLM
|
||||
}
|
||||
if fileCfg.Concurrency.ProposalLLM != nil {
|
||||
c.ProposalLLMConcurrency = *fileCfg.Concurrency.ProposalLLM
|
||||
}
|
||||
if fileCfg.Concurrency.ValidationLLM != nil {
|
||||
v := *fileCfg.Concurrency.ValidationLLM
|
||||
c.ValidationLLMConcurrency = &v
|
||||
}
|
||||
}
|
||||
|
||||
if fileCfg.Chunking != nil {
|
||||
if fileCfg.Chunking.TargetSections != nil {
|
||||
v := *fileCfg.Chunking.TargetSections
|
||||
c.TargetSections = &v
|
||||
}
|
||||
if fileCfg.Chunking.MaxSectionTokens != nil {
|
||||
c.MaxSectionTokens = *fileCfg.Chunking.MaxSectionTokens
|
||||
}
|
||||
if fileCfg.Chunking.MinSectionTokens != nil {
|
||||
c.MinSectionTokens = *fileCfg.Chunking.MinSectionTokens
|
||||
}
|
||||
}
|
||||
|
||||
if fileCfg.Normalization != nil {
|
||||
if fileCfg.Normalization.MaxSegmentGap != nil {
|
||||
c.Normalization.MaxSegmentGap = fileCfg.Normalization.MaxSegmentGap.Seconds()
|
||||
}
|
||||
if fileCfg.Normalization.EllipsisGap != nil {
|
||||
c.Normalization.EllipsisGap = fileCfg.Normalization.EllipsisGap.Seconds()
|
||||
}
|
||||
if fileCfg.Normalization.MaxSegmentDuration != nil {
|
||||
c.Normalization.MaxSegmentDuration = fileCfg.Normalization.MaxSegmentDuration.Seconds()
|
||||
}
|
||||
if fileCfg.Normalization.MaxSegmentTokens != nil {
|
||||
c.Normalization.MaxSegmentTokens = *fileCfg.Normalization.MaxSegmentTokens
|
||||
}
|
||||
}
|
||||
|
||||
if fileCfg.Thresholds != nil {
|
||||
if fileCfg.Thresholds.Glossary != nil {
|
||||
c.Thresholds.Glossary = *fileCfg.Thresholds.Glossary
|
||||
}
|
||||
if fileCfg.Thresholds.Homophones != nil {
|
||||
c.Thresholds.Homophones = *fileCfg.Thresholds.Homophones
|
||||
}
|
||||
if fileCfg.Thresholds.SpokenWord != nil {
|
||||
c.Thresholds.SpokenWord = *fileCfg.Thresholds.SpokenWord
|
||||
}
|
||||
if fileCfg.Thresholds.Grammar != nil {
|
||||
c.Thresholds.Grammar = *fileCfg.Thresholds.Grammar
|
||||
}
|
||||
}
|
||||
|
||||
if fileCfg.Context != nil && fileCfg.Context.Description != nil {
|
||||
c.TranscriptDescription = strings.TrimSpace(*fileCfg.Context.Description)
|
||||
}
|
||||
|
||||
if fileCfg.Diagnostics != nil {
|
||||
if fileCfg.Diagnostics.WorkDir != nil {
|
||||
c.WorkDir = *fileCfg.Diagnostics.WorkDir
|
||||
}
|
||||
if fileCfg.Diagnostics.Retention != nil {
|
||||
c.WorkDirRetention = WorkDirRetention(*fileCfg.Diagnostics.Retention)
|
||||
}
|
||||
}
|
||||
|
||||
c.syncLegacyConcurrencyAliases()
|
||||
if err := c.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
|
||||
name := strings.TrimSpace(envName)
|
||||
if name == "" {
|
||||
return "", fmt.Errorf("must not be empty")
|
||||
}
|
||||
if !envVarNamePattern.MatchString(name) {
|
||||
return "", fmt.Errorf("must be an environment variable name")
|
||||
}
|
||||
if strings.Contains(name, string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("must be an environment variable name")
|
||||
}
|
||||
v, _ := lookup(name)
|
||||
return v, nil
|
||||
}
|
||||
Reference in New Issue
Block a user