323 lines
9.7 KiB
Go
323 lines
9.7 KiB
Go
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
const SupportedFileConfigVersion = 1
|
|
|
|
var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
|
|
|
type FileConfig struct {
|
|
Version int `yaml:"version"`
|
|
Pipeline *FileConfigPipeline `yaml:"pipeline,omitempty"`
|
|
Output *FileConfigOutput `yaml:"output,omitempty"`
|
|
LLM *FileConfigLLM `yaml:"llm,omitempty"`
|
|
Concurrency *FileConfigConcurrency `yaml:"concurrency,omitempty"`
|
|
Chunking *FileConfigChunking `yaml:"chunking,omitempty"`
|
|
Normalization *FileConfigNormalization `yaml:"normalization,omitempty"`
|
|
Thresholds *FileConfigThresholds `yaml:"thresholds,omitempty"`
|
|
Context *FileConfigContext `yaml:"context,omitempty"`
|
|
Diagnostics *FileConfigDiagnostics `yaml:"diagnostics,omitempty"`
|
|
}
|
|
|
|
type FileConfigPipeline struct {
|
|
Modules []string `yaml:"modules,omitempty"`
|
|
}
|
|
|
|
type FileConfigOutput struct {
|
|
Schema *string `yaml:"schema,omitempty"`
|
|
}
|
|
|
|
type FileConfigLLM struct {
|
|
Proposal *FileConfigLLMTarget `yaml:"proposal,omitempty"`
|
|
Validation *FileConfigLLMTarget `yaml:"validation,omitempty"`
|
|
}
|
|
|
|
type FileConfigLLMTarget struct {
|
|
BaseURL *string `yaml:"base_url,omitempty"`
|
|
Model *string `yaml:"model,omitempty"`
|
|
APIKeyEnv *string `yaml:"api_key_env,omitempty"`
|
|
Timeout *fileConfigDurationOrInt `yaml:"timeout,omitempty"`
|
|
MaxRetries *int `yaml:"max_retries,omitempty"`
|
|
}
|
|
|
|
type FileConfigConcurrency struct {
|
|
TotalLLM *int `yaml:"total_llm,omitempty"`
|
|
ProposalLLM *int `yaml:"proposal_llm,omitempty"`
|
|
ValidationLLM *int `yaml:"validation_llm,omitempty"`
|
|
}
|
|
|
|
type FileConfigChunking struct {
|
|
TargetSections *int `yaml:"target_sections,omitempty"`
|
|
MaxSectionTokens *int `yaml:"max_section_tokens,omitempty"`
|
|
MinSectionTokens *int `yaml:"min_section_tokens,omitempty"`
|
|
}
|
|
|
|
type FileConfigNormalization struct {
|
|
MaxSegmentGap *fileConfigDurationOrFloat `yaml:"max_segment_gap,omitempty"`
|
|
EllipsisGap *fileConfigDurationOrFloat `yaml:"ellipsis_gap,omitempty"`
|
|
MaxSegmentDuration *fileConfigDurationOrFloat `yaml:"max_segment_duration,omitempty"`
|
|
MaxSegmentTokens *int `yaml:"max_segment_tokens,omitempty"`
|
|
}
|
|
|
|
type FileConfigThresholds struct {
|
|
Glossary *float64 `yaml:"glossary,omitempty"`
|
|
Homophones *float64 `yaml:"homophones,omitempty"`
|
|
SpokenWord *float64 `yaml:"spoken_word,omitempty"`
|
|
Grammar *float64 `yaml:"grammar,omitempty"`
|
|
}
|
|
|
|
type FileConfigContext struct {
|
|
Description *string `yaml:"description,omitempty"`
|
|
}
|
|
|
|
type FileConfigDiagnostics struct {
|
|
WorkDir *string `yaml:"work_dir,omitempty"`
|
|
Retention *string `yaml:"retention,omitempty"`
|
|
}
|
|
|
|
type fileConfigDurationOrInt struct {
|
|
seconds int
|
|
}
|
|
|
|
func (v *fileConfigDurationOrInt) UnmarshalYAML(node *yaml.Node) error {
|
|
switch node.Kind {
|
|
case yaml.ScalarNode:
|
|
if node.Tag == "!!int" {
|
|
var n int
|
|
if err := node.Decode(&n); err != nil {
|
|
return fmt.Errorf("must be an integer seconds value or duration string")
|
|
}
|
|
v.seconds = n
|
|
return nil
|
|
}
|
|
|
|
var s string
|
|
if err := node.Decode(&s); err != nil {
|
|
return fmt.Errorf("must be an integer seconds value or duration string")
|
|
}
|
|
d, err := time.ParseDuration(strings.TrimSpace(s))
|
|
if err != nil {
|
|
return fmt.Errorf("invalid duration %q", s)
|
|
}
|
|
if d <= 0 {
|
|
v.seconds = int(d / time.Second)
|
|
return nil
|
|
}
|
|
if d%time.Second != 0 {
|
|
return fmt.Errorf("duration %q must resolve to whole seconds", s)
|
|
}
|
|
v.seconds = int(d / time.Second)
|
|
return nil
|
|
default:
|
|
return fmt.Errorf("must be an integer seconds value or duration string")
|
|
}
|
|
}
|
|
|
|
func (v fileConfigDurationOrInt) Seconds() int { return v.seconds }
|
|
|
|
type fileConfigDurationOrFloat struct {
|
|
seconds float64
|
|
}
|
|
|
|
func (v *fileConfigDurationOrFloat) UnmarshalYAML(node *yaml.Node) error {
|
|
switch node.Kind {
|
|
case yaml.ScalarNode:
|
|
if node.Tag == "!!int" || node.Tag == "!!float" {
|
|
var f float64
|
|
if err := node.Decode(&f); err != nil {
|
|
return fmt.Errorf("must be a numeric seconds value or duration string")
|
|
}
|
|
v.seconds = f
|
|
return nil
|
|
}
|
|
var s string
|
|
if err := node.Decode(&s); err != nil {
|
|
return fmt.Errorf("must be a numeric seconds value or duration string")
|
|
}
|
|
d, err := time.ParseDuration(strings.TrimSpace(s))
|
|
if err != nil {
|
|
return fmt.Errorf("invalid duration %q", s)
|
|
}
|
|
v.seconds = d.Seconds()
|
|
return nil
|
|
default:
|
|
return fmt.Errorf("must be a numeric seconds value or duration string")
|
|
}
|
|
}
|
|
|
|
func (v fileConfigDurationOrFloat) Seconds() float64 { return v.seconds }
|
|
|
|
func LoadFileConfig(path string) (FileConfig, error) {
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return FileConfig{}, fmt.Errorf("read config file %q: %w", path, err)
|
|
}
|
|
cfg, err := ParseFileConfigYAML(b)
|
|
if err != nil {
|
|
return FileConfig{}, fmt.Errorf("parse config file %q: %w", path, err)
|
|
}
|
|
return cfg, nil
|
|
}
|
|
|
|
func ParseFileConfigYAML(data []byte) (FileConfig, error) {
|
|
var fileCfg FileConfig
|
|
dec := yaml.NewDecoder(strings.NewReader(string(data)))
|
|
dec.KnownFields(true)
|
|
if err := dec.Decode(&fileCfg); err != nil {
|
|
return FileConfig{}, fmt.Errorf("decode yaml: %w", err)
|
|
}
|
|
if fileCfg.Version == 0 {
|
|
return FileConfig{}, fmt.Errorf("config version is required")
|
|
}
|
|
if fileCfg.Version != SupportedFileConfigVersion {
|
|
return FileConfig{}, fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
|
}
|
|
return fileCfg, nil
|
|
}
|
|
|
|
func (c *Config) ApplyFileConfig(fileCfg FileConfig) error {
|
|
return c.applyFileConfigWithLookup(fileCfg, os.LookupEnv)
|
|
}
|
|
|
|
func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
|
|
if c == nil {
|
|
return fmt.Errorf("config must not be nil")
|
|
}
|
|
if fileCfg.Version != SupportedFileConfigVersion {
|
|
return fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
|
}
|
|
|
|
if fileCfg.Pipeline != nil && len(fileCfg.Pipeline.Modules) > 0 {
|
|
c.Modules = append([]string(nil), fileCfg.Pipeline.Modules...)
|
|
}
|
|
if fileCfg.Output != nil && fileCfg.Output.Schema != nil {
|
|
c.OutputSchema = strings.TrimSpace(*fileCfg.Output.Schema)
|
|
}
|
|
|
|
if fileCfg.LLM != nil {
|
|
if fileCfg.LLM.Proposal != nil {
|
|
patch := llmTargetPatch{
|
|
model: fileCfg.LLM.Proposal.Model,
|
|
baseURL: fileCfg.LLM.Proposal.BaseURL,
|
|
maxRetries: fileCfg.LLM.Proposal.MaxRetries,
|
|
}
|
|
if fileCfg.LLM.Proposal.Timeout != nil {
|
|
timeoutSeconds := fileCfg.LLM.Proposal.Timeout.Seconds()
|
|
patch.timeoutSeconds = &timeoutSeconds
|
|
}
|
|
if fileCfg.LLM.Proposal.APIKeyEnv != nil {
|
|
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Proposal.APIKeyEnv, lookup)
|
|
if err != nil {
|
|
return fmt.Errorf("llm.proposal.api_key_env: %w", err)
|
|
}
|
|
patch.apiKey = &apiKey
|
|
}
|
|
c.applyPrimaryLLMTargetPatch(patch)
|
|
}
|
|
if fileCfg.LLM.Validation != nil {
|
|
patch := llmTargetPatch{
|
|
model: fileCfg.LLM.Validation.Model,
|
|
baseURL: fileCfg.LLM.Validation.BaseURL,
|
|
maxRetries: fileCfg.LLM.Validation.MaxRetries,
|
|
}
|
|
if fileCfg.LLM.Validation.Timeout != nil {
|
|
timeoutSeconds := fileCfg.LLM.Validation.Timeout.Seconds()
|
|
patch.timeoutSeconds = &timeoutSeconds
|
|
}
|
|
if fileCfg.LLM.Validation.APIKeyEnv != nil {
|
|
apiKey, err := resolveAPIKeyEnv(*fileCfg.LLM.Validation.APIKeyEnv, lookup)
|
|
if err != nil {
|
|
return fmt.Errorf("llm.validation.api_key_env: %w", err)
|
|
}
|
|
patch.apiKey = &apiKey
|
|
}
|
|
c.applyValidationLLMTargetPatch(patch)
|
|
}
|
|
}
|
|
|
|
if fileCfg.Concurrency != nil {
|
|
c.applyConcurrencyPatch(concurrencyPatch{
|
|
totalLLM: fileCfg.Concurrency.TotalLLM,
|
|
proposalLLM: fileCfg.Concurrency.ProposalLLM,
|
|
validationLLM: fileCfg.Concurrency.ValidationLLM,
|
|
})
|
|
}
|
|
|
|
if fileCfg.Chunking != nil {
|
|
c.applyChunkingPatch(chunkingPatch{
|
|
targetSections: fileCfg.Chunking.TargetSections,
|
|
maxSectionTokens: fileCfg.Chunking.MaxSectionTokens,
|
|
minSectionTokens: fileCfg.Chunking.MinSectionTokens,
|
|
})
|
|
}
|
|
|
|
if fileCfg.Normalization != nil {
|
|
patch := normalizationPatch{
|
|
maxSegmentTokens: fileCfg.Normalization.MaxSegmentTokens,
|
|
}
|
|
if fileCfg.Normalization.MaxSegmentGap != nil {
|
|
maxSegmentGap := fileCfg.Normalization.MaxSegmentGap.Seconds()
|
|
patch.maxSegmentGap = &maxSegmentGap
|
|
}
|
|
if fileCfg.Normalization.EllipsisGap != nil {
|
|
ellipsisGap := fileCfg.Normalization.EllipsisGap.Seconds()
|
|
patch.ellipsisGap = &ellipsisGap
|
|
}
|
|
if fileCfg.Normalization.MaxSegmentDuration != nil {
|
|
maxSegmentDuration := fileCfg.Normalization.MaxSegmentDuration.Seconds()
|
|
patch.maxSegmentDuration = &maxSegmentDuration
|
|
}
|
|
c.applyNormalizationPatch(patch)
|
|
}
|
|
|
|
if fileCfg.Thresholds != nil {
|
|
c.applyThresholdsPatch(thresholdsPatch{
|
|
glossary: fileCfg.Thresholds.Glossary,
|
|
grammar: fileCfg.Thresholds.Grammar,
|
|
homophones: fileCfg.Thresholds.Homophones,
|
|
spokenWord: fileCfg.Thresholds.SpokenWord,
|
|
})
|
|
}
|
|
|
|
if fileCfg.Context != nil && fileCfg.Context.Description != nil {
|
|
c.applyContextPatch(contextPatch{transcriptDescription: fileCfg.Context.Description})
|
|
}
|
|
|
|
if fileCfg.Diagnostics != nil {
|
|
c.applyDiagnosticsPatch(diagnosticsPatch{
|
|
workDir: fileCfg.Diagnostics.WorkDir,
|
|
workDirRetention: fileCfg.Diagnostics.Retention,
|
|
})
|
|
}
|
|
|
|
c.syncLegacyConcurrencyAliases()
|
|
if err := c.Validate(); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
|
|
name := strings.TrimSpace(envName)
|
|
if name == "" {
|
|
return "", fmt.Errorf("must not be empty")
|
|
}
|
|
if !envVarNamePattern.MatchString(name) {
|
|
return "", fmt.Errorf("must be an environment variable name")
|
|
}
|
|
if strings.Contains(name, string(filepath.Separator)) {
|
|
return "", fmt.Errorf("must be an environment variable name")
|
|
}
|
|
v, _ := lookup(name)
|
|
return v, nil
|
|
}
|