Switch config to Scriptorium profiles
This commit is contained in:
@@ -4,34 +4,25 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
type FileConfig struct {
|
||||
Version int `yaml:"version"`
|
||||
LLMProfiles map[string]FileLLMProfile `yaml:"llm_profiles,omitempty"`
|
||||
Scriptorium *FileScriptoriumConfig `yaml:"scriptorium,omitempty"`
|
||||
Pipelines map[string]FilePipelineProfile `yaml:"pipelines,omitempty"`
|
||||
Concurrency *FileConcurrencyConfig `yaml:"concurrency,omitempty"`
|
||||
Diagnostics *FileDiagnosticsConfig `yaml:"diagnostics,omitempty"`
|
||||
}
|
||||
|
||||
type FileLLMProfile struct {
|
||||
Provider *string `yaml:"provider,omitempty"`
|
||||
BaseURL *string `yaml:"base_url,omitempty"`
|
||||
Model *string `yaml:"model,omitempty"`
|
||||
APIKeyEnv *string `yaml:"api_key_env,omitempty"`
|
||||
Timeout *fileDurationSeconds `yaml:"timeout,omitempty"`
|
||||
MaxRetries *int `yaml:"max_retries,omitempty"`
|
||||
MaxConcurrency *int `yaml:"max_concurrency,omitempty"`
|
||||
type FileScriptoriumConfig struct {
|
||||
ProfileDir *string `yaml:"profile_dir,omitempty"`
|
||||
ProfileFile *string `yaml:"profile_file,omitempty"`
|
||||
}
|
||||
|
||||
type FilePipelineProfile struct {
|
||||
@@ -59,42 +50,6 @@ type FileDiagnosticsConfig struct {
|
||||
Retention *string `yaml:"retention,omitempty"`
|
||||
}
|
||||
|
||||
type fileDurationSeconds struct {
|
||||
seconds int
|
||||
}
|
||||
|
||||
func (d *fileDurationSeconds) UnmarshalYAML(node *yaml.Node) error {
|
||||
if node.Kind != yaml.ScalarNode {
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
if node.Tag == "!!int" {
|
||||
var seconds int
|
||||
if err := node.Decode(&seconds); err != nil {
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
d.seconds = seconds
|
||||
return nil
|
||||
}
|
||||
|
||||
var raw string
|
||||
if err := node.Decode(&raw); err != nil {
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
duration, err := time.ParseDuration(strings.TrimSpace(raw))
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid duration %q", raw)
|
||||
}
|
||||
if duration%time.Second != 0 {
|
||||
return fmt.Errorf("duration %q must resolve to whole seconds", raw)
|
||||
}
|
||||
d.seconds = int(duration / time.Second)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d fileDurationSeconds) Seconds() int {
|
||||
return d.seconds
|
||||
}
|
||||
|
||||
type fileModuleBinding struct {
|
||||
Module string
|
||||
LLMProfile string
|
||||
@@ -196,23 +151,17 @@ func (c *Config) ApplyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
}
|
||||
|
||||
func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
|
||||
_ = lookup
|
||||
if c == nil {
|
||||
return fmt.Errorf("config must not be nil")
|
||||
}
|
||||
if fileCfg.Version != SupportedFileConfigVersion {
|
||||
return fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
||||
}
|
||||
if c.LLMProfiles == nil {
|
||||
c.LLMProfiles = map[string]LLMProfile{}
|
||||
}
|
||||
if c.Pipelines == nil {
|
||||
c.Pipelines = map[string]pipeline.PipelineProfile{}
|
||||
}
|
||||
|
||||
profileIDs, rawLLMProfileIDs, err := normalizedMapKeys(fileCfg.LLMProfiles, "llm profile id")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pipelineIDs, rawPipelineIDs, err := normalizedMapKeys(fileCfg.Pipelines, "pipeline id")
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -267,36 +216,21 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
}
|
||||
}
|
||||
|
||||
for _, profileID := range profileIDs {
|
||||
fileProfile := fileCfg.LLMProfiles[rawLLMProfileIDs[profileID]]
|
||||
profile := c.LLMProfiles[profileID]
|
||||
if fileProfile.Provider != nil {
|
||||
profile.Provider = strings.TrimSpace(*fileProfile.Provider)
|
||||
}
|
||||
if fileProfile.BaseURL != nil {
|
||||
profile.BaseURL = strings.TrimSpace(*fileProfile.BaseURL)
|
||||
}
|
||||
if fileProfile.Model != nil {
|
||||
profile.Model = strings.TrimSpace(*fileProfile.Model)
|
||||
}
|
||||
if fileProfile.APIKeyEnv != nil {
|
||||
apiKey, err := resolveAPIKeyEnv(*fileProfile.APIKeyEnv, lookup)
|
||||
if err != nil {
|
||||
return fmt.Errorf("llm_profiles.%s.api_key_env: %w", profileID, err)
|
||||
if fileCfg.Scriptorium != nil {
|
||||
if fileCfg.Scriptorium.ProfileDir != nil {
|
||||
value := strings.TrimSpace(*fileCfg.Scriptorium.ProfileDir)
|
||||
if value == "" {
|
||||
return fmt.Errorf("scriptorium.profile_dir must not be empty when set")
|
||||
}
|
||||
profile.APIKeyEnv = strings.TrimSpace(*fileProfile.APIKeyEnv)
|
||||
profile.APIKey = apiKey
|
||||
c.Scriptorium.ProfileDir = value
|
||||
}
|
||||
if fileProfile.Timeout != nil {
|
||||
profile.TimeoutSeconds = fileProfile.Timeout.Seconds()
|
||||
if fileCfg.Scriptorium.ProfileFile != nil {
|
||||
value := strings.TrimSpace(*fileCfg.Scriptorium.ProfileFile)
|
||||
if value == "" {
|
||||
return fmt.Errorf("scriptorium.profile_file must not be empty when set")
|
||||
}
|
||||
c.Scriptorium.ProfileFile = value
|
||||
}
|
||||
if fileProfile.MaxRetries != nil {
|
||||
profile.MaxRetries = *fileProfile.MaxRetries
|
||||
}
|
||||
if fileProfile.MaxConcurrency != nil {
|
||||
profile.MaxConcurrency = *fileProfile.MaxConcurrency
|
||||
}
|
||||
c.LLMProfiles[profileID] = profile
|
||||
}
|
||||
|
||||
for _, pipelineID := range pipelineIDs {
|
||||
@@ -408,21 +342,6 @@ func mergeStringMaps(base map[string]string, override map[string]string) map[str
|
||||
return out
|
||||
}
|
||||
|
||||
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
|
||||
name := strings.TrimSpace(envName)
|
||||
if name == "" {
|
||||
return "", fmt.Errorf("must not be empty")
|
||||
}
|
||||
if !envVarNamePattern.MatchString(name) {
|
||||
return "", fmt.Errorf("must be an environment variable name")
|
||||
}
|
||||
value, ok := lookup(name)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%s is not set", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func normalizeOptions(options map[string]any) map[string]any {
|
||||
if len(options) == 0 {
|
||||
return nil
|
||||
|
||||
Reference in New Issue
Block a user