313 lines
9.3 KiB
Go
313 lines
9.3 KiB
Go
package config
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"os"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
|
|
|
type FileConfig struct {
|
|
Version int `yaml:"version"`
|
|
LLMProfiles map[string]FileLLMProfile `yaml:"llm_profiles,omitempty"`
|
|
Pipelines map[string]FilePipelineProfile `yaml:"pipelines,omitempty"`
|
|
Concurrency *FileConcurrencyConfig `yaml:"concurrency,omitempty"`
|
|
Diagnostics *FileDiagnosticsConfig `yaml:"diagnostics,omitempty"`
|
|
}
|
|
|
|
type FileLLMProfile struct {
|
|
Provider *string `yaml:"provider,omitempty"`
|
|
BaseURL *string `yaml:"base_url,omitempty"`
|
|
Model *string `yaml:"model,omitempty"`
|
|
APIKeyEnv *string `yaml:"api_key_env,omitempty"`
|
|
Timeout *fileDurationSeconds `yaml:"timeout,omitempty"`
|
|
MaxRetries *int `yaml:"max_retries,omitempty"`
|
|
MaxConcurrency *int `yaml:"max_concurrency,omitempty"`
|
|
}
|
|
|
|
type FilePipelineProfile struct {
|
|
Input fileModuleBinding `yaml:"input"`
|
|
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
|
|
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
|
|
Output *fileModuleBinding `yaml:"output,omitempty"`
|
|
}
|
|
|
|
type FileArtifactLaneProfile struct {
|
|
Extract fileModuleBinding `yaml:"extract"`
|
|
Merge *fileModuleBinding `yaml:"merge,omitempty"`
|
|
Normalize *fileModuleBinding `yaml:"normalize,omitempty"`
|
|
Validators []fileModuleBinding `yaml:"validators,omitempty"`
|
|
}
|
|
|
|
type FileConcurrencyConfig struct {
|
|
TotalLLM *int `yaml:"total_llm,omitempty"`
|
|
}
|
|
|
|
type FileDiagnosticsConfig struct {
|
|
WorkDir *string `yaml:"work_dir,omitempty"`
|
|
Retention *string `yaml:"retention,omitempty"`
|
|
}
|
|
|
|
type fileDurationSeconds struct {
|
|
seconds int
|
|
}
|
|
|
|
func (d *fileDurationSeconds) UnmarshalYAML(node *yaml.Node) error {
|
|
if node.Kind != yaml.ScalarNode {
|
|
return fmt.Errorf("must be an integer seconds value or duration string")
|
|
}
|
|
if node.Tag == "!!int" {
|
|
var seconds int
|
|
if err := node.Decode(&seconds); err != nil {
|
|
return fmt.Errorf("must be an integer seconds value or duration string")
|
|
}
|
|
d.seconds = seconds
|
|
return nil
|
|
}
|
|
|
|
var raw string
|
|
if err := node.Decode(&raw); err != nil {
|
|
return fmt.Errorf("must be an integer seconds value or duration string")
|
|
}
|
|
duration, err := time.ParseDuration(strings.TrimSpace(raw))
|
|
if err != nil {
|
|
return fmt.Errorf("invalid duration %q", raw)
|
|
}
|
|
if duration%time.Second != 0 {
|
|
return fmt.Errorf("duration %q must resolve to whole seconds", raw)
|
|
}
|
|
d.seconds = int(duration / time.Second)
|
|
return nil
|
|
}
|
|
|
|
func (d fileDurationSeconds) Seconds() int {
|
|
return d.seconds
|
|
}
|
|
|
|
type fileModuleBinding struct {
|
|
Module string
|
|
LLMProfile string
|
|
Options map[string]any
|
|
}
|
|
|
|
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
|
switch node.Kind {
|
|
case yaml.ScalarNode:
|
|
var module string
|
|
if err := node.Decode(&module); err != nil {
|
|
return fmt.Errorf("module binding must be a string or object")
|
|
}
|
|
b.Module = strings.TrimSpace(module)
|
|
return nil
|
|
case yaml.MappingNode:
|
|
for i := 0; i < len(node.Content); i += 2 {
|
|
keyNode := node.Content[i]
|
|
valueNode := node.Content[i+1]
|
|
switch keyNode.Value {
|
|
case "module":
|
|
var module string
|
|
if err := valueNode.Decode(&module); err != nil {
|
|
return err
|
|
}
|
|
b.Module = strings.TrimSpace(module)
|
|
case "llm_profile":
|
|
var llmProfile string
|
|
if err := valueNode.Decode(&llmProfile); err != nil {
|
|
return err
|
|
}
|
|
b.LLMProfile = strings.TrimSpace(llmProfile)
|
|
case "options":
|
|
var options map[string]any
|
|
if err := valueNode.Decode(&options); err != nil {
|
|
return err
|
|
}
|
|
b.Options = normalizeOptions(options)
|
|
default:
|
|
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
|
|
}
|
|
}
|
|
return nil
|
|
default:
|
|
return fmt.Errorf("module binding must be a string or object")
|
|
}
|
|
}
|
|
|
|
func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
|
|
return pipeline.ModuleBinding{
|
|
Module: strings.TrimSpace(b.Module),
|
|
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
|
Options: cloneOptions(b.Options),
|
|
}
|
|
}
|
|
|
|
func LoadFileConfig(path string) (FileConfig, error) {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return FileConfig{}, fmt.Errorf("read config file %q: %w", path, err)
|
|
}
|
|
cfg, err := ParseFileConfigYAML(data)
|
|
if err != nil {
|
|
return FileConfig{}, fmt.Errorf("parse config file %q: %w", path, err)
|
|
}
|
|
return cfg, nil
|
|
}
|
|
|
|
func ParseFileConfigYAML(data []byte) (FileConfig, error) {
|
|
var fileCfg FileConfig
|
|
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
|
decoder.KnownFields(true)
|
|
if err := decoder.Decode(&fileCfg); err != nil {
|
|
return FileConfig{}, fmt.Errorf("decode yaml: %w", err)
|
|
}
|
|
if fileCfg.Version == 0 {
|
|
return FileConfig{}, fmt.Errorf("config version is required")
|
|
}
|
|
if fileCfg.Version != SupportedFileConfigVersion {
|
|
return FileConfig{}, fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
|
}
|
|
return fileCfg, nil
|
|
}
|
|
|
|
func (c *Config) ApplyFileConfig(fileCfg FileConfig) error {
|
|
return c.applyFileConfigWithLookup(fileCfg, os.LookupEnv)
|
|
}
|
|
|
|
func (c *Config) ApplyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
|
|
return c.applyFileConfigWithLookup(fileCfg, lookup)
|
|
}
|
|
|
|
func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
|
|
if c == nil {
|
|
return fmt.Errorf("config must not be nil")
|
|
}
|
|
if fileCfg.Version != SupportedFileConfigVersion {
|
|
return fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
|
}
|
|
if c.LLMProfiles == nil {
|
|
c.LLMProfiles = map[string]LLMProfile{}
|
|
}
|
|
if c.Pipelines == nil {
|
|
c.Pipelines = map[string]pipeline.PipelineProfile{}
|
|
}
|
|
|
|
for rawID, fileProfile := range fileCfg.LLMProfiles {
|
|
profileID := strings.TrimSpace(rawID)
|
|
if profileID == "" {
|
|
return fmt.Errorf("llm profile id must not be empty")
|
|
}
|
|
profile := c.LLMProfiles[profileID]
|
|
if fileProfile.Provider != nil {
|
|
profile.Provider = strings.TrimSpace(*fileProfile.Provider)
|
|
}
|
|
if fileProfile.BaseURL != nil {
|
|
profile.BaseURL = strings.TrimSpace(*fileProfile.BaseURL)
|
|
}
|
|
if fileProfile.Model != nil {
|
|
profile.Model = strings.TrimSpace(*fileProfile.Model)
|
|
}
|
|
if fileProfile.APIKeyEnv != nil {
|
|
apiKey, err := resolveAPIKeyEnv(*fileProfile.APIKeyEnv, lookup)
|
|
if err != nil {
|
|
return fmt.Errorf("llm_profiles.%s.api_key_env: %w", profileID, err)
|
|
}
|
|
profile.APIKeyEnv = strings.TrimSpace(*fileProfile.APIKeyEnv)
|
|
profile.APIKey = apiKey
|
|
}
|
|
if fileProfile.Timeout != nil {
|
|
profile.TimeoutSeconds = fileProfile.Timeout.Seconds()
|
|
}
|
|
if fileProfile.MaxRetries != nil {
|
|
profile.MaxRetries = *fileProfile.MaxRetries
|
|
}
|
|
if fileProfile.MaxConcurrency != nil {
|
|
profile.MaxConcurrency = *fileProfile.MaxConcurrency
|
|
}
|
|
c.LLMProfiles[profileID] = profile
|
|
}
|
|
|
|
for rawID, filePipeline := range fileCfg.Pipelines {
|
|
pipelineID := strings.TrimSpace(rawID)
|
|
if pipelineID == "" {
|
|
return fmt.Errorf("pipeline id must not be empty")
|
|
}
|
|
profile := pipeline.PipelineProfile{
|
|
ID: pipelineID,
|
|
Input: filePipeline.Input.toPipelineBinding(),
|
|
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
|
|
}
|
|
if filePipeline.Chunk != nil {
|
|
profile.Chunk = filePipeline.Chunk.toPipelineBinding()
|
|
}
|
|
if filePipeline.Output != nil {
|
|
profile.Output = filePipeline.Output.toPipelineBinding()
|
|
}
|
|
for rawLaneID, fileLane := range filePipeline.Artifacts {
|
|
laneID := strings.TrimSpace(rawLaneID)
|
|
if laneID == "" {
|
|
return fmt.Errorf("pipeline %q artifact lane id must not be empty", pipelineID)
|
|
}
|
|
lane := pipeline.ArtifactLaneProfile{
|
|
Extract: fileLane.Extract.toPipelineBinding(),
|
|
}
|
|
if fileLane.Merge != nil {
|
|
lane.Merge = fileLane.Merge.toPipelineBinding()
|
|
}
|
|
if fileLane.Normalize != nil {
|
|
lane.Normalize = fileLane.Normalize.toPipelineBinding()
|
|
}
|
|
if len(fileLane.Validators) > 0 {
|
|
lane.Validators = make([]pipeline.ModuleBinding, len(fileLane.Validators))
|
|
for i, validator := range fileLane.Validators {
|
|
lane.Validators[i] = validator.toPipelineBinding()
|
|
}
|
|
}
|
|
profile.Artifacts[laneID] = lane
|
|
}
|
|
c.Pipelines[pipelineID] = profile
|
|
}
|
|
|
|
if fileCfg.Concurrency != nil && fileCfg.Concurrency.TotalLLM != nil {
|
|
c.Concurrency.TotalLLM = *fileCfg.Concurrency.TotalLLM
|
|
}
|
|
if fileCfg.Diagnostics != nil {
|
|
if fileCfg.Diagnostics.WorkDir != nil {
|
|
c.Diagnostics.WorkDir = strings.TrimSpace(*fileCfg.Diagnostics.WorkDir)
|
|
}
|
|
if fileCfg.Diagnostics.Retention != nil {
|
|
c.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(*fileCfg.Diagnostics.Retention))
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
|
|
name := strings.TrimSpace(envName)
|
|
if name == "" {
|
|
return "", fmt.Errorf("must not be empty")
|
|
}
|
|
if !envVarNamePattern.MatchString(name) {
|
|
return "", fmt.Errorf("must be an environment variable name")
|
|
}
|
|
value, ok := lookup(name)
|
|
if !ok {
|
|
return "", fmt.Errorf("%s is not set", name)
|
|
}
|
|
return value, nil
|
|
}
|
|
|
|
func normalizeOptions(options map[string]any) map[string]any {
|
|
if len(options) == 0 {
|
|
return nil
|
|
}
|
|
return cloneOptions(options)
|
|
}
|