410 lines
13 KiB
Go
410 lines
13 KiB
Go
package config
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
type FileConfig struct {
|
|
Version int `yaml:"version"`
|
|
Scriptorium *FileScriptoriumConfig `yaml:"scriptorium,omitempty"`
|
|
Pipelines map[string]FilePipelineProfile `yaml:"pipelines,omitempty"`
|
|
Concurrency *FileConcurrencyConfig `yaml:"concurrency,omitempty"`
|
|
Diagnostics *FileDiagnosticsConfig `yaml:"diagnostics,omitempty"`
|
|
Workspace *FileWorkspaceConfig `yaml:"workspace,omitempty"`
|
|
}
|
|
|
|
type FileScriptoriumConfig struct {
|
|
ProfileDir *string `yaml:"profile_dir,omitempty"`
|
|
ProfileFile *string `yaml:"profile_file,omitempty"`
|
|
}
|
|
|
|
type FilePipelineProfile struct {
|
|
Input fileModuleBinding `yaml:"input"`
|
|
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
|
|
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
|
|
Output *fileModuleBinding `yaml:"output,omitempty"`
|
|
References map[string]string `yaml:"references,omitempty"`
|
|
}
|
|
|
|
type FileArtifactLaneProfile struct {
|
|
Extract fileModuleBinding `yaml:"extract"`
|
|
Merge *fileModuleBinding `yaml:"merge,omitempty"`
|
|
Normalize *fileModuleBinding `yaml:"normalize,omitempty"`
|
|
Validators []fileModuleBinding `yaml:"validators,omitempty"`
|
|
References map[string]string `yaml:"references,omitempty"`
|
|
}
|
|
|
|
type FileConcurrencyConfig struct {
|
|
TotalLLM *int `yaml:"total_llm,omitempty"`
|
|
}
|
|
|
|
type FileDiagnosticsConfig struct {
|
|
WorkDir *string `yaml:"work_dir,omitempty"`
|
|
Retention *string `yaml:"retention,omitempty"`
|
|
}
|
|
|
|
type FileWorkspaceConfig struct {
|
|
Directory *string `yaml:"directory,omitempty"`
|
|
Diagnostics *FileWorkspaceDiagnosticsConfig `yaml:"diagnostics,omitempty"`
|
|
Resume *FileWorkspaceEnabledConfig `yaml:"resume,omitempty"`
|
|
Debug *FileWorkspaceEnabledConfig `yaml:"debug,omitempty"`
|
|
}
|
|
|
|
type FileWorkspaceDiagnosticsConfig struct {
|
|
Enabled *bool `yaml:"enabled,omitempty"`
|
|
Retention *string `yaml:"retention,omitempty"`
|
|
}
|
|
|
|
type FileWorkspaceEnabledConfig struct {
|
|
Enabled *bool `yaml:"enabled,omitempty"`
|
|
}
|
|
|
|
type fileModuleBinding struct {
|
|
Module string
|
|
LLMProfile string
|
|
Retries int
|
|
Options map[string]any
|
|
References map[string]string
|
|
Validators pipeline.ValidatorOverride
|
|
}
|
|
|
|
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
|
switch node.Kind {
|
|
case yaml.ScalarNode:
|
|
var module string
|
|
if err := node.Decode(&module); err != nil {
|
|
return fmt.Errorf("module binding must be a string or object")
|
|
}
|
|
b.Module = strings.TrimSpace(module)
|
|
return nil
|
|
case yaml.MappingNode:
|
|
for i := 0; i < len(node.Content); i += 2 {
|
|
keyNode := node.Content[i]
|
|
valueNode := node.Content[i+1]
|
|
switch keyNode.Value {
|
|
case "module":
|
|
var module string
|
|
if err := valueNode.Decode(&module); err != nil {
|
|
return err
|
|
}
|
|
b.Module = strings.TrimSpace(module)
|
|
case "llm_profile":
|
|
var llmProfile string
|
|
if err := valueNode.Decode(&llmProfile); err != nil {
|
|
return err
|
|
}
|
|
b.LLMProfile = strings.TrimSpace(llmProfile)
|
|
case "retries":
|
|
var retries int
|
|
if err := valueNode.Decode(&retries); err != nil {
|
|
return err
|
|
}
|
|
b.Retries = retries
|
|
case "options":
|
|
var options map[string]any
|
|
if err := valueNode.Decode(&options); err != nil {
|
|
return err
|
|
}
|
|
b.Options = normalizeOptions(options)
|
|
case "references":
|
|
var references map[string]string
|
|
if err := valueNode.Decode(&references); err != nil {
|
|
return err
|
|
}
|
|
b.References = references
|
|
case "validators":
|
|
b.Validators.Set = true
|
|
var validators []fileModuleBinding
|
|
if err := valueNode.Decode(&validators); err != nil {
|
|
return err
|
|
}
|
|
b.Validators.Validators = make([]pipeline.ModuleBinding, len(validators))
|
|
for i, validator := range validators {
|
|
b.Validators.Validators[i] = validator.toPipelineBinding()
|
|
}
|
|
default:
|
|
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
|
|
}
|
|
}
|
|
return nil
|
|
default:
|
|
return fmt.Errorf("module binding must be a string or object")
|
|
}
|
|
}
|
|
|
|
func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
|
|
return pipeline.ModuleBinding{
|
|
Module: strings.TrimSpace(b.Module),
|
|
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
|
Retries: b.Retries,
|
|
Options: cloneOptions(b.Options),
|
|
References: normalizedStringMap(b.References),
|
|
Validators: b.Validators,
|
|
}
|
|
}
|
|
|
|
func LoadFileConfig(path string) (FileConfig, error) {
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return FileConfig{}, fmt.Errorf("read config file %q: %w", path, err)
|
|
}
|
|
cfg, err := ParseFileConfigYAML(data)
|
|
if err != nil {
|
|
return FileConfig{}, fmt.Errorf("parse config file %q: %w", path, err)
|
|
}
|
|
return cfg, nil
|
|
}
|
|
|
|
func ParseFileConfigYAML(data []byte) (FileConfig, error) {
|
|
var fileCfg FileConfig
|
|
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
|
decoder.KnownFields(true)
|
|
if err := decoder.Decode(&fileCfg); err != nil {
|
|
return FileConfig{}, fmt.Errorf("decode yaml: %w", err)
|
|
}
|
|
if fileCfg.Version == 0 {
|
|
return FileConfig{}, fmt.Errorf("config version is required")
|
|
}
|
|
if fileCfg.Version != SupportedFileConfigVersion {
|
|
return FileConfig{}, fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
|
}
|
|
return fileCfg, nil
|
|
}
|
|
|
|
func (c *Config) ApplyFileConfig(fileCfg FileConfig) error {
|
|
return c.applyFileConfigWithLookup(fileCfg, os.LookupEnv)
|
|
}
|
|
|
|
func (c *Config) ApplyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
|
|
return c.applyFileConfigWithLookup(fileCfg, lookup)
|
|
}
|
|
|
|
func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
|
|
_ = lookup
|
|
if c == nil {
|
|
return fmt.Errorf("config must not be nil")
|
|
}
|
|
if fileCfg.Version != SupportedFileConfigVersion {
|
|
return fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
|
}
|
|
if c.Pipelines == nil {
|
|
c.Pipelines = map[string]pipeline.PipelineProfile{}
|
|
}
|
|
|
|
pipelineIDs, rawPipelineIDs, err := normalizedMapKeys(fileCfg.Pipelines, "pipeline id")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, pipelineID := range pipelineIDs {
|
|
filePipeline := fileCfg.Pipelines[rawPipelineIDs[pipelineID]]
|
|
if _, _, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)); err != nil {
|
|
return err
|
|
}
|
|
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
|
|
return err
|
|
}
|
|
if filePipeline.Chunk != nil {
|
|
if _, _, err := normalizedMapKeys(filePipeline.Chunk.References, fmt.Sprintf("pipeline %q chunk reference slot", pipelineID)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if _, _, err := normalizedMapKeys(filePipeline.Input.References, fmt.Sprintf("pipeline %q input reference slot", pipelineID)); err != nil {
|
|
return err
|
|
}
|
|
if filePipeline.Output != nil {
|
|
if _, _, err := normalizedMapKeys(filePipeline.Output.References, fmt.Sprintf("pipeline %q output reference slot", pipelineID)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
for rawLaneID, fileLane := range filePipeline.Artifacts {
|
|
laneID := strings.TrimSpace(rawLaneID)
|
|
if laneID == "" {
|
|
continue
|
|
}
|
|
if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
|
|
return err
|
|
}
|
|
if _, _, err := normalizedMapKeys(fileLane.Extract.References, fmt.Sprintf("pipeline %q lane %q extract reference slot", pipelineID, laneID)); err != nil {
|
|
return err
|
|
}
|
|
if fileLane.Merge != nil {
|
|
if _, _, err := normalizedMapKeys(fileLane.Merge.References, fmt.Sprintf("pipeline %q lane %q merge reference slot", pipelineID, laneID)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if fileLane.Normalize != nil {
|
|
if _, _, err := normalizedMapKeys(fileLane.Normalize.References, fmt.Sprintf("pipeline %q lane %q normalize reference slot", pipelineID, laneID)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
for i, validator := range fileLane.Validators {
|
|
if _, _, err := normalizedMapKeys(validator.References, fmt.Sprintf("pipeline %q lane %q validator[%d] reference slot", pipelineID, laneID, i)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if fileCfg.Scriptorium != nil {
|
|
if fileCfg.Scriptorium.ProfileDir != nil {
|
|
value := strings.TrimSpace(*fileCfg.Scriptorium.ProfileDir)
|
|
if value == "" {
|
|
return fmt.Errorf("scriptorium.profile_dir must not be empty when set")
|
|
}
|
|
c.Scriptorium.ProfileDir = value
|
|
}
|
|
if fileCfg.Scriptorium.ProfileFile != nil {
|
|
value := strings.TrimSpace(*fileCfg.Scriptorium.ProfileFile)
|
|
if value == "" {
|
|
return fmt.Errorf("scriptorium.profile_file must not be empty when set")
|
|
}
|
|
c.Scriptorium.ProfileFile = value
|
|
}
|
|
}
|
|
|
|
for _, pipelineID := range pipelineIDs {
|
|
filePipeline := fileCfg.Pipelines[rawPipelineIDs[pipelineID]]
|
|
laneIDs, rawLaneIDs, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
profile := pipeline.PipelineProfile{
|
|
ID: pipelineID,
|
|
Input: filePipeline.Input.toPipelineBinding(),
|
|
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
|
|
References: normalizedStringMap(filePipeline.References),
|
|
}
|
|
if filePipeline.Chunk != nil {
|
|
profile.Chunk = filePipeline.Chunk.toPipelineBinding()
|
|
}
|
|
if filePipeline.Output != nil {
|
|
profile.Output = filePipeline.Output.toPipelineBinding()
|
|
}
|
|
for _, laneID := range laneIDs {
|
|
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
|
|
extract := fileLane.Extract.toPipelineBinding()
|
|
extract.References = mergeStringMaps(normalizedStringMap(fileLane.References), extract.References)
|
|
lane := pipeline.ArtifactLaneProfile{
|
|
Extract: extract,
|
|
References: normalizedStringMap(fileLane.References),
|
|
}
|
|
if fileLane.Merge != nil {
|
|
lane.Merge = fileLane.Merge.toPipelineBinding()
|
|
}
|
|
if fileLane.Normalize != nil {
|
|
lane.Normalize = fileLane.Normalize.toPipelineBinding()
|
|
}
|
|
if len(fileLane.Validators) > 0 {
|
|
lane.Validators = make([]pipeline.ModuleBinding, len(fileLane.Validators))
|
|
for i, validator := range fileLane.Validators {
|
|
lane.Validators[i] = validator.toPipelineBinding()
|
|
}
|
|
}
|
|
profile.Artifacts[laneID] = lane
|
|
}
|
|
c.Pipelines[pipelineID] = profile
|
|
}
|
|
|
|
if fileCfg.Concurrency != nil && fileCfg.Concurrency.TotalLLM != nil {
|
|
c.Concurrency.TotalLLM = *fileCfg.Concurrency.TotalLLM
|
|
}
|
|
if fileCfg.Diagnostics != nil {
|
|
if fileCfg.Diagnostics.WorkDir != nil {
|
|
c.Diagnostics.WorkDir = strings.TrimSpace(*fileCfg.Diagnostics.WorkDir)
|
|
}
|
|
if fileCfg.Diagnostics.Retention != nil {
|
|
c.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(*fileCfg.Diagnostics.Retention))
|
|
}
|
|
}
|
|
if fileCfg.Workspace != nil {
|
|
if fileCfg.Workspace.Directory != nil {
|
|
c.Workspace.Directory = strings.TrimSpace(*fileCfg.Workspace.Directory)
|
|
}
|
|
if fileCfg.Workspace.Diagnostics != nil {
|
|
if fileCfg.Workspace.Diagnostics.Enabled != nil {
|
|
c.Workspace.Diagnostics.Enabled = *fileCfg.Workspace.Diagnostics.Enabled
|
|
c.Workspace.Diagnostics.enabledSet = true
|
|
}
|
|
if fileCfg.Workspace.Diagnostics.Retention != nil {
|
|
c.Workspace.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(*fileCfg.Workspace.Diagnostics.Retention))
|
|
c.Workspace.Diagnostics.retentionSet = true
|
|
}
|
|
}
|
|
if fileCfg.Workspace.Resume != nil && fileCfg.Workspace.Resume.Enabled != nil {
|
|
c.Workspace.Resume.Enabled = *fileCfg.Workspace.Resume.Enabled
|
|
}
|
|
if fileCfg.Workspace.Debug != nil && fileCfg.Workspace.Debug.Enabled != nil {
|
|
c.Workspace.Debug.Enabled = *fileCfg.Workspace.Debug.Enabled
|
|
}
|
|
}
|
|
c.RecomputeEffectiveDiagnostics()
|
|
|
|
return nil
|
|
}
|
|
|
|
func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, map[string]string, error) {
|
|
keys := make([]string, 0, len(values))
|
|
rawByNormalized := make(map[string]string, len(values))
|
|
for rawID := range values {
|
|
id := strings.TrimSpace(rawID)
|
|
if id == "" {
|
|
return nil, nil, fmt.Errorf("%s must not be empty", keyName)
|
|
}
|
|
if _, ok := rawByNormalized[id]; ok {
|
|
return nil, nil, fmt.Errorf("%s %q is duplicated after trimming", keyName, id)
|
|
}
|
|
rawByNormalized[id] = rawID
|
|
keys = append(keys, id)
|
|
}
|
|
sort.Strings(keys)
|
|
return keys, rawByNormalized, nil
|
|
}
|
|
|
|
func normalizedStringMap(values map[string]string) map[string]string {
|
|
if len(values) == 0 {
|
|
return nil
|
|
}
|
|
out := make(map[string]string, len(values))
|
|
keys := make([]string, 0, len(values))
|
|
rawByNormalized := make(map[string]string, len(values))
|
|
for rawKey := range values {
|
|
key := strings.TrimSpace(rawKey)
|
|
rawByNormalized[key] = rawKey
|
|
keys = append(keys, key)
|
|
}
|
|
sort.Strings(keys)
|
|
for _, key := range keys {
|
|
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
|
|
}
|
|
return out
|
|
}
|
|
|
|
func mergeStringMaps(base map[string]string, override map[string]string) map[string]string {
|
|
if len(base) == 0 && len(override) == 0 {
|
|
return nil
|
|
}
|
|
out := make(map[string]string, len(base)+len(override))
|
|
for key, value := range base {
|
|
out[key] = value
|
|
}
|
|
for key, value := range override {
|
|
out[key] = value
|
|
}
|
|
return out
|
|
}
|
|
|
|
func normalizeOptions(options map[string]any) map[string]any {
|
|
if len(options) == 0 {
|
|
return nil
|
|
}
|
|
return cloneOptions(options)
|
|
}
|