393 lines
9.9 KiB
Go
393 lines
9.9 KiB
Go
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
// LoadPipeline loads pipeline configuration from a YAML file with strict field checking.
|
|
func LoadPipeline(path string) (*PipelineConfig, error) {
|
|
var cfg PipelineConfig
|
|
if err := decodeStrictYAML("pipeline", path, &cfg); err != nil {
|
|
return nil, fmt.Errorf("load pipeline config: %w", err)
|
|
}
|
|
applyPipelineDefaults(&cfg)
|
|
return &cfg, nil
|
|
}
|
|
|
|
// LoadSession loads session configuration from a YAML file with strict field checking.
|
|
func LoadSession(path string) (*SessionConfig, error) {
|
|
return LoadSessionWithOptions(path, SessionLoadOptions{})
|
|
}
|
|
|
|
// SessionLoadOptions configures session template rendering behavior.
|
|
type SessionLoadOptions struct {
|
|
SessionID string
|
|
PreviousSessionID string
|
|
}
|
|
|
|
// LoadSessionWithOptions loads session configuration from a YAML file with
|
|
// strict field checking after template rendering.
|
|
func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfig, error) {
|
|
sessionBytes, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("load session config: session file %q: open: %w", path, err)
|
|
}
|
|
|
|
rendered, err := renderSessionTemplate(string(sessionBytes), opts)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("load session config: %w", err)
|
|
}
|
|
|
|
var cfg SessionConfig
|
|
if err := decodeStrictYAMLFromReader("session", path, strings.NewReader(rendered), &cfg); err != nil {
|
|
return nil, fmt.Errorf("load session config: %w", err)
|
|
}
|
|
if strings.TrimSpace(opts.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != strings.TrimSpace(opts.SessionID) {
|
|
return nil, fmt.Errorf(
|
|
"load session config: session file %q: session_id mismatch: --session-id %q does not match rendered session_id %q",
|
|
path,
|
|
strings.TrimSpace(opts.SessionID),
|
|
strings.TrimSpace(cfg.SessionID),
|
|
)
|
|
}
|
|
if strings.TrimSpace(opts.PreviousSessionID) != "" &&
|
|
strings.TrimSpace(cfg.PreviousSessionID) != "" &&
|
|
strings.TrimSpace(cfg.PreviousSessionID) != strings.TrimSpace(opts.PreviousSessionID) {
|
|
return nil, fmt.Errorf(
|
|
"load session config: session file %q: previous_session_id mismatch: --previous-session-id %q does not match rendered previous_session_id %q",
|
|
path,
|
|
strings.TrimSpace(opts.PreviousSessionID),
|
|
strings.TrimSpace(cfg.PreviousSessionID),
|
|
)
|
|
}
|
|
return &cfg, nil
|
|
}
|
|
|
|
// Load loads and resolves combined pipeline and session configuration.
|
|
func Load(pipelinePath, sessionPath string) (*Config, error) {
|
|
return LoadWithSessionOptions(pipelinePath, sessionPath, SessionLoadOptions{})
|
|
}
|
|
|
|
// LoadWithSessionOptions loads and resolves combined pipeline and session
|
|
// configuration with session template options.
|
|
func LoadWithSessionOptions(pipelinePath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
|
|
pipelineCfg, err := LoadPipeline(pipelinePath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
sessionCfg, err := LoadSessionWithOptions(sessionPath, sessionOpts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &Config{
|
|
Pipeline: pipelineCfg,
|
|
Session: sessionCfg,
|
|
PipelinePath: pipelinePath,
|
|
SessionPath: sessionPath,
|
|
}, nil
|
|
}
|
|
|
|
func decodeStrictYAML(kind, path string, out any) error {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return fmt.Errorf("%s file %q: open: %w", kind, path, err)
|
|
}
|
|
defer f.Close()
|
|
|
|
return decodeStrictYAMLFromReader(kind, path, f, out)
|
|
}
|
|
|
|
func decodeStrictYAMLFromReader(kind, path string, r io.Reader, out any) error {
|
|
dec := yaml.NewDecoder(r)
|
|
dec.KnownFields(true)
|
|
if err := dec.Decode(out); err != nil {
|
|
return fmt.Errorf("%s file %q: strict decode failed: %w", kind, path, err)
|
|
}
|
|
|
|
var extra any
|
|
if err := dec.Decode(&extra); err != nil && err != io.EOF {
|
|
return fmt.Errorf("%s file %q: trailing content decode failed: %w", kind, path, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
var sessionTemplatePattern = regexp.MustCompile(`\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}`)
|
|
|
|
func renderSessionTemplate(content string, opts SessionLoadOptions) (string, error) {
|
|
sessionID := strings.TrimSpace(opts.SessionID)
|
|
previousSessionID := strings.TrimSpace(opts.PreviousSessionID)
|
|
rendered := content
|
|
if sessionID != "" {
|
|
rendered = replaceTemplateVariable(rendered, "session_id", sessionID)
|
|
}
|
|
if previousSessionID != "" {
|
|
rendered = replaceTemplateVariable(rendered, "previous_session_id", previousSessionID)
|
|
}
|
|
|
|
unresolved := sessionTemplatePattern.FindAllStringSubmatch(rendered, -1)
|
|
if len(unresolved) > 0 {
|
|
seenVars := map[string]struct{}{}
|
|
vars := make([]string, 0, len(unresolved))
|
|
for _, m := range unresolved {
|
|
if len(m) > 1 {
|
|
name := m[1]
|
|
if _, ok := seenVars[name]; ok {
|
|
continue
|
|
}
|
|
seenVars[name] = struct{}{}
|
|
vars = append(vars, name)
|
|
}
|
|
}
|
|
sort.Strings(vars)
|
|
if len(vars) > 0 {
|
|
hints := unresolvedTemplateHints(vars)
|
|
return "", fmt.Errorf(
|
|
"session file template rendering failed: unresolved template variable(s): %s%s",
|
|
strings.Join(vars, ", "),
|
|
hints,
|
|
)
|
|
}
|
|
return "", fmt.Errorf("session file template rendering failed: unresolved template placeholders remain")
|
|
}
|
|
|
|
return rendered, nil
|
|
}
|
|
|
|
func replaceTemplateVariable(content, name, value string) string {
|
|
rendered := strings.ReplaceAll(content, "{{"+name+"}}", value)
|
|
rendered = strings.ReplaceAll(rendered, "{{ "+name+" }}", value)
|
|
return rendered
|
|
}
|
|
|
|
func unresolvedTemplateHints(vars []string) string {
|
|
seen := map[string]struct{}{}
|
|
flags := make([]string, 0, 2)
|
|
for _, name := range vars {
|
|
switch name {
|
|
case "session_id":
|
|
if _, ok := seen["--session-id"]; !ok {
|
|
seen["--session-id"] = struct{}{}
|
|
flags = append(flags, "--session-id")
|
|
}
|
|
case "previous_session_id":
|
|
if _, ok := seen["--previous-session-id"]; !ok {
|
|
seen["--previous-session-id"] = struct{}{}
|
|
flags = append(flags, "--previous-session-id")
|
|
}
|
|
}
|
|
}
|
|
if len(flags) == 0 {
|
|
return ""
|
|
}
|
|
return "; pass " + strings.Join(flags, " and ") + " when using those template variable(s)"
|
|
}
|
|
|
|
func shortName(path, fallback string) string {
|
|
base := filepath.Base(path)
|
|
if base == "." || base == string(filepath.Separator) {
|
|
return fallback
|
|
}
|
|
return base
|
|
}
|
|
|
|
func applyPipelineDefaults(cfg *PipelineConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
applyWorkspaceDefaults(&cfg.Workspace)
|
|
applyStorageDefaults(&cfg.Storage)
|
|
applySpoolDefaults(&cfg.Spool)
|
|
applyArchiveDefaults(&cfg.Archive)
|
|
applyWhisperXDefaults(&cfg.WhisperX)
|
|
applySeriatimDefaults(&cfg.Seriatim)
|
|
applyAuditaDefaults(&cfg.Audita)
|
|
if cfg.Normalize == nil {
|
|
cfg.Normalize = &NormalizeConfig{}
|
|
}
|
|
applyNormalizeDefaults(cfg.Normalize)
|
|
applyTrimDefaults(cfg.Trim)
|
|
applyScriptoriumDefaults(cfg.Scriptorium)
|
|
}
|
|
|
|
func applyWorkspaceDefaults(cfg *WorkspaceConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Root == "" {
|
|
cfg.Root = DefaultWorkspaceRoot
|
|
}
|
|
}
|
|
|
|
func applyStorageDefaults(cfg *StorageConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.S3 == nil {
|
|
cfg.S3 = &StorageS3Config{}
|
|
}
|
|
if cfg.S3.RootPrefix == "" {
|
|
cfg.S3.RootPrefix = DefaultStorageS3RootPrefix
|
|
}
|
|
if cfg.S3.AccessKeyIDEnv == "" {
|
|
cfg.S3.AccessKeyIDEnv = DefaultS3AccessKeyIDEnv
|
|
}
|
|
if cfg.S3.SecretKeyEnv == "" {
|
|
cfg.S3.SecretKeyEnv = DefaultS3SecretAccessKeyEnv
|
|
}
|
|
}
|
|
|
|
func applySpoolDefaults(cfg *SpoolConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Root == "" {
|
|
cfg.Root = DefaultSpoolRoot
|
|
}
|
|
}
|
|
|
|
func applyArchiveDefaults(cfg **ArchiveConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if *cfg == nil {
|
|
*cfg = &ArchiveConfig{}
|
|
}
|
|
|
|
if (*cfg).Enabled == nil {
|
|
(*cfg).Enabled = boolPtr(DefaultArchiveEnabled)
|
|
}
|
|
if (*cfg).UploadRun == nil {
|
|
(*cfg).UploadRun = boolPtr(DefaultArchiveUploadRun)
|
|
}
|
|
if len((*cfg).PromoteArtifacts) == 0 {
|
|
(*cfg).PromoteArtifacts = append([]ArchivePromotionRule(nil), DefaultArchivePromoteArtifacts...)
|
|
}
|
|
for i := range (*cfg).PromoteArtifacts {
|
|
if (*cfg).PromoteArtifacts[i].Required == nil {
|
|
(*cfg).PromoteArtifacts[i].Required = boolPtr(true)
|
|
}
|
|
}
|
|
}
|
|
|
|
func applyWhisperXDefaults(cfg *WhisperXConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Language == "" {
|
|
cfg.Language = DefaultWhisperXLanguage
|
|
}
|
|
if cfg.Timeout == "" {
|
|
cfg.Timeout = DefaultWhisperXTimeout
|
|
}
|
|
if cfg.RetryDelay == "" {
|
|
cfg.RetryDelay = DefaultWhisperXRetryDelay
|
|
}
|
|
if cfg.Concurrency == nil {
|
|
cfg.Concurrency = intPtr(DefaultWhisperXConcurrency)
|
|
}
|
|
if cfg.Retries == nil {
|
|
cfg.Retries = intPtr(DefaultWhisperXRetries)
|
|
}
|
|
}
|
|
|
|
func intPtr(v int) *int {
|
|
p := v
|
|
return &p
|
|
}
|
|
|
|
func applySeriatimDefaults(cfg *SeriatimConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Binary == "" {
|
|
cfg.Binary = DefaultSeriatimBinary
|
|
}
|
|
if cfg.Timeout == "" {
|
|
cfg.Timeout = DefaultSeriatimTimeout
|
|
}
|
|
if cfg.OutputSchema == "" {
|
|
cfg.OutputSchema = DefaultSeriatimOutputSchema
|
|
}
|
|
if cfg.CoalesceGap == nil {
|
|
cfg.CoalesceGap = float64Ptr(DefaultSeriatimCoalesceGap)
|
|
}
|
|
if cfg.Report == nil {
|
|
cfg.Report = boolPtr(DefaultSeriatimReport)
|
|
}
|
|
}
|
|
|
|
func applyAuditaDefaults(cfg *AuditaConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Binary == "" {
|
|
cfg.Binary = DefaultAuditaBinary
|
|
}
|
|
if cfg.Timeout == "" {
|
|
cfg.Timeout = DefaultAuditaTimeout
|
|
}
|
|
if cfg.Report == nil {
|
|
cfg.Report = boolPtr(DefaultAuditaReport)
|
|
}
|
|
}
|
|
|
|
func applyScriptoriumDefaults(cfg *ScriptoriumConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Binary == "" {
|
|
cfg.Binary = DefaultScriptoriumBinary
|
|
}
|
|
if cfg.Timeout == "" {
|
|
cfg.Timeout = DefaultScriptoriumTimeout
|
|
}
|
|
}
|
|
|
|
func applyTrimDefaults(cfg *TrimConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Bounds.Timeout == "" {
|
|
cfg.Bounds.Timeout = DefaultTrimBoundsTimeout
|
|
}
|
|
if cfg.Seriatim.Report == nil {
|
|
cfg.Seriatim.Report = boolPtr(DefaultTrimSeriatimReport)
|
|
}
|
|
}
|
|
|
|
func applyNormalizeDefaults(cfg *NormalizeConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.OutputSchema == "" {
|
|
cfg.OutputSchema = DefaultNormalizeOutputSchema
|
|
}
|
|
if cfg.OutputPath == "" && !cfg.outputPathWasSet() {
|
|
cfg.OutputPath = DefaultNormalizeOutputPath
|
|
}
|
|
if cfg.Report == nil {
|
|
cfg.Report = boolPtr(DefaultNormalizeReport)
|
|
}
|
|
}
|
|
|
|
func float64Ptr(v float64) *float64 {
|
|
p := v
|
|
return &p
|
|
}
|
|
|
|
func boolPtr(v bool) *bool {
|
|
p := v
|
|
return &p
|
|
}
|