546 lines
15 KiB
Go
546 lines
15 KiB
Go
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
// LoadPipeline loads pipeline configuration from a YAML file with strict field checking.
|
|
func LoadPipeline(path string) (*PipelineConfig, error) {
|
|
var cfg PipelineConfig
|
|
if err := decodeStrictYAML("pipeline", path, &cfg); err != nil {
|
|
return nil, fmt.Errorf("load pipeline config: %w", err)
|
|
}
|
|
applyPipelineDefaults(&cfg)
|
|
return &cfg, nil
|
|
}
|
|
|
|
// LoadCampaign loads campaign configuration from a YAML file with strict field checking.
|
|
func LoadCampaign(path string) (*CampaignConfig, error) {
|
|
var cfg CampaignConfig
|
|
if err := decodeStrictYAML("campaign", path, &cfg); err != nil {
|
|
return nil, fmt.Errorf("load campaign config: %w", err)
|
|
}
|
|
return &cfg, nil
|
|
}
|
|
|
|
// LoadSession loads session configuration from a YAML file with strict field checking.
|
|
func LoadSession(path string) (*SessionConfig, error) {
|
|
return LoadSessionWithOptions(path, SessionLoadOptions{})
|
|
}
|
|
|
|
// SessionLoadOptions configures session template rendering behavior.
|
|
type SessionLoadOptions struct {
|
|
SessionID string
|
|
PreviousSessionID string
|
|
}
|
|
|
|
// LoadSessionWithOptions loads session configuration from a YAML file with
|
|
// strict field checking after template rendering.
|
|
func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfig, error) {
|
|
sessionBytes, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("load session config: session file %q: open: %w", path, err)
|
|
}
|
|
return LoadSessionBytesWithOptions(path, sessionBytes, opts)
|
|
}
|
|
|
|
// LoadSessionBytesWithOptions loads session configuration from YAML bytes with
|
|
// strict field checking after template rendering.
|
|
func LoadSessionBytesWithOptions(label string, data []byte, opts SessionLoadOptions) (*SessionConfig, error) {
|
|
rendered, err := renderSessionTemplate(string(data), opts)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("load session config: %w", err)
|
|
}
|
|
|
|
var cfg SessionConfig
|
|
if err := decodeStrictYAMLFromReader("session", label, strings.NewReader(rendered), &cfg); err != nil {
|
|
return nil, fmt.Errorf("load session config: %w", err)
|
|
}
|
|
if strings.TrimSpace(opts.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != strings.TrimSpace(opts.SessionID) {
|
|
return nil, fmt.Errorf(
|
|
"load session config: session file %q: session_id mismatch: --session-id %q does not match rendered session_id %q",
|
|
label,
|
|
strings.TrimSpace(opts.SessionID),
|
|
strings.TrimSpace(cfg.SessionID),
|
|
)
|
|
}
|
|
if strings.TrimSpace(opts.PreviousSessionID) != "" &&
|
|
strings.TrimSpace(cfg.PreviousSessionID) != "" &&
|
|
strings.TrimSpace(cfg.PreviousSessionID) != strings.TrimSpace(opts.PreviousSessionID) {
|
|
return nil, fmt.Errorf(
|
|
"load session config: session file %q: previous_session_id mismatch: --previous-session-id %q does not match rendered previous_session_id %q",
|
|
label,
|
|
strings.TrimSpace(opts.PreviousSessionID),
|
|
strings.TrimSpace(cfg.PreviousSessionID),
|
|
)
|
|
}
|
|
return &cfg, nil
|
|
}
|
|
|
|
// LoadArchiveLockStoreBytes loads a mutable session lock store with strict
|
|
// field checking and source validation.
|
|
func LoadArchiveLockStoreBytes(label string, data []byte, scriptorium *ScriptoriumConfig) (*ArchiveLockStore, error) {
|
|
var store ArchiveLockStore
|
|
if err := decodeStrictYAMLFromReader("archive lock store", label, strings.NewReader(string(data)), &store); err != nil {
|
|
return nil, fmt.Errorf("load archive lock store: %w", err)
|
|
}
|
|
locks, err := ValidateArchiveLockRules(store.Locks, scriptorium, "locks")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("load archive lock store: %w", err)
|
|
}
|
|
store.Locks = locks
|
|
return &store, nil
|
|
}
|
|
|
|
// MarshalArchiveLockStore serializes a mutable lock store as strict-compatible YAML.
|
|
func MarshalArchiveLockStore(store *ArchiveLockStore) ([]byte, error) {
|
|
if store == nil {
|
|
store = &ArchiveLockStore{}
|
|
}
|
|
data, err := yaml.Marshal(store)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("marshal archive lock store: %w", err)
|
|
}
|
|
return data, nil
|
|
}
|
|
|
|
// Load loads and resolves combined pipeline, campaign, and session configuration.
|
|
// Passing only a session path is supported for package-internal compatibility;
|
|
// in that form campaign.yml is expected next to the session file.
|
|
func Load(pipelinePath string, paths ...string) (*Config, error) {
|
|
campaignPath, sessionPath, err := campaignSessionPaths(paths...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
|
}
|
|
|
|
// LoadWithSessionOptions loads and resolves combined pipeline, campaign, and
|
|
// session configuration with session template options.
|
|
func LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
|
|
pipelineCfg, err := LoadPipeline(pipelinePath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
campaignCfg, err := LoadCampaign(campaignPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
sessionCfg, err := LoadSessionWithOptions(sessionPath, sessionOpts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return Resolve(pipelinePath, pipelineCfg, campaignPath, campaignCfg, sessionPath, sessionCfg, SessionSource{
|
|
Source: "session_config",
|
|
LocalPath: sessionPath,
|
|
})
|
|
}
|
|
|
|
// Resolve builds final stage-facing configuration from already loaded
|
|
// pipeline, campaign, and session documents.
|
|
func Resolve(pipelinePath string, pipelineCfg *PipelineConfig, campaignPath string, campaignCfg *CampaignConfig, sessionPath string, sessionCfg *SessionConfig, sessionSource SessionSource) (*Config, error) {
|
|
stableInputs, err := mergeCampaignSession(campaignCfg, sessionCfg, campaignPath, sessionPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if strings.TrimSpace(sessionSource.Source) == "" {
|
|
sessionSource.Source = "session_config"
|
|
}
|
|
if strings.TrimSpace(sessionSource.LocalPath) == "" {
|
|
sessionSource.LocalPath = sessionPath
|
|
}
|
|
|
|
return &Config{
|
|
Pipeline: pipelineCfg,
|
|
Campaign: campaignCfg,
|
|
Session: sessionCfg,
|
|
PipelinePath: pipelinePath,
|
|
CampaignPath: campaignPath,
|
|
SessionPath: sessionPath,
|
|
StableInputs: stableInputs,
|
|
SessionSource: sessionSource,
|
|
}, nil
|
|
}
|
|
|
|
func campaignSessionPaths(paths ...string) (campaignPath, sessionPath string, err error) {
|
|
switch len(paths) {
|
|
case 1:
|
|
sessionPath = paths[0]
|
|
campaignPath = filepath.Join(filepath.Dir(sessionPath), "campaign.yml")
|
|
case 2:
|
|
campaignPath = paths[0]
|
|
sessionPath = paths[1]
|
|
default:
|
|
return "", "", fmt.Errorf("load config: expected session path or campaign and session paths")
|
|
}
|
|
return campaignPath, sessionPath, nil
|
|
}
|
|
|
|
func mergeCampaignSession(campaignCfg *CampaignConfig, sessionCfg *SessionConfig, campaignPath, sessionPath string) (ResolvedStableInputs, error) {
|
|
if campaignCfg == nil {
|
|
return ResolvedStableInputs{}, fmt.Errorf("campaign config is required")
|
|
}
|
|
if sessionCfg == nil {
|
|
return ResolvedStableInputs{}, fmt.Errorf("session config is required")
|
|
}
|
|
|
|
campaignName := strings.TrimSpace(campaignCfg.Campaign)
|
|
sessionCampaign := strings.TrimSpace(sessionCfg.Campaign)
|
|
if sessionCampaign != "" && campaignName != "" && sessionCampaign != campaignName {
|
|
return ResolvedStableInputs{}, fmt.Errorf(
|
|
"campaign/session config invalid: session campaign %q does not match campaign config %q",
|
|
sessionCampaign,
|
|
campaignName,
|
|
)
|
|
}
|
|
if sessionCampaign == "" {
|
|
sessionCfg.Campaign = campaignName
|
|
}
|
|
|
|
stable := ResolvedStableInputs{
|
|
SpeakersFile: selectStableInput(
|
|
campaignCfg.Inputs.SpeakersFile,
|
|
sessionCfg.Inputs.SpeakersFile,
|
|
campaignPath,
|
|
sessionPath,
|
|
),
|
|
AutocorrectFile: selectStableInput(
|
|
campaignCfg.Inputs.AutocorrectFile,
|
|
sessionCfg.Inputs.AutocorrectFile,
|
|
campaignPath,
|
|
sessionPath,
|
|
),
|
|
GlossaryFile: selectStableInput(
|
|
campaignCfg.Inputs.GlossaryFile,
|
|
sessionCfg.Inputs.GlossaryFile,
|
|
campaignPath,
|
|
sessionPath,
|
|
),
|
|
}
|
|
|
|
sessionCfg.Inputs.SpeakersFile = stable.SpeakersFile.Path
|
|
sessionCfg.Inputs.AutocorrectFile = stable.AutocorrectFile.Path
|
|
sessionCfg.Inputs.GlossaryFile = stable.GlossaryFile.Path
|
|
return stable, nil
|
|
}
|
|
|
|
func selectStableInput(campaignValue, sessionValue, campaignPath, sessionPath string) ResolvedInputFile {
|
|
if strings.TrimSpace(sessionValue) != "" {
|
|
return ResolvedInputFile{
|
|
Path: sessionValue,
|
|
ConfigPath: sessionPath,
|
|
Source: "session_config",
|
|
}
|
|
}
|
|
return ResolvedInputFile{
|
|
Path: campaignValue,
|
|
ConfigPath: campaignPath,
|
|
Source: "campaign_config",
|
|
}
|
|
}
|
|
|
|
func decodeStrictYAML(kind, path string, out any) error {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return fmt.Errorf("%s file %q: open: %w", kind, path, err)
|
|
}
|
|
defer f.Close()
|
|
|
|
return decodeStrictYAMLFromReader(kind, path, f, out)
|
|
}
|
|
|
|
func decodeStrictYAMLFromReader(kind, path string, r io.Reader, out any) error {
|
|
dec := yaml.NewDecoder(r)
|
|
dec.KnownFields(true)
|
|
if err := dec.Decode(out); err != nil {
|
|
return fmt.Errorf("%s file %q: strict decode failed: %w", kind, path, err)
|
|
}
|
|
|
|
var extra any
|
|
if err := dec.Decode(&extra); err != nil && err != io.EOF {
|
|
return fmt.Errorf("%s file %q: trailing content decode failed: %w", kind, path, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
var sessionTemplatePattern = regexp.MustCompile(`\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}`)
|
|
|
|
func renderSessionTemplate(content string, opts SessionLoadOptions) (string, error) {
|
|
sessionID := strings.TrimSpace(opts.SessionID)
|
|
previousSessionID := strings.TrimSpace(opts.PreviousSessionID)
|
|
rendered := content
|
|
if sessionID != "" {
|
|
rendered = replaceTemplateVariable(rendered, "session_id", sessionID)
|
|
}
|
|
if previousSessionID != "" {
|
|
rendered = replaceTemplateVariable(rendered, "previous_session_id", previousSessionID)
|
|
}
|
|
|
|
unresolved := sessionTemplatePattern.FindAllStringSubmatch(rendered, -1)
|
|
if len(unresolved) > 0 {
|
|
seenVars := map[string]struct{}{}
|
|
vars := make([]string, 0, len(unresolved))
|
|
for _, m := range unresolved {
|
|
if len(m) > 1 {
|
|
name := m[1]
|
|
if _, ok := seenVars[name]; ok {
|
|
continue
|
|
}
|
|
seenVars[name] = struct{}{}
|
|
vars = append(vars, name)
|
|
}
|
|
}
|
|
sort.Strings(vars)
|
|
if len(vars) > 0 {
|
|
hints := unresolvedTemplateHints(vars)
|
|
return "", fmt.Errorf(
|
|
"session file template rendering failed: unresolved template variable(s): %s%s",
|
|
strings.Join(vars, ", "),
|
|
hints,
|
|
)
|
|
}
|
|
return "", fmt.Errorf("session file template rendering failed: unresolved template placeholders remain")
|
|
}
|
|
|
|
return rendered, nil
|
|
}
|
|
|
|
func replaceTemplateVariable(content, name, value string) string {
|
|
rendered := strings.ReplaceAll(content, "{{"+name+"}}", value)
|
|
rendered = strings.ReplaceAll(rendered, "{{ "+name+" }}", value)
|
|
return rendered
|
|
}
|
|
|
|
func unresolvedTemplateHints(vars []string) string {
|
|
seen := map[string]struct{}{}
|
|
flags := make([]string, 0, 2)
|
|
for _, name := range vars {
|
|
switch name {
|
|
case "session_id":
|
|
if _, ok := seen["--session-id"]; !ok {
|
|
seen["--session-id"] = struct{}{}
|
|
flags = append(flags, "--session-id")
|
|
}
|
|
case "previous_session_id":
|
|
if _, ok := seen["--previous-session-id"]; !ok {
|
|
seen["--previous-session-id"] = struct{}{}
|
|
flags = append(flags, "--previous-session-id")
|
|
}
|
|
}
|
|
}
|
|
if len(flags) == 0 {
|
|
return ""
|
|
}
|
|
return "; pass " + strings.Join(flags, " and ") + " when using those template variable(s)"
|
|
}
|
|
|
|
func shortName(path, fallback string) string {
|
|
base := filepath.Base(path)
|
|
if base == "." || base == string(filepath.Separator) {
|
|
return fallback
|
|
}
|
|
return base
|
|
}
|
|
|
|
func applyPipelineDefaults(cfg *PipelineConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
applyWorkspaceDefaults(&cfg.Workspace)
|
|
applyStorageDefaults(&cfg.Storage)
|
|
applySpoolDefaults(&cfg.Spool)
|
|
applyArchiveDefaults(&cfg.Archive)
|
|
applyWhisperXDefaults(&cfg.WhisperX)
|
|
applySeriatimDefaults(&cfg.Seriatim)
|
|
applyAuditaDefaults(&cfg.Audita)
|
|
if cfg.Normalize == nil {
|
|
cfg.Normalize = &NormalizeConfig{}
|
|
}
|
|
applyNormalizeDefaults(cfg.Normalize)
|
|
applyTrimDefaults(cfg.Trim)
|
|
applyScriptoriumDefaults(cfg.Scriptorium)
|
|
}
|
|
|
|
func applyWorkspaceDefaults(cfg *WorkspaceConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Root == "" {
|
|
cfg.Root = DefaultWorkspaceRoot
|
|
}
|
|
}
|
|
|
|
func applyStorageDefaults(cfg *StorageConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.S3 == nil {
|
|
cfg.S3 = &StorageS3Config{}
|
|
}
|
|
if cfg.S3.RootPrefix == "" {
|
|
cfg.S3.RootPrefix = DefaultStorageS3RootPrefix
|
|
}
|
|
if cfg.S3.AccessKeyIDEnv == "" {
|
|
cfg.S3.AccessKeyIDEnv = DefaultS3AccessKeyIDEnv
|
|
}
|
|
if cfg.S3.SecretKeyEnv == "" {
|
|
cfg.S3.SecretKeyEnv = DefaultS3SecretAccessKeyEnv
|
|
}
|
|
}
|
|
|
|
func applySpoolDefaults(cfg *SpoolConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Root == "" {
|
|
cfg.Root = DefaultSpoolRoot
|
|
}
|
|
}
|
|
|
|
func applyArchiveDefaults(cfg **ArchiveConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if *cfg == nil {
|
|
*cfg = &ArchiveConfig{}
|
|
}
|
|
|
|
if (*cfg).Enabled == nil {
|
|
(*cfg).Enabled = boolPtr(DefaultArchiveEnabled)
|
|
}
|
|
if (*cfg).UploadRun == nil {
|
|
(*cfg).UploadRun = boolPtr(DefaultArchiveUploadRun)
|
|
}
|
|
if len((*cfg).PromoteArtifacts) == 0 {
|
|
(*cfg).PromoteArtifacts = append([]ArchivePromotionRule(nil), DefaultArchivePromoteArtifacts...)
|
|
}
|
|
for i := range (*cfg).PromoteArtifacts {
|
|
if (*cfg).PromoteArtifacts[i].Required == nil {
|
|
(*cfg).PromoteArtifacts[i].Required = boolPtr(true)
|
|
}
|
|
}
|
|
}
|
|
|
|
func applyWhisperXDefaults(cfg *WhisperXConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Language == "" {
|
|
cfg.Language = DefaultWhisperXLanguage
|
|
}
|
|
if cfg.Timeout == "" {
|
|
cfg.Timeout = DefaultWhisperXTimeout
|
|
}
|
|
if cfg.RetryDelay == "" {
|
|
cfg.RetryDelay = DefaultWhisperXRetryDelay
|
|
}
|
|
if cfg.Concurrency == nil {
|
|
cfg.Concurrency = intPtr(DefaultWhisperXConcurrency)
|
|
}
|
|
if cfg.Retries == nil {
|
|
cfg.Retries = intPtr(DefaultWhisperXRetries)
|
|
}
|
|
}
|
|
|
|
func intPtr(v int) *int {
|
|
p := v
|
|
return &p
|
|
}
|
|
|
|
func applySeriatimDefaults(cfg *SeriatimConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Binary == "" {
|
|
cfg.Binary = DefaultSeriatimBinary
|
|
}
|
|
if cfg.Timeout == "" {
|
|
cfg.Timeout = DefaultSeriatimTimeout
|
|
}
|
|
if cfg.OutputSchema == "" {
|
|
cfg.OutputSchema = DefaultSeriatimOutputSchema
|
|
}
|
|
if cfg.CoalesceGap == nil {
|
|
cfg.CoalesceGap = float64Ptr(DefaultSeriatimCoalesceGap)
|
|
}
|
|
if cfg.Report == nil {
|
|
cfg.Report = boolPtr(DefaultSeriatimReport)
|
|
}
|
|
}
|
|
|
|
func applyAuditaDefaults(cfg *AuditaConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Binary == "" {
|
|
cfg.Binary = DefaultAuditaBinary
|
|
}
|
|
if cfg.Timeout == "" {
|
|
cfg.Timeout = DefaultAuditaTimeout
|
|
}
|
|
if cfg.Report == nil {
|
|
cfg.Report = boolPtr(DefaultAuditaReport)
|
|
}
|
|
}
|
|
|
|
func applyScriptoriumDefaults(cfg *ScriptoriumConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Binary == "" {
|
|
cfg.Binary = DefaultScriptoriumBinary
|
|
}
|
|
if cfg.Timeout == "" {
|
|
cfg.Timeout = DefaultScriptoriumTimeout
|
|
}
|
|
}
|
|
|
|
func applyTrimDefaults(cfg *TrimConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Bounds.Timeout == "" {
|
|
cfg.Bounds.Timeout = DefaultTrimBoundsTimeout
|
|
}
|
|
if cfg.Seriatim.Report == nil {
|
|
cfg.Seriatim.Report = boolPtr(DefaultTrimSeriatimReport)
|
|
}
|
|
}
|
|
|
|
func applyNormalizeDefaults(cfg *NormalizeConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.OutputSchema == "" {
|
|
cfg.OutputSchema = DefaultNormalizeOutputSchema
|
|
}
|
|
if cfg.OutputPath == "" && !cfg.outputPathWasSet() {
|
|
cfg.OutputPath = DefaultNormalizeOutputPath
|
|
}
|
|
if cfg.Report == nil {
|
|
cfg.Report = boolPtr(DefaultNormalizeReport)
|
|
}
|
|
}
|
|
|
|
func float64Ptr(v float64) *float64 {
|
|
p := v
|
|
return &p
|
|
}
|
|
|
|
func boolPtr(v bool) *bool {
|
|
p := v
|
|
return &p
|
|
}
|