Files
narratio/internal/config/load.go

534 lines
14 KiB
Go

package config
import (
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"gopkg.in/yaml.v3"
)
// LoadPipeline loads pipeline configuration from a YAML file with strict field checking.
func LoadPipeline(path string) (*PipelineConfig, error) {
var cfg PipelineConfig
if err := decodeStrictYAML("pipeline", path, &cfg); err != nil {
return nil, fmt.Errorf("load pipeline config: %w", err)
}
applyPipelineDefaults(&cfg)
return &cfg, nil
}
// LoadCampaign loads campaign configuration from a YAML file with strict field checking.
func LoadCampaign(path string) (*CampaignConfig, error) {
var cfg CampaignConfig
if err := decodeStrictYAML("campaign", path, &cfg); err != nil {
return nil, fmt.Errorf("load campaign config: %w", err)
}
return &cfg, nil
}
// LoadSession loads session configuration from a YAML file with strict field checking.
func LoadSession(path string) (*SessionConfig, error) {
return LoadSessionWithOptions(path, SessionLoadOptions{})
}
// SessionLoadOptions configures expected session identity checks.
type SessionLoadOptions struct {
SessionID string
PreviousSessionID string
}
// LoadSessionWithOptions loads session configuration from a YAML file with
// strict field checking.
func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfig, error) {
sessionBytes, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("load session config: session file %q: open: %w", path, err)
}
return LoadSessionBytesWithOptions(path, sessionBytes, opts)
}
// LoadSessionBytesWithOptions loads session configuration from YAML bytes with
// strict field checking.
func LoadSessionBytesWithOptions(label string, data []byte, opts SessionLoadOptions) (*SessionConfig, error) {
if err := rejectSessionTemplatePlaceholders(label, string(data)); err != nil {
return nil, fmt.Errorf("load session config: %w", err)
}
var cfg SessionConfig
if err := decodeStrictYAMLFromReader("session", label, strings.NewReader(string(data)), &cfg); err != nil {
return nil, fmt.Errorf("load session config: %w", err)
}
if strings.TrimSpace(opts.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != strings.TrimSpace(opts.SessionID) {
return nil, fmt.Errorf(
"load session config: session file %q: session_id mismatch: --session-id %q does not match session_id %q",
label,
strings.TrimSpace(opts.SessionID),
strings.TrimSpace(cfg.SessionID),
)
}
if strings.TrimSpace(opts.PreviousSessionID) != "" &&
strings.TrimSpace(cfg.PreviousSessionID) != "" &&
strings.TrimSpace(cfg.PreviousSessionID) != strings.TrimSpace(opts.PreviousSessionID) {
return nil, fmt.Errorf(
"load session config: session file %q: previous_session_id mismatch: --previous-session-id %q does not match previous_session_id %q",
label,
strings.TrimSpace(opts.PreviousSessionID),
strings.TrimSpace(cfg.PreviousSessionID),
)
}
return &cfg, nil
}
// LoadPublishLockStoreBytes loads a mutable session lock store with strict
// field checking and source validation.
func LoadPublishLockStoreBytes(label string, data []byte, scriptorium *ScriptoriumConfig) (*PublishLockStore, error) {
var store PublishLockStore
if err := decodeStrictYAMLFromReader("publish lock store", label, strings.NewReader(string(data)), &store); err != nil {
return nil, fmt.Errorf("load publish lock store: %w", err)
}
locks, err := ValidatePublishLockRules(store.Locks, scriptorium, "locks")
if err != nil {
return nil, fmt.Errorf("load publish lock store: %w", err)
}
store.Locks = locks
return &store, nil
}
// MarshalPublishLockStore serializes a mutable lock store as strict-compatible YAML.
func MarshalPublishLockStore(store *PublishLockStore) ([]byte, error) {
if store == nil {
store = &PublishLockStore{}
}
data, err := yaml.Marshal(store)
if err != nil {
return nil, fmt.Errorf("marshal publish lock store: %w", err)
}
return data, nil
}
// Load loads and resolves combined pipeline, campaign, and session configuration.
// Passing only a session path is supported for package-internal compatibility;
// in that form campaign.yml is expected next to the session file.
func Load(pipelinePath string, paths ...string) (*Config, error) {
campaignPath, sessionPath, err := campaignSessionPaths(paths...)
if err != nil {
return nil, err
}
return LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
}
// LoadWithSessionOptions loads and resolves combined pipeline, campaign, and
// session configuration with expected session identity checks.
func LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
pipelineCfg, err := LoadPipeline(pipelinePath)
if err != nil {
return nil, err
}
campaignCfg, err := LoadCampaign(campaignPath)
if err != nil {
return nil, err
}
sessionCfg, err := LoadSessionWithOptions(sessionPath, sessionOpts)
if err != nil {
return nil, err
}
return Resolve(pipelinePath, pipelineCfg, campaignPath, campaignCfg, sessionPath, sessionCfg, SessionSource{
Source: "session_config",
LocalPath: sessionPath,
})
}
// Resolve builds final stage-facing configuration from already loaded
// pipeline, campaign, and session documents.
func Resolve(pipelinePath string, pipelineCfg *PipelineConfig, campaignPath string, campaignCfg *CampaignConfig, sessionPath string, sessionCfg *SessionConfig, sessionSource SessionSource) (*Config, error) {
stableInputs, err := mergeCampaignSession(campaignCfg, sessionCfg, campaignPath, sessionPath)
if err != nil {
return nil, err
}
if strings.TrimSpace(sessionSource.Source) == "" {
sessionSource.Source = "session_config"
}
if strings.TrimSpace(sessionSource.LocalPath) == "" {
sessionSource.LocalPath = sessionPath
}
return &Config{
Pipeline: pipelineCfg,
Campaign: campaignCfg,
Session: sessionCfg,
PipelinePath: pipelinePath,
CampaignPath: campaignPath,
SessionPath: sessionPath,
StableInputs: stableInputs,
SessionSource: sessionSource,
}, nil
}
func campaignSessionPaths(paths ...string) (campaignPath, sessionPath string, err error) {
switch len(paths) {
case 1:
sessionPath = paths[0]
campaignPath = filepath.Join(filepath.Dir(sessionPath), "campaign.yml")
case 2:
campaignPath = paths[0]
sessionPath = paths[1]
default:
return "", "", fmt.Errorf("load config: expected session path or campaign and session paths")
}
return campaignPath, sessionPath, nil
}
func mergeCampaignSession(campaignCfg *CampaignConfig, sessionCfg *SessionConfig, campaignPath, sessionPath string) (ResolvedStableInputs, error) {
if campaignCfg == nil {
return ResolvedStableInputs{}, fmt.Errorf("campaign config is required")
}
if sessionCfg == nil {
return ResolvedStableInputs{}, fmt.Errorf("session config is required")
}
campaignName := CampaignID(campaignCfg)
sessionCampaign := strings.TrimSpace(sessionCfg.Campaign)
if sessionCampaign != "" && campaignName != "" && sessionCampaign != campaignName {
return ResolvedStableInputs{}, fmt.Errorf(
"campaign/session config invalid: session campaign %q does not match campaign config %q",
sessionCampaign,
campaignName,
)
}
if sessionCampaign == "" {
sessionCfg.Campaign = campaignName
}
stable := ResolvedStableInputs{
SpeakersFile: selectStableInput(
campaignCfg.Inputs.SpeakersFile,
sessionCfg.Inputs.SpeakersFile,
campaignPath,
sessionPath,
),
AutocorrectFile: selectStableInput(
campaignCfg.Inputs.AutocorrectFile,
sessionCfg.Inputs.AutocorrectFile,
campaignPath,
sessionPath,
),
GlossaryFile: selectStableInput(
campaignCfg.Inputs.GlossaryFile,
sessionCfg.Inputs.GlossaryFile,
campaignPath,
sessionPath,
),
}
sessionCfg.Inputs.SpeakersFile = stable.SpeakersFile.Path
sessionCfg.Inputs.AutocorrectFile = stable.AutocorrectFile.Path
sessionCfg.Inputs.GlossaryFile = stable.GlossaryFile.Path
return stable, nil
}
// CampaignID returns the canonical campaign identity from campaign config.
func CampaignID(cfg *CampaignConfig) string {
if cfg == nil {
return ""
}
return strings.TrimSpace(cfg.CampaignID)
}
func selectStableInput(campaignValue, sessionValue, campaignPath, sessionPath string) ResolvedInputFile {
if strings.TrimSpace(sessionValue) != "" {
return ResolvedInputFile{
Path: sessionValue,
ConfigPath: sessionPath,
Source: "session_config",
}
}
return ResolvedInputFile{
Path: campaignValue,
ConfigPath: campaignPath,
Source: "campaign_config",
}
}
func decodeStrictYAML(kind, path string, out any) error {
f, err := os.Open(path)
if err != nil {
return fmt.Errorf("%s file %q: open: %w", kind, path, err)
}
defer f.Close()
return decodeStrictYAMLFromReader(kind, path, f, out)
}
func decodeStrictYAMLFromReader(kind, path string, r io.Reader, out any) error {
dec := yaml.NewDecoder(r)
dec.KnownFields(true)
if err := dec.Decode(out); err != nil {
return fmt.Errorf("%s file %q: strict decode failed: %w", kind, path, err)
}
var extra any
if err := dec.Decode(&extra); err != nil && err != io.EOF {
return fmt.Errorf("%s file %q: trailing content decode failed: %w", kind, path, err)
}
return nil
}
var sessionTemplatePlaceholderPattern = regexp.MustCompile(`\{\{[^}]*\}\}`)
var sessionTemplateVariablePattern = regexp.MustCompile(`\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}`)
func rejectSessionTemplatePlaceholders(label, content string) error {
placeholders := sessionTemplatePlaceholderPattern.FindAllString(content, -1)
if len(placeholders) == 0 {
return nil
}
seen := map[string]struct{}{}
vars := make([]string, 0, len(placeholders))
for _, placeholder := range placeholders {
name := strings.TrimSpace(placeholder)
if match := sessionTemplateVariablePattern.FindStringSubmatch(placeholder); len(match) > 1 {
name = match[1]
}
if _, ok := seen[name]; ok {
continue
}
seen[name] = struct{}{}
vars = append(vars, name)
}
sort.Strings(vars)
return fmt.Errorf(
"session file %q contains template placeholder(s): %s; session.yml must be concrete; run narratio session init to generate it",
label,
strings.Join(vars, ", "),
)
}
func shortName(path, fallback string) string {
base := filepath.Base(path)
if base == "." || base == string(filepath.Separator) {
return fallback
}
return base
}
func applyPipelineDefaults(cfg *PipelineConfig) {
if cfg == nil {
return
}
applyWorkspaceDefaults(&cfg.Workspace)
applyCampaignsDefaults(&cfg.Campaigns)
applyStorageDefaults(&cfg.Storage)
applySpoolDefaults(&cfg.Spool)
applyCacheDefaults(&cfg.Cache)
applyPublishDefaults(&cfg.Publish)
applyWhisperXDefaults(&cfg.WhisperX)
applySeriatimDefaults(&cfg.Seriatim)
applyAuditaDefaults(&cfg.Audita)
if cfg.Normalize == nil {
cfg.Normalize = &NormalizeConfig{}
}
applyNormalizeDefaults(cfg.Normalize)
applyTrimDefaults(cfg.Trim)
applyScriptoriumDefaults(cfg.Scriptorium)
}
func applyCampaignsDefaults(cfg *CampaignsConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultCampaignsRoot
}
}
func applyWorkspaceDefaults(cfg *WorkspaceConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultWorkspaceRoot
}
}
func applyStorageDefaults(cfg *StorageConfig) {
if cfg == nil {
return
}
if cfg.S3 == nil {
cfg.S3 = &StorageS3Config{}
}
if cfg.S3.RootPrefix == "" {
cfg.S3.RootPrefix = DefaultStorageS3RootPrefix
}
if cfg.S3.AccessKeyIDEnv == "" {
cfg.S3.AccessKeyIDEnv = DefaultS3AccessKeyIDEnv
}
if cfg.S3.SecretKeyEnv == "" {
cfg.S3.SecretKeyEnv = DefaultS3SecretAccessKeyEnv
}
}
func applySpoolDefaults(cfg *SpoolConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultSpoolRoot
}
}
func applyCacheDefaults(cfg *CacheConfig) {
if cfg == nil {
return
}
if cfg.Root == "" {
cfg.Root = DefaultCacheRoot
}
if cfg.S3Audio == nil {
cfg.S3Audio = boolPtr(DefaultCacheS3Audio)
}
}
func applyPublishDefaults(cfg **PublishConfig) {
if cfg == nil {
return
}
if *cfg == nil {
*cfg = &PublishConfig{}
}
if (*cfg).Enabled == nil {
(*cfg).Enabled = boolPtr(DefaultArchiveEnabled)
}
if (*cfg).UploadRun == nil {
(*cfg).UploadRun = boolPtr(DefaultArchiveUploadRun)
}
if len((*cfg).Outputs) == 0 {
(*cfg).Outputs = append([]PublishOutputRule(nil), DefaultPublishOutputs...)
}
for i := range (*cfg).Outputs {
if (*cfg).Outputs[i].Required == nil {
(*cfg).Outputs[i].Required = boolPtr(true)
}
}
}
func applyWhisperXDefaults(cfg *WhisperXConfig) {
if cfg == nil {
return
}
if cfg.Language == "" {
cfg.Language = DefaultWhisperXLanguage
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultWhisperXTimeout
}
if cfg.RetryDelay == "" {
cfg.RetryDelay = DefaultWhisperXRetryDelay
}
if cfg.Concurrency == nil {
cfg.Concurrency = intPtr(DefaultWhisperXConcurrency)
}
if cfg.Retries == nil {
cfg.Retries = intPtr(DefaultWhisperXRetries)
}
}
func intPtr(v int) *int {
p := v
return &p
}
func applySeriatimDefaults(cfg *SeriatimConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultSeriatimBinary
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultSeriatimTimeout
}
if cfg.OutputSchema == "" {
cfg.OutputSchema = DefaultSeriatimOutputSchema
}
if cfg.CoalesceGap == nil {
cfg.CoalesceGap = float64Ptr(DefaultSeriatimCoalesceGap)
}
if cfg.Report == nil {
cfg.Report = boolPtr(DefaultSeriatimReport)
}
}
func applyAuditaDefaults(cfg *AuditaConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultAuditaBinary
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultAuditaTimeout
}
if cfg.Report == nil {
cfg.Report = boolPtr(DefaultAuditaReport)
}
}
func applyScriptoriumDefaults(cfg *ScriptoriumConfig) {
if cfg == nil {
return
}
if cfg.Binary == "" {
cfg.Binary = DefaultScriptoriumBinary
}
if cfg.Timeout == "" {
cfg.Timeout = DefaultScriptoriumTimeout
}
}
func applyTrimDefaults(cfg *TrimConfig) {
if cfg == nil {
return
}
if cfg.Bounds.Timeout == "" {
cfg.Bounds.Timeout = DefaultTrimBoundsTimeout
}
if cfg.Seriatim.Report == nil {
cfg.Seriatim.Report = boolPtr(DefaultTrimSeriatimReport)
}
}
func applyNormalizeDefaults(cfg *NormalizeConfig) {
if cfg == nil {
return
}
if cfg.OutputSchema == "" {
cfg.OutputSchema = DefaultNormalizeOutputSchema
}
if cfg.OutputPath == "" && !cfg.outputPathWasSet() {
cfg.OutputPath = DefaultNormalizeOutputPath
}
if cfg.Report == nil {
cfg.Report = boolPtr(DefaultNormalizeReport)
}
}
func float64Ptr(v float64) *float64 {
p := v
return &p
}
func boolPtr(v bool) *bool {
p := v
return &p
}