534 lines
14 KiB
Go
534 lines
14 KiB
Go
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
// LoadPipeline loads pipeline configuration from a YAML file with strict field checking.
|
|
func LoadPipeline(path string) (*PipelineConfig, error) {
|
|
var cfg PipelineConfig
|
|
if err := decodeStrictYAML("pipeline", path, &cfg); err != nil {
|
|
return nil, fmt.Errorf("load pipeline config: %w", err)
|
|
}
|
|
applyPipelineDefaults(&cfg)
|
|
return &cfg, nil
|
|
}
|
|
|
|
// LoadCampaign loads campaign configuration from a YAML file with strict field checking.
|
|
func LoadCampaign(path string) (*CampaignConfig, error) {
|
|
var cfg CampaignConfig
|
|
if err := decodeStrictYAML("campaign", path, &cfg); err != nil {
|
|
return nil, fmt.Errorf("load campaign config: %w", err)
|
|
}
|
|
return &cfg, nil
|
|
}
|
|
|
|
// LoadSession loads session configuration from a YAML file with strict field checking.
|
|
func LoadSession(path string) (*SessionConfig, error) {
|
|
return LoadSessionWithOptions(path, SessionLoadOptions{})
|
|
}
|
|
|
|
// SessionLoadOptions configures expected session identity checks.
|
|
type SessionLoadOptions struct {
|
|
SessionID string
|
|
PreviousSessionID string
|
|
}
|
|
|
|
// LoadSessionWithOptions loads session configuration from a YAML file with
|
|
// strict field checking.
|
|
func LoadSessionWithOptions(path string, opts SessionLoadOptions) (*SessionConfig, error) {
|
|
sessionBytes, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("load session config: session file %q: open: %w", path, err)
|
|
}
|
|
return LoadSessionBytesWithOptions(path, sessionBytes, opts)
|
|
}
|
|
|
|
// LoadSessionBytesWithOptions loads session configuration from YAML bytes with
|
|
// strict field checking.
|
|
func LoadSessionBytesWithOptions(label string, data []byte, opts SessionLoadOptions) (*SessionConfig, error) {
|
|
if err := rejectSessionTemplatePlaceholders(label, string(data)); err != nil {
|
|
return nil, fmt.Errorf("load session config: %w", err)
|
|
}
|
|
|
|
var cfg SessionConfig
|
|
if err := decodeStrictYAMLFromReader("session", label, strings.NewReader(string(data)), &cfg); err != nil {
|
|
return nil, fmt.Errorf("load session config: %w", err)
|
|
}
|
|
if strings.TrimSpace(opts.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != "" && strings.TrimSpace(cfg.SessionID) != strings.TrimSpace(opts.SessionID) {
|
|
return nil, fmt.Errorf(
|
|
"load session config: session file %q: session_id mismatch: --session-id %q does not match session_id %q",
|
|
label,
|
|
strings.TrimSpace(opts.SessionID),
|
|
strings.TrimSpace(cfg.SessionID),
|
|
)
|
|
}
|
|
if strings.TrimSpace(opts.PreviousSessionID) != "" &&
|
|
strings.TrimSpace(cfg.PreviousSessionID) != "" &&
|
|
strings.TrimSpace(cfg.PreviousSessionID) != strings.TrimSpace(opts.PreviousSessionID) {
|
|
return nil, fmt.Errorf(
|
|
"load session config: session file %q: previous_session_id mismatch: --previous-session-id %q does not match previous_session_id %q",
|
|
label,
|
|
strings.TrimSpace(opts.PreviousSessionID),
|
|
strings.TrimSpace(cfg.PreviousSessionID),
|
|
)
|
|
}
|
|
return &cfg, nil
|
|
}
|
|
|
|
// LoadPublishLockStoreBytes loads a mutable session lock store with strict
|
|
// field checking and source validation.
|
|
func LoadPublishLockStoreBytes(label string, data []byte, scriptorium *ScriptoriumConfig) (*PublishLockStore, error) {
|
|
var store PublishLockStore
|
|
if err := decodeStrictYAMLFromReader("publish lock store", label, strings.NewReader(string(data)), &store); err != nil {
|
|
return nil, fmt.Errorf("load publish lock store: %w", err)
|
|
}
|
|
locks, err := ValidatePublishLockRules(store.Locks, scriptorium, "locks")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("load publish lock store: %w", err)
|
|
}
|
|
store.Locks = locks
|
|
return &store, nil
|
|
}
|
|
|
|
// MarshalPublishLockStore serializes a mutable lock store as strict-compatible YAML.
|
|
func MarshalPublishLockStore(store *PublishLockStore) ([]byte, error) {
|
|
if store == nil {
|
|
store = &PublishLockStore{}
|
|
}
|
|
data, err := yaml.Marshal(store)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("marshal publish lock store: %w", err)
|
|
}
|
|
return data, nil
|
|
}
|
|
|
|
// Load loads and resolves combined pipeline, campaign, and session configuration.
|
|
// Passing only a session path is supported for package-internal compatibility;
|
|
// in that form campaign.yml is expected next to the session file.
|
|
func Load(pipelinePath string, paths ...string) (*Config, error) {
|
|
campaignPath, sessionPath, err := campaignSessionPaths(paths...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, SessionLoadOptions{})
|
|
}
|
|
|
|
// LoadWithSessionOptions loads and resolves combined pipeline, campaign, and
|
|
// session configuration with expected session identity checks.
|
|
func LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath string, sessionOpts SessionLoadOptions) (*Config, error) {
|
|
pipelineCfg, err := LoadPipeline(pipelinePath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
campaignCfg, err := LoadCampaign(campaignPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
sessionCfg, err := LoadSessionWithOptions(sessionPath, sessionOpts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return Resolve(pipelinePath, pipelineCfg, campaignPath, campaignCfg, sessionPath, sessionCfg, SessionSource{
|
|
Source: "session_config",
|
|
LocalPath: sessionPath,
|
|
})
|
|
}
|
|
|
|
// Resolve builds final stage-facing configuration from already loaded
|
|
// pipeline, campaign, and session documents.
|
|
func Resolve(pipelinePath string, pipelineCfg *PipelineConfig, campaignPath string, campaignCfg *CampaignConfig, sessionPath string, sessionCfg *SessionConfig, sessionSource SessionSource) (*Config, error) {
|
|
stableInputs, err := mergeCampaignSession(campaignCfg, sessionCfg, campaignPath, sessionPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if strings.TrimSpace(sessionSource.Source) == "" {
|
|
sessionSource.Source = "session_config"
|
|
}
|
|
if strings.TrimSpace(sessionSource.LocalPath) == "" {
|
|
sessionSource.LocalPath = sessionPath
|
|
}
|
|
|
|
return &Config{
|
|
Pipeline: pipelineCfg,
|
|
Campaign: campaignCfg,
|
|
Session: sessionCfg,
|
|
PipelinePath: pipelinePath,
|
|
CampaignPath: campaignPath,
|
|
SessionPath: sessionPath,
|
|
StableInputs: stableInputs,
|
|
SessionSource: sessionSource,
|
|
}, nil
|
|
}
|
|
|
|
func campaignSessionPaths(paths ...string) (campaignPath, sessionPath string, err error) {
|
|
switch len(paths) {
|
|
case 1:
|
|
sessionPath = paths[0]
|
|
campaignPath = filepath.Join(filepath.Dir(sessionPath), "campaign.yml")
|
|
case 2:
|
|
campaignPath = paths[0]
|
|
sessionPath = paths[1]
|
|
default:
|
|
return "", "", fmt.Errorf("load config: expected session path or campaign and session paths")
|
|
}
|
|
return campaignPath, sessionPath, nil
|
|
}
|
|
|
|
func mergeCampaignSession(campaignCfg *CampaignConfig, sessionCfg *SessionConfig, campaignPath, sessionPath string) (ResolvedStableInputs, error) {
|
|
if campaignCfg == nil {
|
|
return ResolvedStableInputs{}, fmt.Errorf("campaign config is required")
|
|
}
|
|
if sessionCfg == nil {
|
|
return ResolvedStableInputs{}, fmt.Errorf("session config is required")
|
|
}
|
|
|
|
campaignName := CampaignID(campaignCfg)
|
|
sessionCampaign := strings.TrimSpace(sessionCfg.Campaign)
|
|
if sessionCampaign != "" && campaignName != "" && sessionCampaign != campaignName {
|
|
return ResolvedStableInputs{}, fmt.Errorf(
|
|
"campaign/session config invalid: session campaign %q does not match campaign config %q",
|
|
sessionCampaign,
|
|
campaignName,
|
|
)
|
|
}
|
|
if sessionCampaign == "" {
|
|
sessionCfg.Campaign = campaignName
|
|
}
|
|
|
|
stable := ResolvedStableInputs{
|
|
SpeakersFile: selectStableInput(
|
|
campaignCfg.Inputs.SpeakersFile,
|
|
sessionCfg.Inputs.SpeakersFile,
|
|
campaignPath,
|
|
sessionPath,
|
|
),
|
|
AutocorrectFile: selectStableInput(
|
|
campaignCfg.Inputs.AutocorrectFile,
|
|
sessionCfg.Inputs.AutocorrectFile,
|
|
campaignPath,
|
|
sessionPath,
|
|
),
|
|
GlossaryFile: selectStableInput(
|
|
campaignCfg.Inputs.GlossaryFile,
|
|
sessionCfg.Inputs.GlossaryFile,
|
|
campaignPath,
|
|
sessionPath,
|
|
),
|
|
}
|
|
|
|
sessionCfg.Inputs.SpeakersFile = stable.SpeakersFile.Path
|
|
sessionCfg.Inputs.AutocorrectFile = stable.AutocorrectFile.Path
|
|
sessionCfg.Inputs.GlossaryFile = stable.GlossaryFile.Path
|
|
return stable, nil
|
|
}
|
|
|
|
// CampaignID returns the canonical campaign identity from campaign config.
|
|
func CampaignID(cfg *CampaignConfig) string {
|
|
if cfg == nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(cfg.CampaignID)
|
|
}
|
|
|
|
func selectStableInput(campaignValue, sessionValue, campaignPath, sessionPath string) ResolvedInputFile {
|
|
if strings.TrimSpace(sessionValue) != "" {
|
|
return ResolvedInputFile{
|
|
Path: sessionValue,
|
|
ConfigPath: sessionPath,
|
|
Source: "session_config",
|
|
}
|
|
}
|
|
return ResolvedInputFile{
|
|
Path: campaignValue,
|
|
ConfigPath: campaignPath,
|
|
Source: "campaign_config",
|
|
}
|
|
}
|
|
|
|
func decodeStrictYAML(kind, path string, out any) error {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return fmt.Errorf("%s file %q: open: %w", kind, path, err)
|
|
}
|
|
defer f.Close()
|
|
|
|
return decodeStrictYAMLFromReader(kind, path, f, out)
|
|
}
|
|
|
|
func decodeStrictYAMLFromReader(kind, path string, r io.Reader, out any) error {
|
|
dec := yaml.NewDecoder(r)
|
|
dec.KnownFields(true)
|
|
if err := dec.Decode(out); err != nil {
|
|
return fmt.Errorf("%s file %q: strict decode failed: %w", kind, path, err)
|
|
}
|
|
|
|
var extra any
|
|
if err := dec.Decode(&extra); err != nil && err != io.EOF {
|
|
return fmt.Errorf("%s file %q: trailing content decode failed: %w", kind, path, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
var sessionTemplatePlaceholderPattern = regexp.MustCompile(`\{\{[^}]*\}\}`)
|
|
var sessionTemplateVariablePattern = regexp.MustCompile(`\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}`)
|
|
|
|
func rejectSessionTemplatePlaceholders(label, content string) error {
|
|
placeholders := sessionTemplatePlaceholderPattern.FindAllString(content, -1)
|
|
if len(placeholders) == 0 {
|
|
return nil
|
|
}
|
|
seen := map[string]struct{}{}
|
|
vars := make([]string, 0, len(placeholders))
|
|
for _, placeholder := range placeholders {
|
|
name := strings.TrimSpace(placeholder)
|
|
if match := sessionTemplateVariablePattern.FindStringSubmatch(placeholder); len(match) > 1 {
|
|
name = match[1]
|
|
}
|
|
if _, ok := seen[name]; ok {
|
|
continue
|
|
}
|
|
seen[name] = struct{}{}
|
|
vars = append(vars, name)
|
|
}
|
|
sort.Strings(vars)
|
|
return fmt.Errorf(
|
|
"session file %q contains template placeholder(s): %s; session.yml must be concrete; run narratio session init to generate it",
|
|
label,
|
|
strings.Join(vars, ", "),
|
|
)
|
|
}
|
|
|
|
func shortName(path, fallback string) string {
|
|
base := filepath.Base(path)
|
|
if base == "." || base == string(filepath.Separator) {
|
|
return fallback
|
|
}
|
|
return base
|
|
}
|
|
|
|
func applyPipelineDefaults(cfg *PipelineConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
applyWorkspaceDefaults(&cfg.Workspace)
|
|
applyCampaignsDefaults(&cfg.Campaigns)
|
|
applyStorageDefaults(&cfg.Storage)
|
|
applySpoolDefaults(&cfg.Spool)
|
|
applyCacheDefaults(&cfg.Cache)
|
|
applyPublishDefaults(&cfg.Publish)
|
|
applyWhisperXDefaults(&cfg.WhisperX)
|
|
applySeriatimDefaults(&cfg.Seriatim)
|
|
applyAuditaDefaults(&cfg.Audita)
|
|
if cfg.Normalize == nil {
|
|
cfg.Normalize = &NormalizeConfig{}
|
|
}
|
|
applyNormalizeDefaults(cfg.Normalize)
|
|
applyTrimDefaults(cfg.Trim)
|
|
applyScriptoriumDefaults(cfg.Scriptorium)
|
|
}
|
|
|
|
func applyCampaignsDefaults(cfg *CampaignsConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Root == "" {
|
|
cfg.Root = DefaultCampaignsRoot
|
|
}
|
|
}
|
|
|
|
func applyWorkspaceDefaults(cfg *WorkspaceConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Root == "" {
|
|
cfg.Root = DefaultWorkspaceRoot
|
|
}
|
|
}
|
|
|
|
func applyStorageDefaults(cfg *StorageConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.S3 == nil {
|
|
cfg.S3 = &StorageS3Config{}
|
|
}
|
|
if cfg.S3.RootPrefix == "" {
|
|
cfg.S3.RootPrefix = DefaultStorageS3RootPrefix
|
|
}
|
|
if cfg.S3.AccessKeyIDEnv == "" {
|
|
cfg.S3.AccessKeyIDEnv = DefaultS3AccessKeyIDEnv
|
|
}
|
|
if cfg.S3.SecretKeyEnv == "" {
|
|
cfg.S3.SecretKeyEnv = DefaultS3SecretAccessKeyEnv
|
|
}
|
|
}
|
|
|
|
func applySpoolDefaults(cfg *SpoolConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Root == "" {
|
|
cfg.Root = DefaultSpoolRoot
|
|
}
|
|
}
|
|
|
|
func applyCacheDefaults(cfg *CacheConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Root == "" {
|
|
cfg.Root = DefaultCacheRoot
|
|
}
|
|
if cfg.S3Audio == nil {
|
|
cfg.S3Audio = boolPtr(DefaultCacheS3Audio)
|
|
}
|
|
}
|
|
|
|
func applyPublishDefaults(cfg **PublishConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if *cfg == nil {
|
|
*cfg = &PublishConfig{}
|
|
}
|
|
|
|
if (*cfg).Enabled == nil {
|
|
(*cfg).Enabled = boolPtr(DefaultArchiveEnabled)
|
|
}
|
|
if (*cfg).UploadRun == nil {
|
|
(*cfg).UploadRun = boolPtr(DefaultArchiveUploadRun)
|
|
}
|
|
if len((*cfg).Outputs) == 0 {
|
|
(*cfg).Outputs = append([]PublishOutputRule(nil), DefaultPublishOutputs...)
|
|
}
|
|
for i := range (*cfg).Outputs {
|
|
if (*cfg).Outputs[i].Required == nil {
|
|
(*cfg).Outputs[i].Required = boolPtr(true)
|
|
}
|
|
}
|
|
}
|
|
|
|
func applyWhisperXDefaults(cfg *WhisperXConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Language == "" {
|
|
cfg.Language = DefaultWhisperXLanguage
|
|
}
|
|
if cfg.Timeout == "" {
|
|
cfg.Timeout = DefaultWhisperXTimeout
|
|
}
|
|
if cfg.RetryDelay == "" {
|
|
cfg.RetryDelay = DefaultWhisperXRetryDelay
|
|
}
|
|
if cfg.Concurrency == nil {
|
|
cfg.Concurrency = intPtr(DefaultWhisperXConcurrency)
|
|
}
|
|
if cfg.Retries == nil {
|
|
cfg.Retries = intPtr(DefaultWhisperXRetries)
|
|
}
|
|
}
|
|
|
|
func intPtr(v int) *int {
|
|
p := v
|
|
return &p
|
|
}
|
|
|
|
func applySeriatimDefaults(cfg *SeriatimConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Binary == "" {
|
|
cfg.Binary = DefaultSeriatimBinary
|
|
}
|
|
if cfg.Timeout == "" {
|
|
cfg.Timeout = DefaultSeriatimTimeout
|
|
}
|
|
if cfg.OutputSchema == "" {
|
|
cfg.OutputSchema = DefaultSeriatimOutputSchema
|
|
}
|
|
if cfg.CoalesceGap == nil {
|
|
cfg.CoalesceGap = float64Ptr(DefaultSeriatimCoalesceGap)
|
|
}
|
|
if cfg.Report == nil {
|
|
cfg.Report = boolPtr(DefaultSeriatimReport)
|
|
}
|
|
}
|
|
|
|
func applyAuditaDefaults(cfg *AuditaConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Binary == "" {
|
|
cfg.Binary = DefaultAuditaBinary
|
|
}
|
|
if cfg.Timeout == "" {
|
|
cfg.Timeout = DefaultAuditaTimeout
|
|
}
|
|
if cfg.Report == nil {
|
|
cfg.Report = boolPtr(DefaultAuditaReport)
|
|
}
|
|
}
|
|
|
|
func applyScriptoriumDefaults(cfg *ScriptoriumConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Binary == "" {
|
|
cfg.Binary = DefaultScriptoriumBinary
|
|
}
|
|
if cfg.Timeout == "" {
|
|
cfg.Timeout = DefaultScriptoriumTimeout
|
|
}
|
|
}
|
|
|
|
func applyTrimDefaults(cfg *TrimConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.Bounds.Timeout == "" {
|
|
cfg.Bounds.Timeout = DefaultTrimBoundsTimeout
|
|
}
|
|
if cfg.Seriatim.Report == nil {
|
|
cfg.Seriatim.Report = boolPtr(DefaultTrimSeriatimReport)
|
|
}
|
|
}
|
|
|
|
func applyNormalizeDefaults(cfg *NormalizeConfig) {
|
|
if cfg == nil {
|
|
return
|
|
}
|
|
if cfg.OutputSchema == "" {
|
|
cfg.OutputSchema = DefaultNormalizeOutputSchema
|
|
}
|
|
if cfg.OutputPath == "" && !cfg.outputPathWasSet() {
|
|
cfg.OutputPath = DefaultNormalizeOutputPath
|
|
}
|
|
if cfg.Report == nil {
|
|
cfg.Report = boolPtr(DefaultNormalizeReport)
|
|
}
|
|
}
|
|
|
|
func float64Ptr(v float64) *float64 {
|
|
p := v
|
|
return &p
|
|
}
|
|
|
|
func boolPtr(v bool) *bool {
|
|
p := v
|
|
return &p
|
|
}
|