1009 lines
34 KiB
Go
1009 lines
34 KiB
Go
package config
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"net/url"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
|
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
|
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
|
)
|
|
|
|
// Validate checks resolved configuration for required fields and parseable durations.
|
|
func Validate(cfg *Config) error {
|
|
if cfg == nil {
|
|
return fmt.Errorf("config is nil")
|
|
}
|
|
if cfg.Pipeline == nil {
|
|
return fmt.Errorf("pipeline config is required")
|
|
}
|
|
if cfg.Campaign == nil {
|
|
return fmt.Errorf("campaign config is required")
|
|
}
|
|
if cfg.Session == nil {
|
|
return fmt.Errorf("session config is required")
|
|
}
|
|
|
|
if err := validatePipeline(cfg.Pipeline); err != nil {
|
|
return fmt.Errorf("pipeline config %q invalid: %w", shortName(cfg.PipelinePath, "pipeline.yml"), err)
|
|
}
|
|
if err := validateCampaign(cfg.Campaign); err != nil {
|
|
return fmt.Errorf("campaign config %q invalid: %w", shortName(cfg.CampaignPath, "campaign.yml"), err)
|
|
}
|
|
if err := validateSession(cfg.Session); err != nil {
|
|
return fmt.Errorf("session config %q invalid: %w", shortName(cfg.SessionPath, "session.yml"), err)
|
|
}
|
|
if err := validateCrossConfig(cfg.Pipeline, cfg.Session); err != nil {
|
|
return fmt.Errorf("pipeline/session config invalid: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func validateCampaign(cfg *CampaignConfig) error {
|
|
if cfg == nil {
|
|
return fmt.Errorf("campaign config is required")
|
|
}
|
|
if CampaignID(cfg) == "" {
|
|
return fmt.Errorf("campaign.campaign_id is required")
|
|
}
|
|
if err := pathsafe.ValidateOpaqueSegment(cfg.CampaignID); err != nil {
|
|
return fmt.Errorf("campaign.campaign_id: %w", err)
|
|
}
|
|
if strings.TrimSpace(cfg.Inputs.SpeakersFile) == "" {
|
|
return fmt.Errorf("campaign.inputs.speakers_file is required")
|
|
}
|
|
if strings.TrimSpace(cfg.Inputs.AutocorrectFile) == "" {
|
|
return fmt.Errorf("campaign.inputs.autocorrect_file is required")
|
|
}
|
|
if strings.TrimSpace(cfg.Inputs.GlossaryFile) == "" {
|
|
return fmt.Errorf("campaign.inputs.glossary_file is required")
|
|
}
|
|
if strings.TrimSpace(cfg.Inputs.PlayersFile) == "" {
|
|
return fmt.Errorf("campaign.inputs.players_file is required")
|
|
}
|
|
if strings.TrimSpace(cfg.Inputs.PartyFile) == "" {
|
|
return fmt.Errorf("campaign.inputs.party_file is required")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validatePipeline(cfg *PipelineConfig) error {
|
|
if strings.TrimSpace(cfg.Workspace.Root) == "" {
|
|
return fmt.Errorf("pipeline.workspace.root is required")
|
|
}
|
|
if campaignID := cfg.Campaigns.DefaultCampaignID; campaignID != "" {
|
|
if err := pathsafe.ValidateOpaqueSegment(campaignID); err != nil {
|
|
return fmt.Errorf("pipeline.campaigns.default_campaign_id: %w", err)
|
|
}
|
|
}
|
|
if err := validateSecrets(cfg.Secrets); err != nil {
|
|
return err
|
|
}
|
|
if err := validateStorage(cfg.Storage); err != nil {
|
|
return err
|
|
}
|
|
if err := validateSpool(cfg.Spool); err != nil {
|
|
return err
|
|
}
|
|
if err := validateCache(cfg.Cache); err != nil {
|
|
return err
|
|
}
|
|
if err := validateNotarius(cfg.Notarius, cfg.Scriptorium); err != nil {
|
|
return err
|
|
}
|
|
if err := validatePublish(cfg.Publish, cfg.Scriptorium, cfg.Notarius); err != nil {
|
|
return err
|
|
}
|
|
if err := validateWhisperX(cfg.WhisperX); err != nil {
|
|
return err
|
|
}
|
|
if err := validateSeriatim(cfg.Seriatim); err != nil {
|
|
return err
|
|
}
|
|
if err := validateAudita(cfg.Audita); err != nil {
|
|
return err
|
|
}
|
|
if err := validateNormalize(cfg.Normalize); err != nil {
|
|
return err
|
|
}
|
|
if err := validateTrim(cfg.Trim); err != nil {
|
|
return err
|
|
}
|
|
if err := validateRender(cfg.Render); err != nil {
|
|
return err
|
|
}
|
|
if err := validateScriptorium(cfg.Scriptorium, cfg.Notarius); err != nil {
|
|
return err
|
|
}
|
|
if err := validateNotification(cfg.Notification); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func validateNotification(cfg NotificationConfig) error {
|
|
if strings.EqualFold(strings.TrimSpace(cfg.Mode), DefaultNotificationMode) {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("pipeline.notification.mode must be %q until a notification provider is configured", DefaultNotificationMode)
|
|
}
|
|
|
|
func validateStorage(cfg StorageConfig) error {
|
|
backend := strings.ToLower(strings.TrimSpace(cfg.Backend))
|
|
switch backend {
|
|
case "", StorageBackendLocal:
|
|
if cfg.S3 != nil {
|
|
return fmt.Errorf("pipeline.storage.s3 is only supported when pipeline.storage.backend is s3")
|
|
}
|
|
return nil
|
|
case StorageBackendS3:
|
|
if cfg.S3 == nil {
|
|
return fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3")
|
|
}
|
|
default:
|
|
return fmt.Errorf("pipeline.storage.backend must be one of: local, s3")
|
|
}
|
|
if strings.TrimSpace(cfg.S3.RootPrefix) == "" {
|
|
return fmt.Errorf("pipeline.storage.s3.root_prefix must be non-empty")
|
|
}
|
|
if err := validateRelativeSafePath("pipeline.storage.s3.root_prefix", cfg.S3.RootPrefix); err != nil {
|
|
return err
|
|
}
|
|
if cfg.S3.Endpoint != "" && strings.TrimSpace(cfg.S3.Endpoint) == "" {
|
|
return fmt.Errorf("pipeline.storage.s3.endpoint must be non-empty when provided")
|
|
}
|
|
if err := validateEnvVarNameField("pipeline.storage.s3.access_key_id_env", cfg.S3.AccessKeyIDEnv); err != nil {
|
|
return err
|
|
}
|
|
if err := validateEnvVarNameField("pipeline.storage.s3.secret_access_key_env", cfg.S3.SecretKeyEnv); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateSpool(cfg SpoolConfig) error {
|
|
return nil
|
|
}
|
|
|
|
func validateCache(cfg CacheConfig) error {
|
|
if cfg.S3Audio != nil && *cfg.S3Audio && strings.TrimSpace(cfg.Root) == "" {
|
|
return fmt.Errorf("pipeline.cache.root is required when pipeline.cache.s3_audio is true")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig, notarius *NotariusConfig) error {
|
|
if cfg == nil {
|
|
return nil
|
|
}
|
|
configuredOutputs := scriptoriumOutputPathMap(scriptorium)
|
|
extractionOutputs := notariusOutputKeySet(notarius)
|
|
seenDest := map[string]struct{}{}
|
|
for i, item := range cfg.Outputs {
|
|
prefix := fmt.Sprintf("pipeline.publish.outputs[%d]", i)
|
|
source := strings.TrimSpace(item.Source)
|
|
if source == "" {
|
|
return fmt.Errorf("%s.source is required", prefix)
|
|
}
|
|
if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil {
|
|
return fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
|
}
|
|
dest := strings.TrimSpace(item.Dest)
|
|
if dest == "" {
|
|
derivedDest, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(source, "", configuredOutputs, extractionOutputs)
|
|
if err != nil {
|
|
return fmt.Errorf("%s.dest is required when destination cannot be derived from %q: %w", prefix, source, err)
|
|
}
|
|
dest = derivedDest
|
|
cfg.Outputs[i].Dest = derivedDest
|
|
}
|
|
normalizedDest, err := pathsafe.NormalizeRelativeDestination(dest)
|
|
if err != nil {
|
|
switch {
|
|
case errors.Is(err, pathsafe.ErrRelativePathAbsolute):
|
|
return fmt.Errorf("%s.dest must be a relative path", prefix)
|
|
case errors.Is(err, pathsafe.ErrRelativePathEscape):
|
|
return fmt.Errorf("%s.dest must not contain path traversal", prefix)
|
|
default:
|
|
return fmt.Errorf("%s.dest must be non-empty", prefix)
|
|
}
|
|
}
|
|
if _, ok := seenDest[normalizedDest]; ok {
|
|
return fmt.Errorf("%s.dest %q duplicates another publish output destination", prefix, dest)
|
|
}
|
|
seenDest[normalizedDest] = struct{}{}
|
|
}
|
|
locks, err := ValidatePublishLockRules(cfg.Locks, scriptorium, notarius, "pipeline.publish.locks")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cfg.Locks = locks
|
|
return nil
|
|
}
|
|
|
|
// ValidatePublishLockRules validates and normalizes source-based publish locks.
|
|
func ValidatePublishLockRules(locks []PublishLockRule, scriptorium *ScriptoriumConfig, notarius *NotariusConfig, label string) ([]PublishLockRule, error) {
|
|
seenLocks := map[string]struct{}{}
|
|
out := make([]PublishLockRule, 0, len(locks))
|
|
configuredOutputs := scriptoriumOutputPathMap(scriptorium)
|
|
extractionOutputs := notariusOutputKeySet(notarius)
|
|
if strings.TrimSpace(label) == "" {
|
|
label = "publish.locks"
|
|
}
|
|
for i, item := range locks {
|
|
prefix := fmt.Sprintf("%s[%d]", label, i)
|
|
source := strings.TrimSpace(item.Source)
|
|
if source == "" {
|
|
return nil, fmt.Errorf("%s.source is required", prefix)
|
|
}
|
|
if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil {
|
|
return nil, fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
|
}
|
|
if _, ok := seenLocks[source]; ok {
|
|
return nil, fmt.Errorf("%s.source %q duplicates another publish lock source", prefix, source)
|
|
}
|
|
seenLocks[source] = struct{}{}
|
|
out = append(out, PublishLockRule{
|
|
Source: source,
|
|
Reason: strings.TrimSpace(item.Reason),
|
|
})
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// MergePublishLockRules returns the union of static and remote locks. Static
|
|
// locks win when both sources contain the same lock.
|
|
func MergePublishLockRules(staticLocks, remoteLocks []PublishLockRule) []PublishLockRule {
|
|
out := make([]PublishLockRule, 0, len(staticLocks)+len(remoteLocks))
|
|
seen := map[string]struct{}{}
|
|
for _, item := range staticLocks {
|
|
source := strings.TrimSpace(item.Source)
|
|
if source == "" {
|
|
continue
|
|
}
|
|
out = append(out, PublishLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
|
|
seen[source] = struct{}{}
|
|
}
|
|
for _, item := range remoteLocks {
|
|
source := strings.TrimSpace(item.Source)
|
|
if source == "" {
|
|
continue
|
|
}
|
|
if _, ok := seen[source]; ok {
|
|
continue
|
|
}
|
|
out = append(out, PublishLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
|
|
seen[source] = struct{}{}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func scriptoriumOutputPathMap(scriptorium *ScriptoriumConfig) map[string]string {
|
|
out := map[string]string{}
|
|
if scriptorium == nil || len(scriptorium.Artifacts) == 0 {
|
|
return out
|
|
}
|
|
for key, artifactCfg := range scriptorium.Artifacts {
|
|
out[strings.TrimSpace(key)] = strings.TrimSpace(artifactCfg.OutputPath)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func notariusOutputKeySet(notarius *NotariusConfig) map[string]struct{} {
|
|
out := map[string]struct{}{}
|
|
if notarius == nil {
|
|
return out
|
|
}
|
|
for key := range notarius.Outputs {
|
|
trimmed := strings.TrimSpace(key)
|
|
if artifactpolicy.IsConfiguredKey(trimmed) {
|
|
out[trimmed] = struct{}{}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func validateSecrets(cfg *SecretsConfig) error {
|
|
if cfg == nil {
|
|
return nil
|
|
}
|
|
if strings.TrimSpace(cfg.EnvDir) == "" {
|
|
return fmt.Errorf("pipeline.secrets.env_dir must be non-empty when pipeline.secrets is configured")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateNormalize(cfg *NormalizeConfig) error {
|
|
if cfg == nil {
|
|
return nil
|
|
}
|
|
if strings.TrimSpace(cfg.OutputPath) == "" {
|
|
return fmt.Errorf("pipeline.normalize.output_path must be non-empty")
|
|
}
|
|
if err := validateSeriatimOutputSchema("pipeline.normalize.output_schema", cfg.OutputSchema); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateTrim(cfg *TrimConfig) error {
|
|
if cfg == nil {
|
|
return nil
|
|
}
|
|
if cfg.Enabled == nil {
|
|
return fmt.Errorf("pipeline.trim.enabled must be set (defaults should populate this)")
|
|
}
|
|
if !*cfg.Enabled {
|
|
return nil
|
|
}
|
|
|
|
if strings.TrimSpace(cfg.OutputPath) == "" {
|
|
return fmt.Errorf("pipeline.trim.output_path is required when pipeline.trim.enabled is true")
|
|
}
|
|
if strings.TrimSpace(cfg.Bounds.PromptID) == "" {
|
|
return fmt.Errorf("pipeline.trim.bounds.prompt_id is required when pipeline.trim.enabled is true")
|
|
}
|
|
if strings.TrimSpace(cfg.Bounds.TranscriptInputName) == "" {
|
|
return fmt.Errorf("pipeline.trim.bounds.transcript_input_name is required when pipeline.trim.enabled is true")
|
|
}
|
|
if strings.TrimSpace(cfg.Bounds.OutputPath) == "" {
|
|
return fmt.Errorf("pipeline.trim.bounds.output_path is required when pipeline.trim.enabled is true")
|
|
}
|
|
if err := validateDuration("pipeline.trim.bounds.timeout", cfg.Bounds.Timeout); err != nil {
|
|
return err
|
|
}
|
|
if cfg.Bounds.RenderDebug && strings.TrimSpace(cfg.Bounds.RenderOutputPath) == "" {
|
|
return fmt.Errorf("pipeline.trim.bounds.render_output_path is required when pipeline.trim.bounds.render_debug is true")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func validateRender(cfg *RenderConfig) error {
|
|
if cfg == nil {
|
|
return nil
|
|
}
|
|
if cfg.Enabled == nil {
|
|
return fmt.Errorf("pipeline.render.enabled must be set (defaults should populate this)")
|
|
}
|
|
if cfg.IncludeTimestamps == nil {
|
|
return fmt.Errorf("pipeline.render.include_timestamps must be set (defaults should populate this)")
|
|
}
|
|
if cfg.IncludeSegmentIDs == nil {
|
|
return fmt.Errorf("pipeline.render.include_segment_ids must be set (defaults should populate this)")
|
|
}
|
|
format := strings.TrimSpace(cfg.Format)
|
|
if format != "markdown" {
|
|
return fmt.Errorf("pipeline.render.format must be markdown")
|
|
}
|
|
if cfg.Title != "" && strings.TrimSpace(cfg.Title) == "" {
|
|
return fmt.Errorf("pipeline.render.title must be non-empty when provided")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateWhisperX(cfg WhisperXConfig) error {
|
|
if strings.TrimSpace(cfg.TranscribeURL) == "" {
|
|
return fmt.Errorf("pipeline.whisperx.transcribe_url is required")
|
|
}
|
|
u, err := url.Parse(cfg.TranscribeURL)
|
|
if err != nil || !u.IsAbs() || u.Host == "" || (strings.ToLower(u.Scheme) != "http" && strings.ToLower(u.Scheme) != "https") {
|
|
if err != nil {
|
|
return fmt.Errorf("pipeline.whisperx.transcribe_url must be an absolute http or https URL: %w", err)
|
|
}
|
|
return fmt.Errorf("pipeline.whisperx.transcribe_url must be an absolute http or https URL")
|
|
}
|
|
if err := validateDuration("pipeline.whisperx.timeout", cfg.Timeout); err != nil {
|
|
return err
|
|
}
|
|
if err := validateDuration("pipeline.whisperx.retry_delay", cfg.RetryDelay); err != nil {
|
|
return err
|
|
}
|
|
if cfg.Retries == nil {
|
|
return fmt.Errorf("pipeline.whisperx.retries must be set (defaults should populate this)")
|
|
}
|
|
if *cfg.Retries < 0 {
|
|
return fmt.Errorf("pipeline.whisperx.retries must be >= 0")
|
|
}
|
|
if cfg.Concurrency == nil {
|
|
return fmt.Errorf("pipeline.whisperx.concurrency must be set (defaults should populate this)")
|
|
}
|
|
if *cfg.Concurrency <= 0 {
|
|
return fmt.Errorf("pipeline.whisperx.concurrency must be > 0")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateSeriatim(cfg SeriatimConfig) error {
|
|
if strings.TrimSpace(cfg.Binary) == "" {
|
|
return fmt.Errorf("pipeline.seriatim.binary is required")
|
|
}
|
|
if err := validateDuration("pipeline.seriatim.timeout", cfg.Timeout); err != nil {
|
|
return err
|
|
}
|
|
if err := validateSeriatimOutputSchema("pipeline.seriatim.output_schema", cfg.OutputSchema); err != nil {
|
|
return err
|
|
}
|
|
|
|
if cfg.CoalesceGap == nil {
|
|
return fmt.Errorf("pipeline.seriatim.coalesce_gap must be set (defaults should populate this)")
|
|
}
|
|
if *cfg.CoalesceGap < 0 {
|
|
return fmt.Errorf("pipeline.seriatim.coalesce_gap must be >= 0")
|
|
}
|
|
|
|
for _, item := range []struct {
|
|
name string
|
|
value *float64
|
|
}{
|
|
{name: "pipeline.seriatim.env.overlap_word_run_gap", value: cfg.Env.OverlapWordRunGap},
|
|
{name: "pipeline.seriatim.env.overlap_word_run_reorder_window", value: cfg.Env.OverlapWordRunReorderWindow},
|
|
{name: "pipeline.seriatim.env.backchannel_max_duration", value: cfg.Env.BackchannelMaxDuration},
|
|
{name: "pipeline.seriatim.env.filler_max_duration", value: cfg.Env.FillerMaxDuration},
|
|
} {
|
|
if item.value != nil && *item.value <= 0 {
|
|
return fmt.Errorf("%s must be > 0 when provided", item.name)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func validateSeriatimOutputSchema(field, value string) error {
|
|
switch strings.TrimSpace(value) {
|
|
case "seriatim-minimal", "seriatim-intermediate", "seriatim-full":
|
|
return nil
|
|
default:
|
|
return fmt.Errorf("%s must be one of: seriatim-minimal, seriatim-intermediate, seriatim-full", field)
|
|
}
|
|
}
|
|
|
|
func validateAudita(cfg AuditaConfig) error {
|
|
if strings.TrimSpace(cfg.Binary) == "" {
|
|
return fmt.Errorf("pipeline.audita.binary is required")
|
|
}
|
|
if err := validateDuration("pipeline.audita.timeout", cfg.Timeout); err != nil {
|
|
return err
|
|
}
|
|
for i, m := range cfg.Modules {
|
|
module := strings.TrimSpace(m)
|
|
if module == "" {
|
|
return fmt.Errorf("pipeline.audita.modules[%d] must be non-empty", i)
|
|
}
|
|
switch module {
|
|
case "glossary", "homophones", "spoken_word", "grammar":
|
|
default:
|
|
return fmt.Errorf("pipeline.audita.modules[%d] must be one of: glossary, homophones, spoken_word, grammar", i)
|
|
}
|
|
}
|
|
if strings.TrimSpace(cfg.BaseURL) != "" {
|
|
u, err := url.Parse(cfg.BaseURL)
|
|
if err != nil || u.Scheme == "" || u.Host == "" {
|
|
if err != nil {
|
|
return fmt.Errorf("pipeline.audita.base_url must be a valid URL: %w", err)
|
|
}
|
|
return fmt.Errorf("pipeline.audita.base_url must be a valid URL")
|
|
}
|
|
}
|
|
if cfg.TotalLLMConcurrency != nil && *cfg.TotalLLMConcurrency <= 0 {
|
|
return fmt.Errorf("pipeline.audita.total_llm_concurrency must be > 0")
|
|
}
|
|
if cfg.ProposalLLMConcurrency != nil && *cfg.ProposalLLMConcurrency <= 0 {
|
|
return fmt.Errorf("pipeline.audita.proposal_llm_concurrency must be > 0")
|
|
}
|
|
if cfg.ValidationLLMConcurrency != nil && *cfg.ValidationLLMConcurrency <= 0 {
|
|
return fmt.Errorf("pipeline.audita.validation_llm_concurrency must be > 0")
|
|
}
|
|
if strings.TrimSpace(cfg.TranscriptDescription) == "" && cfg.TranscriptDescription != "" {
|
|
return fmt.Errorf("pipeline.audita.transcript_description must be non-empty when provided")
|
|
}
|
|
if strings.TrimSpace(cfg.ConfigPath) == "" && cfg.ConfigPath != "" {
|
|
return fmt.Errorf("pipeline.audita.config_path must be non-empty when provided")
|
|
}
|
|
switch strings.TrimSpace(cfg.OutputSchema) {
|
|
case "", "bare-segments", "audita-v1":
|
|
default:
|
|
return fmt.Errorf("pipeline.audita.output_schema must be one of: bare-segments, audita-v1")
|
|
}
|
|
switch strings.TrimSpace(cfg.WorkDirRetention) {
|
|
case "", "always", "auto", "never":
|
|
default:
|
|
return fmt.Errorf("pipeline.audita.work_dir_retention must be one of: always, auto, never")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateNotarius(cfg *NotariusConfig, scriptorium *ScriptoriumConfig) error {
|
|
if cfg == nil || !cfg.Enabled {
|
|
return nil
|
|
}
|
|
if strings.TrimSpace(cfg.Binary) == "" {
|
|
return fmt.Errorf("pipeline.notarius.binary is required when pipeline.notarius.enabled is true")
|
|
}
|
|
if strings.TrimSpace(cfg.ConfigPath) == "" {
|
|
return fmt.Errorf("pipeline.notarius.config_path is required when pipeline.notarius.enabled is true")
|
|
}
|
|
if strings.TrimSpace(cfg.PipelineID) == "" {
|
|
return fmt.Errorf("pipeline.notarius.pipeline_id is required when pipeline.notarius.enabled is true")
|
|
}
|
|
if len(cfg.Outputs) == 0 {
|
|
return fmt.Errorf("pipeline.notarius.outputs must contain at least one output when pipeline.notarius.enabled is true")
|
|
}
|
|
duration, err := time.ParseDuration(strings.TrimSpace(cfg.Timeout))
|
|
if err != nil {
|
|
return fmt.Errorf("pipeline.notarius.timeout must be a valid duration: %w", err)
|
|
}
|
|
if duration <= 0 {
|
|
return fmt.Errorf("pipeline.notarius.timeout must be positive")
|
|
}
|
|
if strings.TrimSpace(cfg.WorkingDirectory) == "" {
|
|
return fmt.Errorf("pipeline.notarius.working_directory is required when pipeline.notarius.enabled is true")
|
|
}
|
|
|
|
reservedSources := map[string]string{}
|
|
for _, spec := range artifactmodel.RuntimeTranscriptArtifacts() {
|
|
reservedSources[spec.SourceID] = "built-in source"
|
|
}
|
|
reservedSources[artifactpolicy.SourceBoundsSession] = "built-in source"
|
|
if scriptorium != nil {
|
|
for key := range scriptorium.Artifacts {
|
|
normalizedKey := strings.TrimSpace(key)
|
|
reservedSources[artifactpolicy.ConfiguredSourceID(normalizedKey)] = "configured Scriptorium source"
|
|
reservedSources[artifactpolicy.PreviousSessionSourceID(normalizedKey)] = "previous-session source"
|
|
}
|
|
}
|
|
|
|
rawKeys := make([]string, 0, len(cfg.Outputs))
|
|
for key := range cfg.Outputs {
|
|
rawKeys = append(rawKeys, key)
|
|
}
|
|
sort.Strings(rawKeys)
|
|
normalizedOutputs := make(map[string]NotariusOutputConfig, len(cfg.Outputs))
|
|
sourceOwners := map[string]string{}
|
|
laneOwners := map[string]string{}
|
|
for _, rawKey := range rawKeys {
|
|
output := cfg.Outputs[rawKey]
|
|
key := strings.TrimSpace(rawKey)
|
|
if !artifactpolicy.IsConfiguredKey(key) {
|
|
return fmt.Errorf("pipeline.notarius.outputs keys must match ^[a-z][a-z0-9_]*$")
|
|
}
|
|
sourceID := artifactpolicy.ExtractionSourceID(key)
|
|
if previousKey, ok := sourceOwners[sourceID]; ok {
|
|
return fmt.Errorf("pipeline.notarius.outputs keys %q and %q normalize to duplicate source %q", previousKey, rawKey, sourceID)
|
|
}
|
|
if owner, ok := reservedSources[sourceID]; ok {
|
|
return fmt.Errorf("pipeline.notarius.outputs.%s source %q collides with %s", key, sourceID, owner)
|
|
}
|
|
sourceOwners[sourceID] = rawKey
|
|
|
|
output.LaneID = strings.TrimSpace(output.LaneID)
|
|
output.MediaType = strings.TrimSpace(output.MediaType)
|
|
output.SchemaID = strings.TrimSpace(output.SchemaID)
|
|
output.SchemaVersion = strings.TrimSpace(output.SchemaVersion)
|
|
output.ModuleKey = strings.TrimSpace(output.ModuleKey)
|
|
prefix := "pipeline.notarius.outputs." + key
|
|
if output.LaneID == "" {
|
|
return fmt.Errorf("%s.lane_id is required", prefix)
|
|
}
|
|
if previousKey, ok := laneOwners[output.LaneID]; ok {
|
|
return fmt.Errorf("%s.lane_id %q duplicates pipeline.notarius.outputs.%s.lane_id", prefix, output.LaneID, previousKey)
|
|
}
|
|
laneOwners[output.LaneID] = key
|
|
if output.MediaType == "" {
|
|
return fmt.Errorf("%s.media_type is required", prefix)
|
|
}
|
|
if output.SchemaID == "" {
|
|
return fmt.Errorf("%s.schema_id is required", prefix)
|
|
}
|
|
if output.SchemaVersion == "" {
|
|
return fmt.Errorf("%s.schema_version is required", prefix)
|
|
}
|
|
normalizedOutputs[key] = output
|
|
}
|
|
cfg.Binary = strings.TrimSpace(cfg.Binary)
|
|
cfg.ConfigPath = filepath.Clean(cfg.ConfigPath)
|
|
cfg.PipelineID = strings.TrimSpace(cfg.PipelineID)
|
|
cfg.Timeout = strings.TrimSpace(cfg.Timeout)
|
|
cfg.WorkingDirectory = filepath.Clean(cfg.WorkingDirectory)
|
|
cfg.Outputs = normalizedOutputs
|
|
return nil
|
|
}
|
|
|
|
func validateScriptorium(cfg *ScriptoriumConfig, notarius *NotariusConfig) error {
|
|
if cfg == nil {
|
|
return nil
|
|
}
|
|
if strings.TrimSpace(cfg.Binary) == "" {
|
|
return fmt.Errorf("pipeline.scriptorium.binary is required when pipeline.scriptorium is configured")
|
|
}
|
|
if cfg.ConfigPath != "" && strings.TrimSpace(cfg.ConfigPath) == "" {
|
|
return fmt.Errorf("pipeline.scriptorium.config_path must be non-empty when provided")
|
|
}
|
|
if err := validateDuration("pipeline.scriptorium.timeout", cfg.Timeout); err != nil {
|
|
return err
|
|
}
|
|
|
|
configuredArtifacts := make(map[string]struct{}, len(cfg.Artifacts))
|
|
referencedArtifacts := make(map[string]struct{})
|
|
for artifactName := range cfg.Artifacts {
|
|
if !artifactpolicy.IsConfiguredKey(artifactName) {
|
|
return fmt.Errorf("pipeline.scriptorium.artifacts keys must match ^[a-z][a-z0-9_]*$")
|
|
}
|
|
configuredArtifacts[artifactName] = struct{}{}
|
|
}
|
|
|
|
for artifactName, artifactCfg := range cfg.Artifacts {
|
|
if artifactCfg.Enabled && strings.TrimSpace(artifactCfg.PromptID) == "" {
|
|
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.prompt_id is required when enabled", artifactName)
|
|
}
|
|
if artifactCfg.Enabled && strings.TrimSpace(artifactCfg.OutputPath) == "" {
|
|
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.output_path is required when enabled", artifactName)
|
|
}
|
|
if strings.TrimSpace(artifactCfg.OutputPath) != "" {
|
|
pathField := "pipeline.scriptorium.artifacts." + artifactName + ".output_path"
|
|
if err := validateRelativeSafePath(pathField, artifactCfg.OutputPath); err != nil {
|
|
return err
|
|
}
|
|
if err := validatePathWithinRoot(pathField, artifactCfg.OutputPath, DefaultScriptoriumArtifactOutputRoot); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := validateDuration("pipeline.scriptorium.artifacts."+artifactName+".timeout", artifactCfg.Timeout); err != nil {
|
|
return err
|
|
}
|
|
|
|
depSet := make(map[string]struct{}, len(artifactCfg.DependsOn))
|
|
for i, depName := range artifactCfg.DependsOn {
|
|
trimmedDep := strings.TrimSpace(depName)
|
|
field := fmt.Sprintf("pipeline.scriptorium.artifacts.%s.depends_on[%d]", artifactName, i)
|
|
if trimmedDep == "" {
|
|
return fmt.Errorf("%s must be non-empty", field)
|
|
}
|
|
if _, ok := configuredArtifacts[trimmedDep]; !ok {
|
|
return fmt.Errorf("%s %q is not a configured artifact key", field, depName)
|
|
}
|
|
if trimmedDep == artifactName {
|
|
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.depends_on must not include itself", artifactName)
|
|
}
|
|
depSet[trimmedDep] = struct{}{}
|
|
referencedArtifacts[trimmedDep] = struct{}{}
|
|
}
|
|
|
|
for inputName, inputCfg := range artifactCfg.Inputs {
|
|
trimmedInputName := strings.TrimSpace(inputName)
|
|
if trimmedInputName == "" {
|
|
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs keys must be non-empty", artifactName)
|
|
}
|
|
source := strings.TrimSpace(inputCfg.Source)
|
|
if source == "" {
|
|
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs.%s.source is required", artifactName, inputName)
|
|
}
|
|
|
|
referencedArtifact, err := validateScriptoriumInputSource(artifactName, inputName, source, configuredArtifacts, notariusOutputKeySet(notarius))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if referencedArtifact != "" {
|
|
if _, ok := depSet[referencedArtifact]; !ok {
|
|
return fmt.Errorf(
|
|
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q requires depends_on entry %q",
|
|
artifactName,
|
|
inputName,
|
|
source,
|
|
referencedArtifact,
|
|
)
|
|
}
|
|
referencedArtifacts[referencedArtifact] = struct{}{}
|
|
}
|
|
}
|
|
for varName, varValue := range artifactCfg.Vars {
|
|
if strings.TrimSpace(varName) == "" {
|
|
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.vars keys must be non-empty", artifactName)
|
|
}
|
|
switch varValue.(type) {
|
|
case bool, string:
|
|
default:
|
|
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.vars.%s must be a string or boolean", artifactName, varName)
|
|
}
|
|
}
|
|
}
|
|
|
|
for artifactName := range referencedArtifacts {
|
|
artifactCfg := cfg.Artifacts[artifactName]
|
|
if strings.TrimSpace(artifactCfg.OutputPath) == "" {
|
|
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.output_path is required when artifact is referenced", artifactName)
|
|
}
|
|
}
|
|
|
|
if err := validateEnabledArtifactDependencyCycles(cfg.Artifacts); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func validateSession(cfg *SessionConfig) error {
|
|
if err := validateSessionIdentifier("session.session_id", cfg.SessionID, true); err != nil {
|
|
return err
|
|
}
|
|
if err := validateSessionIdentifier("session.previous_session_id", cfg.PreviousSessionID, false); err != nil {
|
|
return err
|
|
}
|
|
if strings.TrimSpace(cfg.PreviousSessionID) != "" && strings.TrimSpace(cfg.PreviousSessionID) == strings.TrimSpace(cfg.SessionID) {
|
|
return fmt.Errorf("session.previous_session_id must not equal session.session_id")
|
|
}
|
|
if strings.TrimSpace(cfg.Campaign) == "" {
|
|
return fmt.Errorf("session.campaign is required")
|
|
}
|
|
if err := pathsafe.ValidateOpaqueSegment(cfg.Campaign); err != nil {
|
|
return fmt.Errorf("session.campaign: %w", err)
|
|
}
|
|
|
|
if strings.TrimSpace(cfg.Inputs.SpeakersFile) == "" {
|
|
return fmt.Errorf("session.inputs.speakers_file is required")
|
|
}
|
|
if strings.TrimSpace(cfg.Inputs.AutocorrectFile) == "" {
|
|
return fmt.Errorf("session.inputs.autocorrect_file is required")
|
|
}
|
|
if strings.TrimSpace(cfg.Inputs.GlossaryFile) == "" {
|
|
return fmt.Errorf("session.inputs.glossary_file is required")
|
|
}
|
|
if strings.TrimSpace(cfg.Inputs.PlayersFile) == "" {
|
|
return fmt.Errorf("session.inputs.players_file is required")
|
|
}
|
|
if strings.TrimSpace(cfg.Inputs.PartyFile) == "" {
|
|
return fmt.Errorf("session.inputs.party_file is required")
|
|
}
|
|
|
|
hasAudioDir := strings.TrimSpace(cfg.Inputs.AudioDir) != ""
|
|
hasAudioFiles := len(cfg.Inputs.AudioFiles) > 0
|
|
hasAudioS3 := cfg.Inputs.AudioS3 != nil
|
|
if hasAudioS3 {
|
|
if strings.TrimSpace(cfg.Inputs.AudioS3.Prefix) == "" {
|
|
return fmt.Errorf("session.inputs.audio_s3.prefix is required when session.inputs.audio_s3 is configured")
|
|
}
|
|
if err := validateRelativeSafePath("session.inputs.audio_s3.prefix", cfg.Inputs.AudioS3.Prefix); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if hasAudioS3 && (hasAudioDir || hasAudioFiles) {
|
|
return fmt.Errorf("session.inputs.audio_dir/audio_files and session.inputs.audio_s3 are mutually exclusive")
|
|
}
|
|
if !hasAudioDir && !hasAudioFiles && !hasAudioS3 {
|
|
return fmt.Errorf("session.inputs requires audio_dir, at least one audio_files entry, or audio_s3")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func validateSessionIdentifier(fieldName, value string, required bool) error {
|
|
if strings.TrimSpace(value) == "" {
|
|
if required {
|
|
return fmt.Errorf("%s is required", fieldName)
|
|
}
|
|
return nil
|
|
}
|
|
if err := pathsafe.ValidateOpaqueSegment(value); err != nil {
|
|
return fmt.Errorf("%s: %w", fieldName, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateCrossConfig(pipeline *PipelineConfig, session *SessionConfig) error {
|
|
if pipeline == nil || session == nil {
|
|
return nil
|
|
}
|
|
|
|
audioS3Enabled := session.Inputs.AudioS3 != nil
|
|
publishUploadEnabled := publishUploadConfiguredForS3(pipeline)
|
|
if audioS3Enabled && !strings.EqualFold(strings.TrimSpace(pipeline.Storage.Backend), StorageBackendS3) {
|
|
return fmt.Errorf("pipeline.storage.backend must be s3 when session.inputs.audio_s3 is configured")
|
|
}
|
|
if !audioS3Enabled && !publishUploadEnabled {
|
|
return nil
|
|
}
|
|
if pipeline.Storage.S3 == nil || strings.TrimSpace(pipeline.Storage.S3.Bucket) == "" {
|
|
return fmt.Errorf("pipeline.storage.s3.bucket is required when S3 session audio or publish upload is enabled")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func publishUploadConfiguredForS3(pipeline *PipelineConfig) bool {
|
|
if pipeline == nil || pipeline.Publish == nil {
|
|
return false
|
|
}
|
|
if !strings.EqualFold(strings.TrimSpace(pipeline.Storage.Backend), StorageBackendS3) {
|
|
return false
|
|
}
|
|
enabled := true
|
|
if pipeline.Publish.Enabled != nil {
|
|
enabled = *pipeline.Publish.Enabled
|
|
}
|
|
upload := true
|
|
if pipeline.Publish.UploadRun != nil {
|
|
upload = *pipeline.Publish.UploadRun
|
|
}
|
|
return enabled && upload
|
|
}
|
|
|
|
var windowsAbsPathRE = regexp.MustCompile(`^[A-Za-z]:[\\/].*`)
|
|
var envVarNameRE = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
|
|
|
func validateScriptoriumInputSource(
|
|
artifactName, inputName, source string,
|
|
configuredArtifacts map[string]struct{},
|
|
extractionOutputs map[string]struct{},
|
|
) (string, error) {
|
|
trimmedSource := strings.TrimSpace(source)
|
|
descriptor, err := artifactpolicy.DescribeScriptoriumInputSource(trimmedSource)
|
|
if err != nil {
|
|
if errors.Is(err, artifactpolicy.ErrInvalidPreviousSessionSource) {
|
|
return "", fmt.Errorf(
|
|
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q must reference configured artifact key matching ^[a-z][a-z0-9_]*$",
|
|
artifactName,
|
|
inputName,
|
|
source,
|
|
)
|
|
}
|
|
return "", fmt.Errorf(
|
|
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported",
|
|
artifactName,
|
|
inputName,
|
|
source,
|
|
)
|
|
}
|
|
if err := artifactpolicy.ValidateInputReference(descriptor, configuredArtifacts, extractionOutputs); err != nil {
|
|
var unknownConfigured *artifactpolicy.UnknownConfiguredArtifactError
|
|
if errors.As(err, &unknownConfigured) {
|
|
return "", fmt.Errorf(
|
|
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q references unknown artifact %q",
|
|
artifactName,
|
|
inputName,
|
|
source,
|
|
unknownConfigured.ConfiguredKey,
|
|
)
|
|
}
|
|
var unknownExtraction *artifactpolicy.UnknownExtractionArtifactError
|
|
if errors.As(err, &unknownExtraction) {
|
|
return "", fmt.Errorf(
|
|
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q references unknown extraction output %q",
|
|
artifactName,
|
|
inputName,
|
|
source,
|
|
unknownExtraction.ConfiguredKey,
|
|
)
|
|
}
|
|
return "", fmt.Errorf(
|
|
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported",
|
|
artifactName,
|
|
inputName,
|
|
source,
|
|
)
|
|
}
|
|
if descriptor.Source.Kind == artifactpolicy.SourceKindConfiguredArtifact {
|
|
return descriptor.Source.ConfiguredKey, nil
|
|
}
|
|
return "", nil
|
|
}
|
|
|
|
func validateEnvVarNameField(fieldName, value string) error {
|
|
trimmed := strings.TrimSpace(value)
|
|
if trimmed == "" {
|
|
return fmt.Errorf("%s must be non-empty", fieldName)
|
|
}
|
|
if !envVarNameRE.MatchString(trimmed) {
|
|
return fmt.Errorf("%s must be a valid environment variable name", fieldName)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateRelativeSafePath(fieldName, value string) error {
|
|
trimmed := strings.TrimSpace(value)
|
|
if trimmed == "" {
|
|
return fmt.Errorf("%s must be non-empty", fieldName)
|
|
}
|
|
if filepath.IsAbs(trimmed) || strings.HasPrefix(trimmed, "/") || strings.HasPrefix(trimmed, "\\") || windowsAbsPathRE.MatchString(trimmed) {
|
|
return fmt.Errorf("%s must be a relative path", fieldName)
|
|
}
|
|
|
|
normalized := strings.ReplaceAll(trimmed, "\\", "/")
|
|
for _, segment := range strings.Split(normalized, "/") {
|
|
if segment == ".." {
|
|
return fmt.Errorf("%s must not contain path traversal", fieldName)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validatePathWithinRoot(fieldName, value, root string) error {
|
|
normalizedValue := filepath.ToSlash(filepath.Clean(strings.TrimSpace(value)))
|
|
normalizedRoot := filepath.ToSlash(filepath.Clean(strings.TrimSpace(root)))
|
|
if normalizedValue == normalizedRoot {
|
|
return nil
|
|
}
|
|
if strings.HasPrefix(normalizedValue, normalizedRoot+"/") {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("%s must be under %s/", fieldName, normalizedRoot)
|
|
}
|
|
|
|
func validateEnabledArtifactDependencyCycles(artifacts map[string]ScriptoriumArtifactConfig) error {
|
|
if len(artifacts) == 0 {
|
|
return nil
|
|
}
|
|
|
|
enabled := make(map[string]struct{}, len(artifacts))
|
|
graph := make(map[string][]string, len(artifacts))
|
|
for name, cfg := range artifacts {
|
|
if !cfg.Enabled {
|
|
continue
|
|
}
|
|
enabled[name] = struct{}{}
|
|
}
|
|
for name, cfg := range artifacts {
|
|
if !cfg.Enabled {
|
|
continue
|
|
}
|
|
for _, dep := range cfg.DependsOn {
|
|
trimmedDep := strings.TrimSpace(dep)
|
|
if _, ok := enabled[trimmedDep]; ok {
|
|
graph[name] = append(graph[name], trimmedDep)
|
|
}
|
|
}
|
|
}
|
|
|
|
visiting := make(map[string]bool, len(enabled))
|
|
visited := make(map[string]bool, len(enabled))
|
|
|
|
var visit func(node string) error
|
|
visit = func(node string) error {
|
|
if visiting[node] {
|
|
return fmt.Errorf("pipeline.scriptorium.artifacts enabled dependencies must not contain cycles")
|
|
}
|
|
if visited[node] {
|
|
return nil
|
|
}
|
|
visiting[node] = true
|
|
for _, dep := range graph[node] {
|
|
if err := visit(dep); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
visiting[node] = false
|
|
visited[node] = true
|
|
return nil
|
|
}
|
|
|
|
for node := range enabled {
|
|
if err := visit(node); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateDuration(fieldName, value string) error {
|
|
trimmed := strings.TrimSpace(value)
|
|
if trimmed == "" {
|
|
return nil
|
|
}
|
|
|
|
duration, err := time.ParseDuration(trimmed)
|
|
if err != nil {
|
|
return fmt.Errorf("%s must be a valid duration: %w", fieldName, err)
|
|
}
|
|
if duration <= 0 {
|
|
return fmt.Errorf("%s must be positive", fieldName)
|
|
}
|
|
|
|
return nil
|
|
}
|