package config import ( "errors" "fmt" "path/filepath" "sort" "strings" "time" "gitea.maximumdirect.net/eric/narratio/internal/artifactmodel" "gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy" "gitea.maximumdirect.net/eric/narratio/internal/notariusref" "gitea.maximumdirect.net/eric/narratio/internal/pathsafe" ) // Validate checks resolved configuration for required fields and parseable durations. func Validate(cfg *Config) error { if cfg == nil { return fmt.Errorf("config is nil") } if cfg.Pipeline == nil { return fmt.Errorf("pipeline config is required") } if cfg.Campaign == nil { return fmt.Errorf("campaign config is required") } if cfg.Session == nil { return fmt.Errorf("session config is required") } if err := ValidatePipelineConfig(cfg.Pipeline); err != nil { return fmt.Errorf("pipeline config %q invalid: %w", shortName(cfg.PipelinePath, "pipeline.yml"), err) } if err := ValidateCampaignConfig(cfg.Campaign); err != nil { return fmt.Errorf("campaign config %q invalid: %w", shortName(cfg.CampaignPath, "campaign.yml"), err) } if err := validateSession(cfg.Session); err != nil { return fmt.Errorf("session config %q invalid: %w", shortName(cfg.SessionPath, "session.yml"), err) } if err := validateResolvedPartyInputs(cfg); err != nil { return fmt.Errorf("campaign/session config invalid: %w", err) } if err := validateCrossConfig(cfg.Pipeline, cfg.Session, cfg.StableInputs); err != nil { return fmt.Errorf("pipeline/session config invalid: %w", err) } return nil } // ValidatePipelineConfig validates a loaded, defaulted pipeline without // requiring session configuration. Callers that require party-driven artifact // expansion must resolve a campaign first through LoadPipelineCampaign. func ValidatePipelineConfig(cfg *PipelineConfig) error { if cfg == nil { return fmt.Errorf("pipeline config is required") } return validatePipeline(cfg) } // ValidateCampaignConfig validates a loaded campaign without requiring a // session. Party parsing and canonical-party checks remain owned by // LoadPipelineCampaign, which has the campaign source path available. func ValidateCampaignConfig(cfg *CampaignConfig) error { return validateCampaign(cfg) } func validatePipeline(cfg *PipelineConfig) error { if strings.TrimSpace(cfg.Workspace.Root) == "" { return fmt.Errorf("pipeline.workspace.root is required") } if campaignID := cfg.Campaigns.DefaultCampaignID; campaignID != "" { if err := pathsafe.ValidateOpaqueSegment(campaignID); err != nil { return fmt.Errorf("pipeline.campaigns.default_campaign_id: %w", err) } } if err := validateSecrets(cfg.Secrets); err != nil { return err } if err := validateStorage(cfg.Storage); err != nil { return err } if err := validateSpool(cfg.Spool); err != nil { return err } if err := validateCache(cfg.Cache); err != nil { return err } if err := validateNotarius(cfg.Notarius, cfg.Scriptorium); err != nil { return err } if err := validatePublish(cfg.Publish, cfg.Scriptorium, cfg.Notarius); err != nil { return err } if err := validateWhisperX(cfg.WhisperX); err != nil { return err } if err := validateSeriatim(cfg.Seriatim); err != nil { return err } if err := validateAudita(cfg.Audita); err != nil { return err } if err := validateNormalize(cfg.Normalize); err != nil { return err } if err := validateTrim(cfg.Trim); err != nil { return err } if err := validateRender(cfg.Render); err != nil { return err } if err := validateScriptorium(cfg.Scriptorium, cfg.Notarius); err != nil { return err } if err := validateNotification(cfg.Notification); err != nil { return err } return nil } func validateStorage(cfg StorageConfig) error { backend := strings.ToLower(strings.TrimSpace(cfg.Backend)) switch backend { case "", StorageBackendLocal: if cfg.S3 != nil { return fmt.Errorf("pipeline.storage.s3 is only supported when pipeline.storage.backend is s3") } return nil case StorageBackendS3: if cfg.S3 == nil { return fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3") } default: return fmt.Errorf("pipeline.storage.backend must be one of: local, s3") } if strings.TrimSpace(cfg.S3.RootPrefix) == "" { return fmt.Errorf("pipeline.storage.s3.root_prefix must be non-empty") } if err := validateRelativeSafePath("pipeline.storage.s3.root_prefix", cfg.S3.RootPrefix); err != nil { return err } if cfg.S3.Endpoint != "" && strings.TrimSpace(cfg.S3.Endpoint) == "" { return fmt.Errorf("pipeline.storage.s3.endpoint must be non-empty when provided") } if err := validateEnvVarNameField("pipeline.storage.s3.access_key_id_env", cfg.S3.AccessKeyIDEnv); err != nil { return err } if err := validateEnvVarNameField("pipeline.storage.s3.secret_access_key_env", cfg.S3.SecretKeyEnv); err != nil { return err } return nil } func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig, notarius *NotariusConfig) error { if cfg == nil { return nil } configuredOutputs := scriptoriumOutputPathMap(scriptorium) extractionOutputs := notariusOutputKeySet(notarius) seenDest := map[string]struct{}{} for i, item := range cfg.Outputs { prefix := fmt.Sprintf("pipeline.publish.outputs[%d]", i) source := strings.TrimSpace(item.Source) if source == "" { return fmt.Errorf("%s.source is required", prefix) } if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil { return fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err) } dest := strings.TrimSpace(item.Dest) if dest == "" { derivedDest, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(source, "", configuredOutputs, extractionOutputs) if err != nil { return fmt.Errorf("%s.dest is required when destination cannot be derived from %q: %w", prefix, source, err) } dest = derivedDest cfg.Outputs[i].Dest = derivedDest } normalizedDest, err := pathsafe.NormalizeRelativeDestination(dest) if err != nil { switch { case errors.Is(err, pathsafe.ErrRelativePathAbsolute): return fmt.Errorf("%s.dest must be a relative path", prefix) case errors.Is(err, pathsafe.ErrRelativePathEscape): return fmt.Errorf("%s.dest must not contain path traversal", prefix) default: return fmt.Errorf("%s.dest must be non-empty", prefix) } } if _, ok := seenDest[normalizedDest]; ok { return fmt.Errorf("%s.dest %q duplicates another publish output destination", prefix, dest) } seenDest[normalizedDest] = struct{}{} } locks, err := ValidatePublishLockRules(cfg.Locks, scriptorium, notarius, "pipeline.publish.locks") if err != nil { return err } cfg.Locks = locks return nil } // ValidatePublishLockRules validates and normalizes source-based publish locks. func ValidatePublishLockRules(locks []PublishLockRule, scriptorium *ScriptoriumConfig, notarius *NotariusConfig, label string) ([]PublishLockRule, error) { seenLocks := map[string]struct{}{} out := make([]PublishLockRule, 0, len(locks)) configuredOutputs := scriptoriumOutputPathMap(scriptorium) extractionOutputs := notariusOutputKeySet(notarius) if strings.TrimSpace(label) == "" { label = "publish.locks" } for i, item := range locks { prefix := fmt.Sprintf("%s[%d]", label, i) source := strings.TrimSpace(item.Source) if source == "" { return nil, fmt.Errorf("%s.source is required", prefix) } if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil { return nil, fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err) } if _, ok := seenLocks[source]; ok { return nil, fmt.Errorf("%s.source %q duplicates another publish lock source", prefix, source) } seenLocks[source] = struct{}{} out = append(out, PublishLockRule{ Source: source, Reason: strings.TrimSpace(item.Reason), }) } return out, nil } // MergePublishLockRules returns the union of static and remote locks. Static // locks win when both sources contain the same lock. func MergePublishLockRules(staticLocks, remoteLocks []PublishLockRule) []PublishLockRule { out := make([]PublishLockRule, 0, len(staticLocks)+len(remoteLocks)) seen := map[string]struct{}{} for _, item := range staticLocks { source := strings.TrimSpace(item.Source) if source == "" { continue } out = append(out, PublishLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)}) seen[source] = struct{}{} } for _, item := range remoteLocks { source := strings.TrimSpace(item.Source) if source == "" { continue } if _, ok := seen[source]; ok { continue } out = append(out, PublishLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)}) seen[source] = struct{}{} } return out } func scriptoriumOutputPathMap(scriptorium *ScriptoriumConfig) map[string]string { out := map[string]string{} if scriptorium == nil || len(scriptorium.Artifacts) == 0 { return out } for key, artifactCfg := range scriptorium.Artifacts { out[strings.TrimSpace(key)] = strings.TrimSpace(artifactCfg.OutputPath) } return out } func notariusOutputKeySet(notarius *NotariusConfig) map[string]struct{} { out := map[string]struct{}{} if notarius == nil { return out } for key := range notarius.Outputs { trimmed := strings.TrimSpace(key) if artifactpolicy.IsConfiguredKey(trimmed) { out[trimmed] = struct{}{} } } return out } func validateNotarius(cfg *NotariusConfig, scriptorium *ScriptoriumConfig) error { if cfg == nil || !cfg.Enabled { return nil } if strings.TrimSpace(cfg.Binary) == "" { return fmt.Errorf("pipeline.notarius.binary is required when pipeline.notarius.enabled is true") } if strings.TrimSpace(cfg.ConfigPath) == "" { return fmt.Errorf("pipeline.notarius.config_path is required when pipeline.notarius.enabled is true") } if strings.TrimSpace(cfg.PipelineID) == "" { return fmt.Errorf("pipeline.notarius.pipeline_id is required when pipeline.notarius.enabled is true") } if len(cfg.Outputs) == 0 { return fmt.Errorf("pipeline.notarius.outputs must contain at least one output when pipeline.notarius.enabled is true") } duration, err := time.ParseDuration(strings.TrimSpace(cfg.Timeout)) if err != nil { return fmt.Errorf("pipeline.notarius.timeout must be a valid duration: %w", err) } if duration <= 0 { return fmt.Errorf("pipeline.notarius.timeout must be positive") } if strings.TrimSpace(cfg.WorkingDirectory) == "" { return fmt.Errorf("pipeline.notarius.working_directory is required when pipeline.notarius.enabled is true") } if len(cfg.References) > MaxNotariusReferenceBindings { return fmt.Errorf("pipeline.notarius.references must contain at most %d bindings", MaxNotariusReferenceBindings) } referenceKeys := make([]string, 0, len(cfg.References)) for selector := range cfg.References { referenceKeys = append(referenceKeys, selector) } sort.Strings(referenceKeys) var normalizedReferences map[string]string if cfg.References != nil { normalizedReferences = make(map[string]string, len(cfg.References)) } referenceOwners := make(map[string]string, len(cfg.References)) for _, rawSelector := range referenceKeys { selector, err := notariusref.NormalizeSelector(rawSelector) if err != nil { return fmt.Errorf("pipeline.notarius.references selector %q is invalid: %w", rawSelector, err) } if previous, ok := referenceOwners[selector]; ok { return fmt.Errorf("pipeline.notarius.references selectors %q and %q normalize to %q", previous, rawSelector, selector) } referenceOwners[selector] = rawSelector source := strings.TrimSpace(cfg.References[rawSelector]) if source == "" { return fmt.Errorf("pipeline.notarius.references.%s source is required", selector) } if _, ok := artifactpolicy.DescribePreparedInputSource(source); !ok { return fmt.Errorf("pipeline.notarius.references.%s source %q is not a supported prepared input source", selector, source) } normalizedReferences[selector] = source } reservedSources := map[string]string{} for _, spec := range artifactmodel.RuntimeTranscriptArtifacts() { reservedSources[spec.SourceID] = "built-in source" } reservedSources[artifactpolicy.SourceBoundsSession] = "built-in source" if scriptorium != nil { for key := range scriptorium.Artifacts { normalizedKey := strings.TrimSpace(key) reservedSources[artifactpolicy.ConfiguredSourceID(normalizedKey)] = "configured Scriptorium source" reservedSources[artifactpolicy.PreviousSessionSourceID(normalizedKey)] = "previous-session source" } } rawKeys := make([]string, 0, len(cfg.Outputs)) for key := range cfg.Outputs { rawKeys = append(rawKeys, key) } sort.Strings(rawKeys) normalizedOutputs := make(map[string]NotariusOutputConfig, len(cfg.Outputs)) sourceOwners := map[string]string{} laneOwners := map[string]string{} for _, rawKey := range rawKeys { output := cfg.Outputs[rawKey] key := strings.TrimSpace(rawKey) if !artifactpolicy.IsConfiguredKey(key) { return fmt.Errorf("pipeline.notarius.outputs keys must match ^[a-z][a-z0-9_]*$") } sourceID := artifactpolicy.ExtractionSourceID(key) if previousKey, ok := sourceOwners[sourceID]; ok { return fmt.Errorf("pipeline.notarius.outputs keys %q and %q normalize to duplicate source %q", previousKey, rawKey, sourceID) } if owner, ok := reservedSources[sourceID]; ok { return fmt.Errorf("pipeline.notarius.outputs.%s source %q collides with %s", key, sourceID, owner) } sourceOwners[sourceID] = rawKey output.LaneID = strings.TrimSpace(output.LaneID) output.MediaType = strings.TrimSpace(output.MediaType) output.SchemaID = strings.TrimSpace(output.SchemaID) output.SchemaVersion = strings.TrimSpace(output.SchemaVersion) output.ModuleKey = strings.TrimSpace(output.ModuleKey) prefix := "pipeline.notarius.outputs." + key if output.LaneID == "" { return fmt.Errorf("%s.lane_id is required", prefix) } if previousKey, ok := laneOwners[output.LaneID]; ok { return fmt.Errorf("%s.lane_id %q duplicates pipeline.notarius.outputs.%s.lane_id", prefix, output.LaneID, previousKey) } laneOwners[output.LaneID] = key if output.MediaType == "" { return fmt.Errorf("%s.media_type is required", prefix) } if output.SchemaID == "" { return fmt.Errorf("%s.schema_id is required", prefix) } if output.SchemaVersion == "" { return fmt.Errorf("%s.schema_version is required", prefix) } normalizedOutputs[key] = output } cfg.Binary = strings.TrimSpace(cfg.Binary) cfg.ConfigPath = filepath.Clean(cfg.ConfigPath) cfg.PipelineID = strings.TrimSpace(cfg.PipelineID) cfg.Timeout = strings.TrimSpace(cfg.Timeout) cfg.WorkingDirectory = filepath.Clean(cfg.WorkingDirectory) cfg.References = normalizedReferences cfg.Outputs = normalizedOutputs return nil } func validateScriptorium(cfg *ScriptoriumConfig, notarius *NotariusConfig) error { if cfg == nil { return nil } if strings.TrimSpace(cfg.Binary) == "" { return fmt.Errorf("pipeline.scriptorium.binary is required when pipeline.scriptorium is configured") } if cfg.ConfigPath != "" && strings.TrimSpace(cfg.ConfigPath) == "" { return fmt.Errorf("pipeline.scriptorium.config_path must be non-empty when provided") } if err := validateDuration("pipeline.scriptorium.timeout", cfg.Timeout); err != nil { return err } configuredArtifacts := make(map[string]struct{}, len(cfg.Artifacts)) referencedArtifacts := make(map[string]struct{}) outputOwners := make(map[string]string, len(cfg.Artifacts)) for artifactName := range cfg.Artifacts { if !artifactpolicy.IsConfiguredKey(artifactName) { return fmt.Errorf("pipeline.scriptorium.artifacts keys must match ^[a-z][a-z0-9_]*$") } configuredArtifacts[artifactName] = struct{}{} } for artifactName, artifactCfg := range cfg.Artifacts { if artifactCfg.Enabled && strings.TrimSpace(artifactCfg.PromptID) == "" { return fmt.Errorf("pipeline.scriptorium.artifacts.%s.prompt_id is required when enabled", artifactName) } if artifactCfg.Enabled && strings.TrimSpace(artifactCfg.OutputPath) == "" { return fmt.Errorf("pipeline.scriptorium.artifacts.%s.output_path is required when enabled", artifactName) } if strings.TrimSpace(artifactCfg.OutputPath) != "" { pathField := "pipeline.scriptorium.artifacts." + artifactName + ".output_path" if err := validateRelativeSafePath(pathField, artifactCfg.OutputPath); err != nil { return err } if err := validatePathWithinRoot(pathField, artifactCfg.OutputPath, DefaultScriptoriumArtifactOutputRoot); err != nil { return err } if previous, exists := outputOwners[artifactCfg.OutputPath]; exists { return fmt.Errorf("%s.output_path %q duplicates artifact %q", pathField, artifactCfg.OutputPath, previous) } outputOwners[artifactCfg.OutputPath] = artifactName } if err := validateDuration("pipeline.scriptorium.artifacts."+artifactName+".timeout", artifactCfg.Timeout); err != nil { return err } depSet := make(map[string]struct{}, len(artifactCfg.DependsOn)) for i, depName := range artifactCfg.DependsOn { trimmedDep := strings.TrimSpace(depName) field := fmt.Sprintf("pipeline.scriptorium.artifacts.%s.depends_on[%d]", artifactName, i) if trimmedDep == "" { return fmt.Errorf("%s must be non-empty", field) } if _, ok := configuredArtifacts[trimmedDep]; !ok { return fmt.Errorf("%s %q is not a configured artifact key", field, depName) } if trimmedDep == artifactName { return fmt.Errorf("pipeline.scriptorium.artifacts.%s.depends_on must not include itself", artifactName) } depSet[trimmedDep] = struct{}{} referencedArtifacts[trimmedDep] = struct{}{} } for inputName, inputCfg := range artifactCfg.Inputs { trimmedInputName := strings.TrimSpace(inputName) if trimmedInputName == "" { return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs keys must be non-empty", artifactName) } source := strings.TrimSpace(inputCfg.Source) if source == "" { return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs.%s.source is required", artifactName, inputName) } referencedArtifact, err := validateScriptoriumInputSource(artifactName, inputName, source, configuredArtifacts, notariusOutputKeySet(notarius)) if err != nil { return err } if referencedArtifact != "" { if _, ok := depSet[referencedArtifact]; !ok { return fmt.Errorf( "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q requires depends_on entry %q", artifactName, inputName, source, referencedArtifact, ) } referencedArtifacts[referencedArtifact] = struct{}{} } } for varName, varValue := range artifactCfg.Vars { if strings.TrimSpace(varName) == "" { return fmt.Errorf("pipeline.scriptorium.artifacts.%s.vars keys must be non-empty", artifactName) } switch varValue.(type) { case bool, string: default: return fmt.Errorf("pipeline.scriptorium.artifacts.%s.vars.%s must be a string or boolean", artifactName, varName) } } } for artifactName := range referencedArtifacts { artifactCfg := cfg.Artifacts[artifactName] if strings.TrimSpace(artifactCfg.OutputPath) == "" { return fmt.Errorf("pipeline.scriptorium.artifacts.%s.output_path is required when artifact is referenced", artifactName) } } if err := ValidateScriptoriumArtifactDependencies(cfg.Artifacts); err != nil { return err } return nil } func publishUploadConfiguredForS3(pipeline *PipelineConfig) bool { if pipeline == nil || pipeline.Publish == nil { return false } if !strings.EqualFold(strings.TrimSpace(pipeline.Storage.Backend), StorageBackendS3) { return false } enabled := true if pipeline.Publish.Enabled != nil { enabled = *pipeline.Publish.Enabled } upload := true if pipeline.Publish.UploadRun != nil { upload = *pipeline.Publish.UploadRun } return enabled && upload } func validateScriptoriumInputSource( artifactName, inputName, source string, configuredArtifacts map[string]struct{}, extractionOutputs map[string]struct{}, ) (string, error) { trimmedSource := strings.TrimSpace(source) descriptor, err := artifactpolicy.DescribeScriptoriumInputSource(trimmedSource) if err != nil { if errors.Is(err, artifactpolicy.ErrInvalidPreviousSessionSource) { return "", fmt.Errorf( "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q must reference configured artifact key matching ^[a-z][a-z0-9_]*$", artifactName, inputName, source, ) } return "", fmt.Errorf( "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported", artifactName, inputName, source, ) } if err := artifactpolicy.ValidateInputReference(descriptor, configuredArtifacts, extractionOutputs); err != nil { var unknownConfigured *artifactpolicy.UnknownConfiguredArtifactError if errors.As(err, &unknownConfigured) { return "", fmt.Errorf( "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q references unknown artifact %q", artifactName, inputName, source, unknownConfigured.ConfiguredKey, ) } var unknownExtraction *artifactpolicy.UnknownExtractionArtifactError if errors.As(err, &unknownExtraction) { return "", fmt.Errorf( "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q references unknown extraction output %q", artifactName, inputName, source, unknownExtraction.ConfiguredKey, ) } return "", fmt.Errorf( "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported", artifactName, inputName, source, ) } if descriptor.Source.Kind == artifactpolicy.SourceKindConfiguredArtifact { return descriptor.Source.ConfiguredKey, nil } return "", nil } // ValidateScriptoriumArtifactDependencies validates the configured dependency // graph independently of execution selection. Disabled artifacts remain valid // prerequisites for explicit selections and therefore participate in cycles. func ValidateScriptoriumArtifactDependencies(artifacts map[string]ScriptoriumArtifactConfig) error { if len(artifacts) == 0 { return nil } graph := make(map[string][]string, len(artifacts)) for name, cfg := range artifacts { if !artifactpolicy.IsConfiguredKey(name) { return fmt.Errorf("pipeline.scriptorium.artifacts keys must match ^[a-z][a-z0-9_]*$") } seen := make(map[string]struct{}, len(cfg.DependsOn)) for _, dep := range cfg.DependsOn { trimmedDep := strings.TrimSpace(dep) if trimmedDep == "" { return fmt.Errorf("pipeline.scriptorium.artifacts.%s.depends_on entries must be non-empty", name) } if _, ok := artifacts[trimmedDep]; !ok { return fmt.Errorf("pipeline.scriptorium.artifacts.%s dependency %q is not configured", name, dep) } if trimmedDep == name { return fmt.Errorf("pipeline.scriptorium.artifacts.%s.depends_on must not include itself", name) } if _, duplicate := seen[trimmedDep]; duplicate { continue } seen[trimmedDep] = struct{}{} graph[name] = append(graph[name], trimmedDep) } sort.Strings(graph[name]) } visiting := make(map[string]bool, len(artifacts)) visited := make(map[string]bool, len(artifacts)) var visit func(node string) error visit = func(node string) error { if visiting[node] { return fmt.Errorf("pipeline.scriptorium.artifacts dependencies must not contain cycles") } if visited[node] { return nil } visiting[node] = true for _, dep := range graph[node] { if err := visit(dep); err != nil { return err } } visiting[node] = false visited[node] = true return nil } nodes := make([]string, 0, len(artifacts)) for node := range artifacts { nodes = append(nodes, node) } sort.Strings(nodes) for _, node := range nodes { if err := visit(node); err != nil { return err } } return nil }