From 3136c26d4e2c4b2e69e15125eb39a075331e0423 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 5 Sep 2026 00:24:58 +0000 Subject: [PATCH] Separate configuration domain validators --- docs/roadmap/implementation.md | 2 +- internal/config/validate.go | 563 ------------------------ internal/config/validate_notarius.go | 157 +++++++ internal/config/validate_publish.go | 178 ++++++++ internal/config/validate_scriptorium.go | 254 +++++++++++ 5 files changed, 590 insertions(+), 564 deletions(-) create mode 100644 internal/config/validate_notarius.go create mode 100644 internal/config/validate_publish.go create mode 100644 internal/config/validate_scriptorium.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index d013719..752eb29 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -178,7 +178,7 @@ primitive checks clear package-local owners. This stage is behavior-preserving. ## Stage 2 — Separate Publish, Storage, Notarius, And Scriptorium Owners -**Status:** Pending +**Status:** Completed ### Goal diff --git a/internal/config/validate.go b/internal/config/validate.go index 52073ae..9209213 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -1,16 +1,9 @@ package config import ( - "errors" "fmt" - "path/filepath" - "sort" "strings" - "time" - "gitea.maximumdirect.net/eric/narratio/internal/artifactmodel" - "gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy" - "gitea.maximumdirect.net/eric/narratio/internal/notariusref" "gitea.maximumdirect.net/eric/narratio/internal/pathsafe" ) @@ -119,559 +112,3 @@ func validatePipeline(cfg *PipelineConfig) error { return nil } - -func validateStorage(cfg StorageConfig) error { - backend := strings.ToLower(strings.TrimSpace(cfg.Backend)) - switch backend { - case "", StorageBackendLocal: - if cfg.S3 != nil { - return fmt.Errorf("pipeline.storage.s3 is only supported when pipeline.storage.backend is s3") - } - return nil - case StorageBackendS3: - if cfg.S3 == nil { - return fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3") - } - default: - return fmt.Errorf("pipeline.storage.backend must be one of: local, s3") - } - if strings.TrimSpace(cfg.S3.RootPrefix) == "" { - return fmt.Errorf("pipeline.storage.s3.root_prefix must be non-empty") - } - if err := validateRelativeSafePath("pipeline.storage.s3.root_prefix", cfg.S3.RootPrefix); err != nil { - return err - } - if cfg.S3.Endpoint != "" && strings.TrimSpace(cfg.S3.Endpoint) == "" { - return fmt.Errorf("pipeline.storage.s3.endpoint must be non-empty when provided") - } - if err := validateEnvVarNameField("pipeline.storage.s3.access_key_id_env", cfg.S3.AccessKeyIDEnv); err != nil { - return err - } - if err := validateEnvVarNameField("pipeline.storage.s3.secret_access_key_env", cfg.S3.SecretKeyEnv); err != nil { - return err - } - return nil -} - -func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig, notarius *NotariusConfig) error { - if cfg == nil { - return nil - } - configuredOutputs := scriptoriumOutputPathMap(scriptorium) - extractionOutputs := notariusOutputKeySet(notarius) - seenDest := map[string]struct{}{} - for i, item := range cfg.Outputs { - prefix := fmt.Sprintf("pipeline.publish.outputs[%d]", i) - source := strings.TrimSpace(item.Source) - if source == "" { - return fmt.Errorf("%s.source is required", prefix) - } - if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil { - return fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err) - } - dest := strings.TrimSpace(item.Dest) - if dest == "" { - derivedDest, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(source, "", configuredOutputs, extractionOutputs) - if err != nil { - return fmt.Errorf("%s.dest is required when destination cannot be derived from %q: %w", prefix, source, err) - } - dest = derivedDest - cfg.Outputs[i].Dest = derivedDest - } - normalizedDest, err := pathsafe.NormalizeRelativeDestination(dest) - if err != nil { - switch { - case errors.Is(err, pathsafe.ErrRelativePathAbsolute): - return fmt.Errorf("%s.dest must be a relative path", prefix) - case errors.Is(err, pathsafe.ErrRelativePathEscape): - return fmt.Errorf("%s.dest must not contain path traversal", prefix) - default: - return fmt.Errorf("%s.dest must be non-empty", prefix) - } - } - if _, ok := seenDest[normalizedDest]; ok { - return fmt.Errorf("%s.dest %q duplicates another publish output destination", prefix, dest) - } - seenDest[normalizedDest] = struct{}{} - } - locks, err := ValidatePublishLockRules(cfg.Locks, scriptorium, notarius, "pipeline.publish.locks") - if err != nil { - return err - } - cfg.Locks = locks - return nil -} - -// ValidatePublishLockRules validates and normalizes source-based publish locks. -func ValidatePublishLockRules(locks []PublishLockRule, scriptorium *ScriptoriumConfig, notarius *NotariusConfig, label string) ([]PublishLockRule, error) { - seenLocks := map[string]struct{}{} - out := make([]PublishLockRule, 0, len(locks)) - configuredOutputs := scriptoriumOutputPathMap(scriptorium) - extractionOutputs := notariusOutputKeySet(notarius) - if strings.TrimSpace(label) == "" { - label = "publish.locks" - } - for i, item := range locks { - prefix := fmt.Sprintf("%s[%d]", label, i) - source := strings.TrimSpace(item.Source) - if source == "" { - return nil, fmt.Errorf("%s.source is required", prefix) - } - if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil { - return nil, fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err) - } - if _, ok := seenLocks[source]; ok { - return nil, fmt.Errorf("%s.source %q duplicates another publish lock source", prefix, source) - } - seenLocks[source] = struct{}{} - out = append(out, PublishLockRule{ - Source: source, - Reason: strings.TrimSpace(item.Reason), - }) - } - return out, nil -} - -// MergePublishLockRules returns the union of static and remote locks. Static -// locks win when both sources contain the same lock. -func MergePublishLockRules(staticLocks, remoteLocks []PublishLockRule) []PublishLockRule { - out := make([]PublishLockRule, 0, len(staticLocks)+len(remoteLocks)) - seen := map[string]struct{}{} - for _, item := range staticLocks { - source := strings.TrimSpace(item.Source) - if source == "" { - continue - } - out = append(out, PublishLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)}) - seen[source] = struct{}{} - } - for _, item := range remoteLocks { - source := strings.TrimSpace(item.Source) - if source == "" { - continue - } - if _, ok := seen[source]; ok { - continue - } - out = append(out, PublishLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)}) - seen[source] = struct{}{} - } - return out -} - -func scriptoriumOutputPathMap(scriptorium *ScriptoriumConfig) map[string]string { - out := map[string]string{} - if scriptorium == nil || len(scriptorium.Artifacts) == 0 { - return out - } - for key, artifactCfg := range scriptorium.Artifacts { - out[strings.TrimSpace(key)] = strings.TrimSpace(artifactCfg.OutputPath) - } - return out -} - -func notariusOutputKeySet(notarius *NotariusConfig) map[string]struct{} { - out := map[string]struct{}{} - if notarius == nil { - return out - } - for key := range notarius.Outputs { - trimmed := strings.TrimSpace(key) - if artifactpolicy.IsConfiguredKey(trimmed) { - out[trimmed] = struct{}{} - } - } - return out -} - -func validateNotarius(cfg *NotariusConfig, scriptorium *ScriptoriumConfig) error { - if cfg == nil || !cfg.Enabled { - return nil - } - if strings.TrimSpace(cfg.Binary) == "" { - return fmt.Errorf("pipeline.notarius.binary is required when pipeline.notarius.enabled is true") - } - if strings.TrimSpace(cfg.ConfigPath) == "" { - return fmt.Errorf("pipeline.notarius.config_path is required when pipeline.notarius.enabled is true") - } - if strings.TrimSpace(cfg.PipelineID) == "" { - return fmt.Errorf("pipeline.notarius.pipeline_id is required when pipeline.notarius.enabled is true") - } - if len(cfg.Outputs) == 0 { - return fmt.Errorf("pipeline.notarius.outputs must contain at least one output when pipeline.notarius.enabled is true") - } - duration, err := time.ParseDuration(strings.TrimSpace(cfg.Timeout)) - if err != nil { - return fmt.Errorf("pipeline.notarius.timeout must be a valid duration: %w", err) - } - if duration <= 0 { - return fmt.Errorf("pipeline.notarius.timeout must be positive") - } - if strings.TrimSpace(cfg.WorkingDirectory) == "" { - return fmt.Errorf("pipeline.notarius.working_directory is required when pipeline.notarius.enabled is true") - } - if len(cfg.References) > MaxNotariusReferenceBindings { - return fmt.Errorf("pipeline.notarius.references must contain at most %d bindings", MaxNotariusReferenceBindings) - } - - referenceKeys := make([]string, 0, len(cfg.References)) - for selector := range cfg.References { - referenceKeys = append(referenceKeys, selector) - } - sort.Strings(referenceKeys) - var normalizedReferences map[string]string - if cfg.References != nil { - normalizedReferences = make(map[string]string, len(cfg.References)) - } - referenceOwners := make(map[string]string, len(cfg.References)) - for _, rawSelector := range referenceKeys { - selector, err := notariusref.NormalizeSelector(rawSelector) - if err != nil { - return fmt.Errorf("pipeline.notarius.references selector %q is invalid: %w", rawSelector, err) - } - if previous, ok := referenceOwners[selector]; ok { - return fmt.Errorf("pipeline.notarius.references selectors %q and %q normalize to %q", previous, rawSelector, selector) - } - referenceOwners[selector] = rawSelector - - source := strings.TrimSpace(cfg.References[rawSelector]) - if source == "" { - return fmt.Errorf("pipeline.notarius.references.%s source is required", selector) - } - if _, ok := artifactpolicy.DescribePreparedInputSource(source); !ok { - return fmt.Errorf("pipeline.notarius.references.%s source %q is not a supported prepared input source", selector, source) - } - normalizedReferences[selector] = source - } - - reservedSources := map[string]string{} - for _, spec := range artifactmodel.RuntimeTranscriptArtifacts() { - reservedSources[spec.SourceID] = "built-in source" - } - reservedSources[artifactpolicy.SourceBoundsSession] = "built-in source" - if scriptorium != nil { - for key := range scriptorium.Artifacts { - normalizedKey := strings.TrimSpace(key) - reservedSources[artifactpolicy.ConfiguredSourceID(normalizedKey)] = "configured Scriptorium source" - reservedSources[artifactpolicy.PreviousSessionSourceID(normalizedKey)] = "previous-session source" - } - } - - rawKeys := make([]string, 0, len(cfg.Outputs)) - for key := range cfg.Outputs { - rawKeys = append(rawKeys, key) - } - sort.Strings(rawKeys) - normalizedOutputs := make(map[string]NotariusOutputConfig, len(cfg.Outputs)) - sourceOwners := map[string]string{} - laneOwners := map[string]string{} - for _, rawKey := range rawKeys { - output := cfg.Outputs[rawKey] - key := strings.TrimSpace(rawKey) - if !artifactpolicy.IsConfiguredKey(key) { - return fmt.Errorf("pipeline.notarius.outputs keys must match ^[a-z][a-z0-9_]*$") - } - sourceID := artifactpolicy.ExtractionSourceID(key) - if previousKey, ok := sourceOwners[sourceID]; ok { - return fmt.Errorf("pipeline.notarius.outputs keys %q and %q normalize to duplicate source %q", previousKey, rawKey, sourceID) - } - if owner, ok := reservedSources[sourceID]; ok { - return fmt.Errorf("pipeline.notarius.outputs.%s source %q collides with %s", key, sourceID, owner) - } - sourceOwners[sourceID] = rawKey - - output.LaneID = strings.TrimSpace(output.LaneID) - output.MediaType = strings.TrimSpace(output.MediaType) - output.SchemaID = strings.TrimSpace(output.SchemaID) - output.SchemaVersion = strings.TrimSpace(output.SchemaVersion) - output.ModuleKey = strings.TrimSpace(output.ModuleKey) - prefix := "pipeline.notarius.outputs." + key - if output.LaneID == "" { - return fmt.Errorf("%s.lane_id is required", prefix) - } - if previousKey, ok := laneOwners[output.LaneID]; ok { - return fmt.Errorf("%s.lane_id %q duplicates pipeline.notarius.outputs.%s.lane_id", prefix, output.LaneID, previousKey) - } - laneOwners[output.LaneID] = key - if output.MediaType == "" { - return fmt.Errorf("%s.media_type is required", prefix) - } - if output.SchemaID == "" { - return fmt.Errorf("%s.schema_id is required", prefix) - } - if output.SchemaVersion == "" { - return fmt.Errorf("%s.schema_version is required", prefix) - } - normalizedOutputs[key] = output - } - cfg.Binary = strings.TrimSpace(cfg.Binary) - cfg.ConfigPath = filepath.Clean(cfg.ConfigPath) - cfg.PipelineID = strings.TrimSpace(cfg.PipelineID) - cfg.Timeout = strings.TrimSpace(cfg.Timeout) - cfg.WorkingDirectory = filepath.Clean(cfg.WorkingDirectory) - cfg.References = normalizedReferences - cfg.Outputs = normalizedOutputs - return nil -} - -func validateScriptorium(cfg *ScriptoriumConfig, notarius *NotariusConfig) error { - if cfg == nil { - return nil - } - if strings.TrimSpace(cfg.Binary) == "" { - return fmt.Errorf("pipeline.scriptorium.binary is required when pipeline.scriptorium is configured") - } - if cfg.ConfigPath != "" && strings.TrimSpace(cfg.ConfigPath) == "" { - return fmt.Errorf("pipeline.scriptorium.config_path must be non-empty when provided") - } - if err := validateDuration("pipeline.scriptorium.timeout", cfg.Timeout); err != nil { - return err - } - - configuredArtifacts := make(map[string]struct{}, len(cfg.Artifacts)) - referencedArtifacts := make(map[string]struct{}) - outputOwners := make(map[string]string, len(cfg.Artifacts)) - for artifactName := range cfg.Artifacts { - if !artifactpolicy.IsConfiguredKey(artifactName) { - return fmt.Errorf("pipeline.scriptorium.artifacts keys must match ^[a-z][a-z0-9_]*$") - } - configuredArtifacts[artifactName] = struct{}{} - } - - for artifactName, artifactCfg := range cfg.Artifacts { - if artifactCfg.Enabled && strings.TrimSpace(artifactCfg.PromptID) == "" { - return fmt.Errorf("pipeline.scriptorium.artifacts.%s.prompt_id is required when enabled", artifactName) - } - if artifactCfg.Enabled && strings.TrimSpace(artifactCfg.OutputPath) == "" { - return fmt.Errorf("pipeline.scriptorium.artifacts.%s.output_path is required when enabled", artifactName) - } - if strings.TrimSpace(artifactCfg.OutputPath) != "" { - pathField := "pipeline.scriptorium.artifacts." + artifactName + ".output_path" - if err := validateRelativeSafePath(pathField, artifactCfg.OutputPath); err != nil { - return err - } - if err := validatePathWithinRoot(pathField, artifactCfg.OutputPath, DefaultScriptoriumArtifactOutputRoot); err != nil { - return err - } - if previous, exists := outputOwners[artifactCfg.OutputPath]; exists { - return fmt.Errorf("%s.output_path %q duplicates artifact %q", pathField, artifactCfg.OutputPath, previous) - } - outputOwners[artifactCfg.OutputPath] = artifactName - } - if err := validateDuration("pipeline.scriptorium.artifacts."+artifactName+".timeout", artifactCfg.Timeout); err != nil { - return err - } - - depSet := make(map[string]struct{}, len(artifactCfg.DependsOn)) - for i, depName := range artifactCfg.DependsOn { - trimmedDep := strings.TrimSpace(depName) - field := fmt.Sprintf("pipeline.scriptorium.artifacts.%s.depends_on[%d]", artifactName, i) - if trimmedDep == "" { - return fmt.Errorf("%s must be non-empty", field) - } - if _, ok := configuredArtifacts[trimmedDep]; !ok { - return fmt.Errorf("%s %q is not a configured artifact key", field, depName) - } - if trimmedDep == artifactName { - return fmt.Errorf("pipeline.scriptorium.artifacts.%s.depends_on must not include itself", artifactName) - } - depSet[trimmedDep] = struct{}{} - referencedArtifacts[trimmedDep] = struct{}{} - } - - for inputName, inputCfg := range artifactCfg.Inputs { - trimmedInputName := strings.TrimSpace(inputName) - if trimmedInputName == "" { - return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs keys must be non-empty", artifactName) - } - source := strings.TrimSpace(inputCfg.Source) - if source == "" { - return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs.%s.source is required", artifactName, inputName) - } - - referencedArtifact, err := validateScriptoriumInputSource(artifactName, inputName, source, configuredArtifacts, notariusOutputKeySet(notarius)) - if err != nil { - return err - } - if referencedArtifact != "" { - if _, ok := depSet[referencedArtifact]; !ok { - return fmt.Errorf( - "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q requires depends_on entry %q", - artifactName, - inputName, - source, - referencedArtifact, - ) - } - referencedArtifacts[referencedArtifact] = struct{}{} - } - } - for varName, varValue := range artifactCfg.Vars { - if strings.TrimSpace(varName) == "" { - return fmt.Errorf("pipeline.scriptorium.artifacts.%s.vars keys must be non-empty", artifactName) - } - switch varValue.(type) { - case bool, string: - default: - return fmt.Errorf("pipeline.scriptorium.artifacts.%s.vars.%s must be a string or boolean", artifactName, varName) - } - } - } - - for artifactName := range referencedArtifacts { - artifactCfg := cfg.Artifacts[artifactName] - if strings.TrimSpace(artifactCfg.OutputPath) == "" { - return fmt.Errorf("pipeline.scriptorium.artifacts.%s.output_path is required when artifact is referenced", artifactName) - } - } - - if err := ValidateScriptoriumArtifactDependencies(cfg.Artifacts); err != nil { - return err - } - - return nil -} - -func publishUploadConfiguredForS3(pipeline *PipelineConfig) bool { - if pipeline == nil || pipeline.Publish == nil { - return false - } - if !strings.EqualFold(strings.TrimSpace(pipeline.Storage.Backend), StorageBackendS3) { - return false - } - enabled := true - if pipeline.Publish.Enabled != nil { - enabled = *pipeline.Publish.Enabled - } - upload := true - if pipeline.Publish.UploadRun != nil { - upload = *pipeline.Publish.UploadRun - } - return enabled && upload -} - -func validateScriptoriumInputSource( - artifactName, inputName, source string, - configuredArtifacts map[string]struct{}, - extractionOutputs map[string]struct{}, -) (string, error) { - trimmedSource := strings.TrimSpace(source) - descriptor, err := artifactpolicy.DescribeScriptoriumInputSource(trimmedSource) - if err != nil { - if errors.Is(err, artifactpolicy.ErrInvalidPreviousSessionSource) { - return "", fmt.Errorf( - "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q must reference configured artifact key matching ^[a-z][a-z0-9_]*$", - artifactName, - inputName, - source, - ) - } - return "", fmt.Errorf( - "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported", - artifactName, - inputName, - source, - ) - } - if err := artifactpolicy.ValidateInputReference(descriptor, configuredArtifacts, extractionOutputs); err != nil { - var unknownConfigured *artifactpolicy.UnknownConfiguredArtifactError - if errors.As(err, &unknownConfigured) { - return "", fmt.Errorf( - "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q references unknown artifact %q", - artifactName, - inputName, - source, - unknownConfigured.ConfiguredKey, - ) - } - var unknownExtraction *artifactpolicy.UnknownExtractionArtifactError - if errors.As(err, &unknownExtraction) { - return "", fmt.Errorf( - "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q references unknown extraction output %q", - artifactName, - inputName, - source, - unknownExtraction.ConfiguredKey, - ) - } - return "", fmt.Errorf( - "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported", - artifactName, - inputName, - source, - ) - } - if descriptor.Source.Kind == artifactpolicy.SourceKindConfiguredArtifact { - return descriptor.Source.ConfiguredKey, nil - } - return "", nil -} - -// ValidateScriptoriumArtifactDependencies validates the configured dependency -// graph independently of execution selection. Disabled artifacts remain valid -// prerequisites for explicit selections and therefore participate in cycles. -func ValidateScriptoriumArtifactDependencies(artifacts map[string]ScriptoriumArtifactConfig) error { - if len(artifacts) == 0 { - return nil - } - - graph := make(map[string][]string, len(artifacts)) - for name, cfg := range artifacts { - if !artifactpolicy.IsConfiguredKey(name) { - return fmt.Errorf("pipeline.scriptorium.artifacts keys must match ^[a-z][a-z0-9_]*$") - } - seen := make(map[string]struct{}, len(cfg.DependsOn)) - for _, dep := range cfg.DependsOn { - trimmedDep := strings.TrimSpace(dep) - if trimmedDep == "" { - return fmt.Errorf("pipeline.scriptorium.artifacts.%s.depends_on entries must be non-empty", name) - } - if _, ok := artifacts[trimmedDep]; !ok { - return fmt.Errorf("pipeline.scriptorium.artifacts.%s dependency %q is not configured", name, dep) - } - if trimmedDep == name { - return fmt.Errorf("pipeline.scriptorium.artifacts.%s.depends_on must not include itself", name) - } - if _, duplicate := seen[trimmedDep]; duplicate { - continue - } - seen[trimmedDep] = struct{}{} - graph[name] = append(graph[name], trimmedDep) - } - sort.Strings(graph[name]) - } - - visiting := make(map[string]bool, len(artifacts)) - visited := make(map[string]bool, len(artifacts)) - - var visit func(node string) error - visit = func(node string) error { - if visiting[node] { - return fmt.Errorf("pipeline.scriptorium.artifacts dependencies must not contain cycles") - } - if visited[node] { - return nil - } - visiting[node] = true - for _, dep := range graph[node] { - if err := visit(dep); err != nil { - return err - } - } - visiting[node] = false - visited[node] = true - return nil - } - - nodes := make([]string, 0, len(artifacts)) - for node := range artifacts { - nodes = append(nodes, node) - } - sort.Strings(nodes) - for _, node := range nodes { - if err := visit(node); err != nil { - return err - } - } - return nil -} diff --git a/internal/config/validate_notarius.go b/internal/config/validate_notarius.go new file mode 100644 index 0000000..20c528c --- /dev/null +++ b/internal/config/validate_notarius.go @@ -0,0 +1,157 @@ +package config + +import ( + "fmt" + "path/filepath" + "sort" + "strings" + "time" + + "gitea.maximumdirect.net/eric/narratio/internal/artifactmodel" + "gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy" + "gitea.maximumdirect.net/eric/narratio/internal/notariusref" +) + +func validateNotarius(cfg *NotariusConfig, scriptorium *ScriptoriumConfig) error { + if cfg == nil || !cfg.Enabled { + return nil + } + if strings.TrimSpace(cfg.Binary) == "" { + return fmt.Errorf("pipeline.notarius.binary is required when pipeline.notarius.enabled is true") + } + if strings.TrimSpace(cfg.ConfigPath) == "" { + return fmt.Errorf("pipeline.notarius.config_path is required when pipeline.notarius.enabled is true") + } + if strings.TrimSpace(cfg.PipelineID) == "" { + return fmt.Errorf("pipeline.notarius.pipeline_id is required when pipeline.notarius.enabled is true") + } + if len(cfg.Outputs) == 0 { + return fmt.Errorf("pipeline.notarius.outputs must contain at least one output when pipeline.notarius.enabled is true") + } + duration, err := time.ParseDuration(strings.TrimSpace(cfg.Timeout)) + if err != nil { + return fmt.Errorf("pipeline.notarius.timeout must be a valid duration: %w", err) + } + if duration <= 0 { + return fmt.Errorf("pipeline.notarius.timeout must be positive") + } + if strings.TrimSpace(cfg.WorkingDirectory) == "" { + return fmt.Errorf("pipeline.notarius.working_directory is required when pipeline.notarius.enabled is true") + } + if len(cfg.References) > MaxNotariusReferenceBindings { + return fmt.Errorf("pipeline.notarius.references must contain at most %d bindings", MaxNotariusReferenceBindings) + } + + referenceKeys := make([]string, 0, len(cfg.References)) + for selector := range cfg.References { + referenceKeys = append(referenceKeys, selector) + } + sort.Strings(referenceKeys) + var normalizedReferences map[string]string + if cfg.References != nil { + normalizedReferences = make(map[string]string, len(cfg.References)) + } + referenceOwners := make(map[string]string, len(cfg.References)) + for _, rawSelector := range referenceKeys { + selector, err := notariusref.NormalizeSelector(rawSelector) + if err != nil { + return fmt.Errorf("pipeline.notarius.references selector %q is invalid: %w", rawSelector, err) + } + if previous, ok := referenceOwners[selector]; ok { + return fmt.Errorf("pipeline.notarius.references selectors %q and %q normalize to %q", previous, rawSelector, selector) + } + referenceOwners[selector] = rawSelector + + source := strings.TrimSpace(cfg.References[rawSelector]) + if source == "" { + return fmt.Errorf("pipeline.notarius.references.%s source is required", selector) + } + if _, ok := artifactpolicy.DescribePreparedInputSource(source); !ok { + return fmt.Errorf("pipeline.notarius.references.%s source %q is not a supported prepared input source", selector, source) + } + normalizedReferences[selector] = source + } + + reservedSources := map[string]string{} + for _, spec := range artifactmodel.RuntimeTranscriptArtifacts() { + reservedSources[spec.SourceID] = "built-in source" + } + reservedSources[artifactpolicy.SourceBoundsSession] = "built-in source" + if scriptorium != nil { + for key := range scriptorium.Artifacts { + normalizedKey := strings.TrimSpace(key) + reservedSources[artifactpolicy.ConfiguredSourceID(normalizedKey)] = "configured Scriptorium source" + reservedSources[artifactpolicy.PreviousSessionSourceID(normalizedKey)] = "previous-session source" + } + } + + rawKeys := make([]string, 0, len(cfg.Outputs)) + for key := range cfg.Outputs { + rawKeys = append(rawKeys, key) + } + sort.Strings(rawKeys) + normalizedOutputs := make(map[string]NotariusOutputConfig, len(cfg.Outputs)) + sourceOwners := map[string]string{} + laneOwners := map[string]string{} + for _, rawKey := range rawKeys { + output := cfg.Outputs[rawKey] + key := strings.TrimSpace(rawKey) + if !artifactpolicy.IsConfiguredKey(key) { + return fmt.Errorf("pipeline.notarius.outputs keys must match ^[a-z][a-z0-9_]*$") + } + sourceID := artifactpolicy.ExtractionSourceID(key) + if previousKey, ok := sourceOwners[sourceID]; ok { + return fmt.Errorf("pipeline.notarius.outputs keys %q and %q normalize to duplicate source %q", previousKey, rawKey, sourceID) + } + if owner, ok := reservedSources[sourceID]; ok { + return fmt.Errorf("pipeline.notarius.outputs.%s source %q collides with %s", key, sourceID, owner) + } + sourceOwners[sourceID] = rawKey + + output.LaneID = strings.TrimSpace(output.LaneID) + output.MediaType = strings.TrimSpace(output.MediaType) + output.SchemaID = strings.TrimSpace(output.SchemaID) + output.SchemaVersion = strings.TrimSpace(output.SchemaVersion) + output.ModuleKey = strings.TrimSpace(output.ModuleKey) + prefix := "pipeline.notarius.outputs." + key + if output.LaneID == "" { + return fmt.Errorf("%s.lane_id is required", prefix) + } + if previousKey, ok := laneOwners[output.LaneID]; ok { + return fmt.Errorf("%s.lane_id %q duplicates pipeline.notarius.outputs.%s.lane_id", prefix, output.LaneID, previousKey) + } + laneOwners[output.LaneID] = key + if output.MediaType == "" { + return fmt.Errorf("%s.media_type is required", prefix) + } + if output.SchemaID == "" { + return fmt.Errorf("%s.schema_id is required", prefix) + } + if output.SchemaVersion == "" { + return fmt.Errorf("%s.schema_version is required", prefix) + } + normalizedOutputs[key] = output + } + cfg.Binary = strings.TrimSpace(cfg.Binary) + cfg.ConfigPath = filepath.Clean(cfg.ConfigPath) + cfg.PipelineID = strings.TrimSpace(cfg.PipelineID) + cfg.Timeout = strings.TrimSpace(cfg.Timeout) + cfg.WorkingDirectory = filepath.Clean(cfg.WorkingDirectory) + cfg.References = normalizedReferences + cfg.Outputs = normalizedOutputs + return nil +} + +func notariusOutputKeySet(notarius *NotariusConfig) map[string]struct{} { + out := map[string]struct{}{} + if notarius == nil { + return out + } + for key := range notarius.Outputs { + trimmed := strings.TrimSpace(key) + if artifactpolicy.IsConfiguredKey(trimmed) { + out[trimmed] = struct{}{} + } + } + return out +} diff --git a/internal/config/validate_publish.go b/internal/config/validate_publish.go new file mode 100644 index 0000000..9bb3629 --- /dev/null +++ b/internal/config/validate_publish.go @@ -0,0 +1,178 @@ +package config + +import ( + "errors" + "fmt" + "strings" + + "gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy" + "gitea.maximumdirect.net/eric/narratio/internal/pathsafe" +) + +func validateStorage(cfg StorageConfig) error { + backend := strings.ToLower(strings.TrimSpace(cfg.Backend)) + switch backend { + case "", StorageBackendLocal: + if cfg.S3 != nil { + return fmt.Errorf("pipeline.storage.s3 is only supported when pipeline.storage.backend is s3") + } + return nil + case StorageBackendS3: + if cfg.S3 == nil { + return fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3") + } + default: + return fmt.Errorf("pipeline.storage.backend must be one of: local, s3") + } + if strings.TrimSpace(cfg.S3.RootPrefix) == "" { + return fmt.Errorf("pipeline.storage.s3.root_prefix must be non-empty") + } + if err := validateRelativeSafePath("pipeline.storage.s3.root_prefix", cfg.S3.RootPrefix); err != nil { + return err + } + if cfg.S3.Endpoint != "" && strings.TrimSpace(cfg.S3.Endpoint) == "" { + return fmt.Errorf("pipeline.storage.s3.endpoint must be non-empty when provided") + } + if err := validateEnvVarNameField("pipeline.storage.s3.access_key_id_env", cfg.S3.AccessKeyIDEnv); err != nil { + return err + } + if err := validateEnvVarNameField("pipeline.storage.s3.secret_access_key_env", cfg.S3.SecretKeyEnv); err != nil { + return err + } + return nil +} + +func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig, notarius *NotariusConfig) error { + if cfg == nil { + return nil + } + configuredOutputs := scriptoriumOutputPathMap(scriptorium) + extractionOutputs := notariusOutputKeySet(notarius) + seenDest := map[string]struct{}{} + for i, item := range cfg.Outputs { + prefix := fmt.Sprintf("pipeline.publish.outputs[%d]", i) + source := strings.TrimSpace(item.Source) + if source == "" { + return fmt.Errorf("%s.source is required", prefix) + } + if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil { + return fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err) + } + dest := strings.TrimSpace(item.Dest) + if dest == "" { + derivedDest, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(source, "", configuredOutputs, extractionOutputs) + if err != nil { + return fmt.Errorf("%s.dest is required when destination cannot be derived from %q: %w", prefix, source, err) + } + dest = derivedDest + cfg.Outputs[i].Dest = derivedDest + } + normalizedDest, err := pathsafe.NormalizeRelativeDestination(dest) + if err != nil { + switch { + case errors.Is(err, pathsafe.ErrRelativePathAbsolute): + return fmt.Errorf("%s.dest must be a relative path", prefix) + case errors.Is(err, pathsafe.ErrRelativePathEscape): + return fmt.Errorf("%s.dest must not contain path traversal", prefix) + default: + return fmt.Errorf("%s.dest must be non-empty", prefix) + } + } + if _, ok := seenDest[normalizedDest]; ok { + return fmt.Errorf("%s.dest %q duplicates another publish output destination", prefix, dest) + } + seenDest[normalizedDest] = struct{}{} + } + locks, err := ValidatePublishLockRules(cfg.Locks, scriptorium, notarius, "pipeline.publish.locks") + if err != nil { + return err + } + cfg.Locks = locks + return nil +} + +// ValidatePublishLockRules validates and normalizes source-based publish locks. +func ValidatePublishLockRules(locks []PublishLockRule, scriptorium *ScriptoriumConfig, notarius *NotariusConfig, label string) ([]PublishLockRule, error) { + seenLocks := map[string]struct{}{} + out := make([]PublishLockRule, 0, len(locks)) + configuredOutputs := scriptoriumOutputPathMap(scriptorium) + extractionOutputs := notariusOutputKeySet(notarius) + if strings.TrimSpace(label) == "" { + label = "publish.locks" + } + for i, item := range locks { + prefix := fmt.Sprintf("%s[%d]", label, i) + source := strings.TrimSpace(item.Source) + if source == "" { + return nil, fmt.Errorf("%s.source is required", prefix) + } + if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil { + return nil, fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err) + } + if _, ok := seenLocks[source]; ok { + return nil, fmt.Errorf("%s.source %q duplicates another publish lock source", prefix, source) + } + seenLocks[source] = struct{}{} + out = append(out, PublishLockRule{ + Source: source, + Reason: strings.TrimSpace(item.Reason), + }) + } + return out, nil +} + +// MergePublishLockRules returns the union of static and remote locks. Static +// locks win when both sources contain the same lock. +func MergePublishLockRules(staticLocks, remoteLocks []PublishLockRule) []PublishLockRule { + out := make([]PublishLockRule, 0, len(staticLocks)+len(remoteLocks)) + seen := map[string]struct{}{} + for _, item := range staticLocks { + source := strings.TrimSpace(item.Source) + if source == "" { + continue + } + out = append(out, PublishLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)}) + seen[source] = struct{}{} + } + for _, item := range remoteLocks { + source := strings.TrimSpace(item.Source) + if source == "" { + continue + } + if _, ok := seen[source]; ok { + continue + } + out = append(out, PublishLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)}) + seen[source] = struct{}{} + } + return out +} + +func scriptoriumOutputPathMap(scriptorium *ScriptoriumConfig) map[string]string { + out := map[string]string{} + if scriptorium == nil || len(scriptorium.Artifacts) == 0 { + return out + } + for key, artifactCfg := range scriptorium.Artifacts { + out[strings.TrimSpace(key)] = strings.TrimSpace(artifactCfg.OutputPath) + } + return out +} + +func publishUploadConfiguredForS3(pipeline *PipelineConfig) bool { + if pipeline == nil || pipeline.Publish == nil { + return false + } + if !strings.EqualFold(strings.TrimSpace(pipeline.Storage.Backend), StorageBackendS3) { + return false + } + enabled := true + if pipeline.Publish.Enabled != nil { + enabled = *pipeline.Publish.Enabled + } + upload := true + if pipeline.Publish.UploadRun != nil { + upload = *pipeline.Publish.UploadRun + } + return enabled && upload +} diff --git a/internal/config/validate_scriptorium.go b/internal/config/validate_scriptorium.go new file mode 100644 index 0000000..9d3b5c3 --- /dev/null +++ b/internal/config/validate_scriptorium.go @@ -0,0 +1,254 @@ +package config + +import ( + "errors" + "fmt" + "sort" + "strings" + + "gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy" +) + +func validateScriptorium(cfg *ScriptoriumConfig, notarius *NotariusConfig) error { + if cfg == nil { + return nil + } + if strings.TrimSpace(cfg.Binary) == "" { + return fmt.Errorf("pipeline.scriptorium.binary is required when pipeline.scriptorium is configured") + } + if cfg.ConfigPath != "" && strings.TrimSpace(cfg.ConfigPath) == "" { + return fmt.Errorf("pipeline.scriptorium.config_path must be non-empty when provided") + } + if err := validateDuration("pipeline.scriptorium.timeout", cfg.Timeout); err != nil { + return err + } + + configuredArtifacts := make(map[string]struct{}, len(cfg.Artifacts)) + referencedArtifacts := make(map[string]struct{}) + outputOwners := make(map[string]string, len(cfg.Artifacts)) + for artifactName := range cfg.Artifacts { + if !artifactpolicy.IsConfiguredKey(artifactName) { + return fmt.Errorf("pipeline.scriptorium.artifacts keys must match ^[a-z][a-z0-9_]*$") + } + configuredArtifacts[artifactName] = struct{}{} + } + + for artifactName, artifactCfg := range cfg.Artifacts { + if artifactCfg.Enabled && strings.TrimSpace(artifactCfg.PromptID) == "" { + return fmt.Errorf("pipeline.scriptorium.artifacts.%s.prompt_id is required when enabled", artifactName) + } + if artifactCfg.Enabled && strings.TrimSpace(artifactCfg.OutputPath) == "" { + return fmt.Errorf("pipeline.scriptorium.artifacts.%s.output_path is required when enabled", artifactName) + } + if strings.TrimSpace(artifactCfg.OutputPath) != "" { + pathField := "pipeline.scriptorium.artifacts." + artifactName + ".output_path" + if err := validateRelativeSafePath(pathField, artifactCfg.OutputPath); err != nil { + return err + } + if err := validatePathWithinRoot(pathField, artifactCfg.OutputPath, DefaultScriptoriumArtifactOutputRoot); err != nil { + return err + } + if previous, exists := outputOwners[artifactCfg.OutputPath]; exists { + return fmt.Errorf("%s.output_path %q duplicates artifact %q", pathField, artifactCfg.OutputPath, previous) + } + outputOwners[artifactCfg.OutputPath] = artifactName + } + if err := validateDuration("pipeline.scriptorium.artifacts."+artifactName+".timeout", artifactCfg.Timeout); err != nil { + return err + } + + depSet := make(map[string]struct{}, len(artifactCfg.DependsOn)) + for i, depName := range artifactCfg.DependsOn { + trimmedDep := strings.TrimSpace(depName) + field := fmt.Sprintf("pipeline.scriptorium.artifacts.%s.depends_on[%d]", artifactName, i) + if trimmedDep == "" { + return fmt.Errorf("%s must be non-empty", field) + } + if _, ok := configuredArtifacts[trimmedDep]; !ok { + return fmt.Errorf("%s %q is not a configured artifact key", field, depName) + } + if trimmedDep == artifactName { + return fmt.Errorf("pipeline.scriptorium.artifacts.%s.depends_on must not include itself", artifactName) + } + depSet[trimmedDep] = struct{}{} + referencedArtifacts[trimmedDep] = struct{}{} + } + + for inputName, inputCfg := range artifactCfg.Inputs { + trimmedInputName := strings.TrimSpace(inputName) + if trimmedInputName == "" { + return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs keys must be non-empty", artifactName) + } + source := strings.TrimSpace(inputCfg.Source) + if source == "" { + return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs.%s.source is required", artifactName, inputName) + } + + referencedArtifact, err := validateScriptoriumInputSource(artifactName, inputName, source, configuredArtifacts, notariusOutputKeySet(notarius)) + if err != nil { + return err + } + if referencedArtifact != "" { + if _, ok := depSet[referencedArtifact]; !ok { + return fmt.Errorf( + "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q requires depends_on entry %q", + artifactName, + inputName, + source, + referencedArtifact, + ) + } + referencedArtifacts[referencedArtifact] = struct{}{} + } + } + for varName, varValue := range artifactCfg.Vars { + if strings.TrimSpace(varName) == "" { + return fmt.Errorf("pipeline.scriptorium.artifacts.%s.vars keys must be non-empty", artifactName) + } + switch varValue.(type) { + case bool, string: + default: + return fmt.Errorf("pipeline.scriptorium.artifacts.%s.vars.%s must be a string or boolean", artifactName, varName) + } + } + } + + for artifactName := range referencedArtifacts { + artifactCfg := cfg.Artifacts[artifactName] + if strings.TrimSpace(artifactCfg.OutputPath) == "" { + return fmt.Errorf("pipeline.scriptorium.artifacts.%s.output_path is required when artifact is referenced", artifactName) + } + } + + if err := ValidateScriptoriumArtifactDependencies(cfg.Artifacts); err != nil { + return err + } + + return nil +} + +func validateScriptoriumInputSource( + artifactName, inputName, source string, + configuredArtifacts map[string]struct{}, + extractionOutputs map[string]struct{}, +) (string, error) { + trimmedSource := strings.TrimSpace(source) + descriptor, err := artifactpolicy.DescribeScriptoriumInputSource(trimmedSource) + if err != nil { + if errors.Is(err, artifactpolicy.ErrInvalidPreviousSessionSource) { + return "", fmt.Errorf( + "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q must reference configured artifact key matching ^[a-z][a-z0-9_]*$", + artifactName, + inputName, + source, + ) + } + return "", fmt.Errorf( + "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported", + artifactName, + inputName, + source, + ) + } + if err := artifactpolicy.ValidateInputReference(descriptor, configuredArtifacts, extractionOutputs); err != nil { + var unknownConfigured *artifactpolicy.UnknownConfiguredArtifactError + if errors.As(err, &unknownConfigured) { + return "", fmt.Errorf( + "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q references unknown artifact %q", + artifactName, + inputName, + source, + unknownConfigured.ConfiguredKey, + ) + } + var unknownExtraction *artifactpolicy.UnknownExtractionArtifactError + if errors.As(err, &unknownExtraction) { + return "", fmt.Errorf( + "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q references unknown extraction output %q", + artifactName, + inputName, + source, + unknownExtraction.ConfiguredKey, + ) + } + return "", fmt.Errorf( + "pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported", + artifactName, + inputName, + source, + ) + } + if descriptor.Source.Kind == artifactpolicy.SourceKindConfiguredArtifact { + return descriptor.Source.ConfiguredKey, nil + } + return "", nil +} + +// ValidateScriptoriumArtifactDependencies validates the configured dependency +// graph independently of execution selection. Disabled artifacts remain valid +// prerequisites for explicit selections and therefore participate in cycles. +func ValidateScriptoriumArtifactDependencies(artifacts map[string]ScriptoriumArtifactConfig) error { + if len(artifacts) == 0 { + return nil + } + + graph := make(map[string][]string, len(artifacts)) + for name, cfg := range artifacts { + if !artifactpolicy.IsConfiguredKey(name) { + return fmt.Errorf("pipeline.scriptorium.artifacts keys must match ^[a-z][a-z0-9_]*$") + } + seen := make(map[string]struct{}, len(cfg.DependsOn)) + for _, dep := range cfg.DependsOn { + trimmedDep := strings.TrimSpace(dep) + if trimmedDep == "" { + return fmt.Errorf("pipeline.scriptorium.artifacts.%s.depends_on entries must be non-empty", name) + } + if _, ok := artifacts[trimmedDep]; !ok { + return fmt.Errorf("pipeline.scriptorium.artifacts.%s dependency %q is not configured", name, dep) + } + if trimmedDep == name { + return fmt.Errorf("pipeline.scriptorium.artifacts.%s.depends_on must not include itself", name) + } + if _, duplicate := seen[trimmedDep]; duplicate { + continue + } + seen[trimmedDep] = struct{}{} + graph[name] = append(graph[name], trimmedDep) + } + sort.Strings(graph[name]) + } + + visiting := make(map[string]bool, len(artifacts)) + visited := make(map[string]bool, len(artifacts)) + + var visit func(node string) error + visit = func(node string) error { + if visiting[node] { + return fmt.Errorf("pipeline.scriptorium.artifacts dependencies must not contain cycles") + } + if visited[node] { + return nil + } + visiting[node] = true + for _, dep := range graph[node] { + if err := visit(dep); err != nil { + return err + } + } + visiting[node] = false + visited[node] = true + return nil + } + + nodes := make([]string, 0, len(artifacts)) + for node := range artifacts { + nodes = append(nodes, node) + } + sort.Strings(nodes) + for _, node := range nodes { + if err := visit(node); err != nil { + return err + } + } + return nil +}