Separate configuration domain validators
This commit is contained in:
@@ -178,7 +178,7 @@ primitive checks clear package-local owners. This stage is behavior-preserving.
|
||||
|
||||
## Stage 2 — Separate Publish, Storage, Notarius, And Scriptorium Owners
|
||||
|
||||
**Status:** Pending
|
||||
**Status:** Completed
|
||||
|
||||
### Goal
|
||||
|
||||
|
||||
@@ -1,16 +1,9 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/notariusref"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
||||
)
|
||||
|
||||
@@ -119,559 +112,3 @@ func validatePipeline(cfg *PipelineConfig) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateStorage(cfg StorageConfig) error {
|
||||
backend := strings.ToLower(strings.TrimSpace(cfg.Backend))
|
||||
switch backend {
|
||||
case "", StorageBackendLocal:
|
||||
if cfg.S3 != nil {
|
||||
return fmt.Errorf("pipeline.storage.s3 is only supported when pipeline.storage.backend is s3")
|
||||
}
|
||||
return nil
|
||||
case StorageBackendS3:
|
||||
if cfg.S3 == nil {
|
||||
return fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3")
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("pipeline.storage.backend must be one of: local, s3")
|
||||
}
|
||||
if strings.TrimSpace(cfg.S3.RootPrefix) == "" {
|
||||
return fmt.Errorf("pipeline.storage.s3.root_prefix must be non-empty")
|
||||
}
|
||||
if err := validateRelativeSafePath("pipeline.storage.s3.root_prefix", cfg.S3.RootPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.S3.Endpoint != "" && strings.TrimSpace(cfg.S3.Endpoint) == "" {
|
||||
return fmt.Errorf("pipeline.storage.s3.endpoint must be non-empty when provided")
|
||||
}
|
||||
if err := validateEnvVarNameField("pipeline.storage.s3.access_key_id_env", cfg.S3.AccessKeyIDEnv); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateEnvVarNameField("pipeline.storage.s3.secret_access_key_env", cfg.S3.SecretKeyEnv); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig, notarius *NotariusConfig) error {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
configuredOutputs := scriptoriumOutputPathMap(scriptorium)
|
||||
extractionOutputs := notariusOutputKeySet(notarius)
|
||||
seenDest := map[string]struct{}{}
|
||||
for i, item := range cfg.Outputs {
|
||||
prefix := fmt.Sprintf("pipeline.publish.outputs[%d]", i)
|
||||
source := strings.TrimSpace(item.Source)
|
||||
if source == "" {
|
||||
return fmt.Errorf("%s.source is required", prefix)
|
||||
}
|
||||
if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil {
|
||||
return fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
||||
}
|
||||
dest := strings.TrimSpace(item.Dest)
|
||||
if dest == "" {
|
||||
derivedDest, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(source, "", configuredOutputs, extractionOutputs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s.dest is required when destination cannot be derived from %q: %w", prefix, source, err)
|
||||
}
|
||||
dest = derivedDest
|
||||
cfg.Outputs[i].Dest = derivedDest
|
||||
}
|
||||
normalizedDest, err := pathsafe.NormalizeRelativeDestination(dest)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, pathsafe.ErrRelativePathAbsolute):
|
||||
return fmt.Errorf("%s.dest must be a relative path", prefix)
|
||||
case errors.Is(err, pathsafe.ErrRelativePathEscape):
|
||||
return fmt.Errorf("%s.dest must not contain path traversal", prefix)
|
||||
default:
|
||||
return fmt.Errorf("%s.dest must be non-empty", prefix)
|
||||
}
|
||||
}
|
||||
if _, ok := seenDest[normalizedDest]; ok {
|
||||
return fmt.Errorf("%s.dest %q duplicates another publish output destination", prefix, dest)
|
||||
}
|
||||
seenDest[normalizedDest] = struct{}{}
|
||||
}
|
||||
locks, err := ValidatePublishLockRules(cfg.Locks, scriptorium, notarius, "pipeline.publish.locks")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg.Locks = locks
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidatePublishLockRules validates and normalizes source-based publish locks.
|
||||
func ValidatePublishLockRules(locks []PublishLockRule, scriptorium *ScriptoriumConfig, notarius *NotariusConfig, label string) ([]PublishLockRule, error) {
|
||||
seenLocks := map[string]struct{}{}
|
||||
out := make([]PublishLockRule, 0, len(locks))
|
||||
configuredOutputs := scriptoriumOutputPathMap(scriptorium)
|
||||
extractionOutputs := notariusOutputKeySet(notarius)
|
||||
if strings.TrimSpace(label) == "" {
|
||||
label = "publish.locks"
|
||||
}
|
||||
for i, item := range locks {
|
||||
prefix := fmt.Sprintf("%s[%d]", label, i)
|
||||
source := strings.TrimSpace(item.Source)
|
||||
if source == "" {
|
||||
return nil, fmt.Errorf("%s.source is required", prefix)
|
||||
}
|
||||
if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil {
|
||||
return nil, fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
||||
}
|
||||
if _, ok := seenLocks[source]; ok {
|
||||
return nil, fmt.Errorf("%s.source %q duplicates another publish lock source", prefix, source)
|
||||
}
|
||||
seenLocks[source] = struct{}{}
|
||||
out = append(out, PublishLockRule{
|
||||
Source: source,
|
||||
Reason: strings.TrimSpace(item.Reason),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// MergePublishLockRules returns the union of static and remote locks. Static
|
||||
// locks win when both sources contain the same lock.
|
||||
func MergePublishLockRules(staticLocks, remoteLocks []PublishLockRule) []PublishLockRule {
|
||||
out := make([]PublishLockRule, 0, len(staticLocks)+len(remoteLocks))
|
||||
seen := map[string]struct{}{}
|
||||
for _, item := range staticLocks {
|
||||
source := strings.TrimSpace(item.Source)
|
||||
if source == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, PublishLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
|
||||
seen[source] = struct{}{}
|
||||
}
|
||||
for _, item := range remoteLocks {
|
||||
source := strings.TrimSpace(item.Source)
|
||||
if source == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[source]; ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, PublishLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
|
||||
seen[source] = struct{}{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func scriptoriumOutputPathMap(scriptorium *ScriptoriumConfig) map[string]string {
|
||||
out := map[string]string{}
|
||||
if scriptorium == nil || len(scriptorium.Artifacts) == 0 {
|
||||
return out
|
||||
}
|
||||
for key, artifactCfg := range scriptorium.Artifacts {
|
||||
out[strings.TrimSpace(key)] = strings.TrimSpace(artifactCfg.OutputPath)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func notariusOutputKeySet(notarius *NotariusConfig) map[string]struct{} {
|
||||
out := map[string]struct{}{}
|
||||
if notarius == nil {
|
||||
return out
|
||||
}
|
||||
for key := range notarius.Outputs {
|
||||
trimmed := strings.TrimSpace(key)
|
||||
if artifactpolicy.IsConfiguredKey(trimmed) {
|
||||
out[trimmed] = struct{}{}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func validateNotarius(cfg *NotariusConfig, scriptorium *ScriptoriumConfig) error {
|
||||
if cfg == nil || !cfg.Enabled {
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(cfg.Binary) == "" {
|
||||
return fmt.Errorf("pipeline.notarius.binary is required when pipeline.notarius.enabled is true")
|
||||
}
|
||||
if strings.TrimSpace(cfg.ConfigPath) == "" {
|
||||
return fmt.Errorf("pipeline.notarius.config_path is required when pipeline.notarius.enabled is true")
|
||||
}
|
||||
if strings.TrimSpace(cfg.PipelineID) == "" {
|
||||
return fmt.Errorf("pipeline.notarius.pipeline_id is required when pipeline.notarius.enabled is true")
|
||||
}
|
||||
if len(cfg.Outputs) == 0 {
|
||||
return fmt.Errorf("pipeline.notarius.outputs must contain at least one output when pipeline.notarius.enabled is true")
|
||||
}
|
||||
duration, err := time.ParseDuration(strings.TrimSpace(cfg.Timeout))
|
||||
if err != nil {
|
||||
return fmt.Errorf("pipeline.notarius.timeout must be a valid duration: %w", err)
|
||||
}
|
||||
if duration <= 0 {
|
||||
return fmt.Errorf("pipeline.notarius.timeout must be positive")
|
||||
}
|
||||
if strings.TrimSpace(cfg.WorkingDirectory) == "" {
|
||||
return fmt.Errorf("pipeline.notarius.working_directory is required when pipeline.notarius.enabled is true")
|
||||
}
|
||||
if len(cfg.References) > MaxNotariusReferenceBindings {
|
||||
return fmt.Errorf("pipeline.notarius.references must contain at most %d bindings", MaxNotariusReferenceBindings)
|
||||
}
|
||||
|
||||
referenceKeys := make([]string, 0, len(cfg.References))
|
||||
for selector := range cfg.References {
|
||||
referenceKeys = append(referenceKeys, selector)
|
||||
}
|
||||
sort.Strings(referenceKeys)
|
||||
var normalizedReferences map[string]string
|
||||
if cfg.References != nil {
|
||||
normalizedReferences = make(map[string]string, len(cfg.References))
|
||||
}
|
||||
referenceOwners := make(map[string]string, len(cfg.References))
|
||||
for _, rawSelector := range referenceKeys {
|
||||
selector, err := notariusref.NormalizeSelector(rawSelector)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pipeline.notarius.references selector %q is invalid: %w", rawSelector, err)
|
||||
}
|
||||
if previous, ok := referenceOwners[selector]; ok {
|
||||
return fmt.Errorf("pipeline.notarius.references selectors %q and %q normalize to %q", previous, rawSelector, selector)
|
||||
}
|
||||
referenceOwners[selector] = rawSelector
|
||||
|
||||
source := strings.TrimSpace(cfg.References[rawSelector])
|
||||
if source == "" {
|
||||
return fmt.Errorf("pipeline.notarius.references.%s source is required", selector)
|
||||
}
|
||||
if _, ok := artifactpolicy.DescribePreparedInputSource(source); !ok {
|
||||
return fmt.Errorf("pipeline.notarius.references.%s source %q is not a supported prepared input source", selector, source)
|
||||
}
|
||||
normalizedReferences[selector] = source
|
||||
}
|
||||
|
||||
reservedSources := map[string]string{}
|
||||
for _, spec := range artifactmodel.RuntimeTranscriptArtifacts() {
|
||||
reservedSources[spec.SourceID] = "built-in source"
|
||||
}
|
||||
reservedSources[artifactpolicy.SourceBoundsSession] = "built-in source"
|
||||
if scriptorium != nil {
|
||||
for key := range scriptorium.Artifacts {
|
||||
normalizedKey := strings.TrimSpace(key)
|
||||
reservedSources[artifactpolicy.ConfiguredSourceID(normalizedKey)] = "configured Scriptorium source"
|
||||
reservedSources[artifactpolicy.PreviousSessionSourceID(normalizedKey)] = "previous-session source"
|
||||
}
|
||||
}
|
||||
|
||||
rawKeys := make([]string, 0, len(cfg.Outputs))
|
||||
for key := range cfg.Outputs {
|
||||
rawKeys = append(rawKeys, key)
|
||||
}
|
||||
sort.Strings(rawKeys)
|
||||
normalizedOutputs := make(map[string]NotariusOutputConfig, len(cfg.Outputs))
|
||||
sourceOwners := map[string]string{}
|
||||
laneOwners := map[string]string{}
|
||||
for _, rawKey := range rawKeys {
|
||||
output := cfg.Outputs[rawKey]
|
||||
key := strings.TrimSpace(rawKey)
|
||||
if !artifactpolicy.IsConfiguredKey(key) {
|
||||
return fmt.Errorf("pipeline.notarius.outputs keys must match ^[a-z][a-z0-9_]*$")
|
||||
}
|
||||
sourceID := artifactpolicy.ExtractionSourceID(key)
|
||||
if previousKey, ok := sourceOwners[sourceID]; ok {
|
||||
return fmt.Errorf("pipeline.notarius.outputs keys %q and %q normalize to duplicate source %q", previousKey, rawKey, sourceID)
|
||||
}
|
||||
if owner, ok := reservedSources[sourceID]; ok {
|
||||
return fmt.Errorf("pipeline.notarius.outputs.%s source %q collides with %s", key, sourceID, owner)
|
||||
}
|
||||
sourceOwners[sourceID] = rawKey
|
||||
|
||||
output.LaneID = strings.TrimSpace(output.LaneID)
|
||||
output.MediaType = strings.TrimSpace(output.MediaType)
|
||||
output.SchemaID = strings.TrimSpace(output.SchemaID)
|
||||
output.SchemaVersion = strings.TrimSpace(output.SchemaVersion)
|
||||
output.ModuleKey = strings.TrimSpace(output.ModuleKey)
|
||||
prefix := "pipeline.notarius.outputs." + key
|
||||
if output.LaneID == "" {
|
||||
return fmt.Errorf("%s.lane_id is required", prefix)
|
||||
}
|
||||
if previousKey, ok := laneOwners[output.LaneID]; ok {
|
||||
return fmt.Errorf("%s.lane_id %q duplicates pipeline.notarius.outputs.%s.lane_id", prefix, output.LaneID, previousKey)
|
||||
}
|
||||
laneOwners[output.LaneID] = key
|
||||
if output.MediaType == "" {
|
||||
return fmt.Errorf("%s.media_type is required", prefix)
|
||||
}
|
||||
if output.SchemaID == "" {
|
||||
return fmt.Errorf("%s.schema_id is required", prefix)
|
||||
}
|
||||
if output.SchemaVersion == "" {
|
||||
return fmt.Errorf("%s.schema_version is required", prefix)
|
||||
}
|
||||
normalizedOutputs[key] = output
|
||||
}
|
||||
cfg.Binary = strings.TrimSpace(cfg.Binary)
|
||||
cfg.ConfigPath = filepath.Clean(cfg.ConfigPath)
|
||||
cfg.PipelineID = strings.TrimSpace(cfg.PipelineID)
|
||||
cfg.Timeout = strings.TrimSpace(cfg.Timeout)
|
||||
cfg.WorkingDirectory = filepath.Clean(cfg.WorkingDirectory)
|
||||
cfg.References = normalizedReferences
|
||||
cfg.Outputs = normalizedOutputs
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateScriptorium(cfg *ScriptoriumConfig, notarius *NotariusConfig) error {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(cfg.Binary) == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.binary is required when pipeline.scriptorium is configured")
|
||||
}
|
||||
if cfg.ConfigPath != "" && strings.TrimSpace(cfg.ConfigPath) == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.config_path must be non-empty when provided")
|
||||
}
|
||||
if err := validateDuration("pipeline.scriptorium.timeout", cfg.Timeout); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
configuredArtifacts := make(map[string]struct{}, len(cfg.Artifacts))
|
||||
referencedArtifacts := make(map[string]struct{})
|
||||
outputOwners := make(map[string]string, len(cfg.Artifacts))
|
||||
for artifactName := range cfg.Artifacts {
|
||||
if !artifactpolicy.IsConfiguredKey(artifactName) {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts keys must match ^[a-z][a-z0-9_]*$")
|
||||
}
|
||||
configuredArtifacts[artifactName] = struct{}{}
|
||||
}
|
||||
|
||||
for artifactName, artifactCfg := range cfg.Artifacts {
|
||||
if artifactCfg.Enabled && strings.TrimSpace(artifactCfg.PromptID) == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.prompt_id is required when enabled", artifactName)
|
||||
}
|
||||
if artifactCfg.Enabled && strings.TrimSpace(artifactCfg.OutputPath) == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.output_path is required when enabled", artifactName)
|
||||
}
|
||||
if strings.TrimSpace(artifactCfg.OutputPath) != "" {
|
||||
pathField := "pipeline.scriptorium.artifacts." + artifactName + ".output_path"
|
||||
if err := validateRelativeSafePath(pathField, artifactCfg.OutputPath); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validatePathWithinRoot(pathField, artifactCfg.OutputPath, DefaultScriptoriumArtifactOutputRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
if previous, exists := outputOwners[artifactCfg.OutputPath]; exists {
|
||||
return fmt.Errorf("%s.output_path %q duplicates artifact %q", pathField, artifactCfg.OutputPath, previous)
|
||||
}
|
||||
outputOwners[artifactCfg.OutputPath] = artifactName
|
||||
}
|
||||
if err := validateDuration("pipeline.scriptorium.artifacts."+artifactName+".timeout", artifactCfg.Timeout); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
depSet := make(map[string]struct{}, len(artifactCfg.DependsOn))
|
||||
for i, depName := range artifactCfg.DependsOn {
|
||||
trimmedDep := strings.TrimSpace(depName)
|
||||
field := fmt.Sprintf("pipeline.scriptorium.artifacts.%s.depends_on[%d]", artifactName, i)
|
||||
if trimmedDep == "" {
|
||||
return fmt.Errorf("%s must be non-empty", field)
|
||||
}
|
||||
if _, ok := configuredArtifacts[trimmedDep]; !ok {
|
||||
return fmt.Errorf("%s %q is not a configured artifact key", field, depName)
|
||||
}
|
||||
if trimmedDep == artifactName {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.depends_on must not include itself", artifactName)
|
||||
}
|
||||
depSet[trimmedDep] = struct{}{}
|
||||
referencedArtifacts[trimmedDep] = struct{}{}
|
||||
}
|
||||
|
||||
for inputName, inputCfg := range artifactCfg.Inputs {
|
||||
trimmedInputName := strings.TrimSpace(inputName)
|
||||
if trimmedInputName == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs keys must be non-empty", artifactName)
|
||||
}
|
||||
source := strings.TrimSpace(inputCfg.Source)
|
||||
if source == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs.%s.source is required", artifactName, inputName)
|
||||
}
|
||||
|
||||
referencedArtifact, err := validateScriptoriumInputSource(artifactName, inputName, source, configuredArtifacts, notariusOutputKeySet(notarius))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if referencedArtifact != "" {
|
||||
if _, ok := depSet[referencedArtifact]; !ok {
|
||||
return fmt.Errorf(
|
||||
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q requires depends_on entry %q",
|
||||
artifactName,
|
||||
inputName,
|
||||
source,
|
||||
referencedArtifact,
|
||||
)
|
||||
}
|
||||
referencedArtifacts[referencedArtifact] = struct{}{}
|
||||
}
|
||||
}
|
||||
for varName, varValue := range artifactCfg.Vars {
|
||||
if strings.TrimSpace(varName) == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.vars keys must be non-empty", artifactName)
|
||||
}
|
||||
switch varValue.(type) {
|
||||
case bool, string:
|
||||
default:
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.vars.%s must be a string or boolean", artifactName, varName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for artifactName := range referencedArtifacts {
|
||||
artifactCfg := cfg.Artifacts[artifactName]
|
||||
if strings.TrimSpace(artifactCfg.OutputPath) == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.output_path is required when artifact is referenced", artifactName)
|
||||
}
|
||||
}
|
||||
|
||||
if err := ValidateScriptoriumArtifactDependencies(cfg.Artifacts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func publishUploadConfiguredForS3(pipeline *PipelineConfig) bool {
|
||||
if pipeline == nil || pipeline.Publish == nil {
|
||||
return false
|
||||
}
|
||||
if !strings.EqualFold(strings.TrimSpace(pipeline.Storage.Backend), StorageBackendS3) {
|
||||
return false
|
||||
}
|
||||
enabled := true
|
||||
if pipeline.Publish.Enabled != nil {
|
||||
enabled = *pipeline.Publish.Enabled
|
||||
}
|
||||
upload := true
|
||||
if pipeline.Publish.UploadRun != nil {
|
||||
upload = *pipeline.Publish.UploadRun
|
||||
}
|
||||
return enabled && upload
|
||||
}
|
||||
|
||||
func validateScriptoriumInputSource(
|
||||
artifactName, inputName, source string,
|
||||
configuredArtifacts map[string]struct{},
|
||||
extractionOutputs map[string]struct{},
|
||||
) (string, error) {
|
||||
trimmedSource := strings.TrimSpace(source)
|
||||
descriptor, err := artifactpolicy.DescribeScriptoriumInputSource(trimmedSource)
|
||||
if err != nil {
|
||||
if errors.Is(err, artifactpolicy.ErrInvalidPreviousSessionSource) {
|
||||
return "", fmt.Errorf(
|
||||
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q must reference configured artifact key matching ^[a-z][a-z0-9_]*$",
|
||||
artifactName,
|
||||
inputName,
|
||||
source,
|
||||
)
|
||||
}
|
||||
return "", fmt.Errorf(
|
||||
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported",
|
||||
artifactName,
|
||||
inputName,
|
||||
source,
|
||||
)
|
||||
}
|
||||
if err := artifactpolicy.ValidateInputReference(descriptor, configuredArtifacts, extractionOutputs); err != nil {
|
||||
var unknownConfigured *artifactpolicy.UnknownConfiguredArtifactError
|
||||
if errors.As(err, &unknownConfigured) {
|
||||
return "", fmt.Errorf(
|
||||
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q references unknown artifact %q",
|
||||
artifactName,
|
||||
inputName,
|
||||
source,
|
||||
unknownConfigured.ConfiguredKey,
|
||||
)
|
||||
}
|
||||
var unknownExtraction *artifactpolicy.UnknownExtractionArtifactError
|
||||
if errors.As(err, &unknownExtraction) {
|
||||
return "", fmt.Errorf(
|
||||
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q references unknown extraction output %q",
|
||||
artifactName,
|
||||
inputName,
|
||||
source,
|
||||
unknownExtraction.ConfiguredKey,
|
||||
)
|
||||
}
|
||||
return "", fmt.Errorf(
|
||||
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported",
|
||||
artifactName,
|
||||
inputName,
|
||||
source,
|
||||
)
|
||||
}
|
||||
if descriptor.Source.Kind == artifactpolicy.SourceKindConfiguredArtifact {
|
||||
return descriptor.Source.ConfiguredKey, nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// ValidateScriptoriumArtifactDependencies validates the configured dependency
|
||||
// graph independently of execution selection. Disabled artifacts remain valid
|
||||
// prerequisites for explicit selections and therefore participate in cycles.
|
||||
func ValidateScriptoriumArtifactDependencies(artifacts map[string]ScriptoriumArtifactConfig) error {
|
||||
if len(artifacts) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
graph := make(map[string][]string, len(artifacts))
|
||||
for name, cfg := range artifacts {
|
||||
if !artifactpolicy.IsConfiguredKey(name) {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts keys must match ^[a-z][a-z0-9_]*$")
|
||||
}
|
||||
seen := make(map[string]struct{}, len(cfg.DependsOn))
|
||||
for _, dep := range cfg.DependsOn {
|
||||
trimmedDep := strings.TrimSpace(dep)
|
||||
if trimmedDep == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.depends_on entries must be non-empty", name)
|
||||
}
|
||||
if _, ok := artifacts[trimmedDep]; !ok {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s dependency %q is not configured", name, dep)
|
||||
}
|
||||
if trimmedDep == name {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.depends_on must not include itself", name)
|
||||
}
|
||||
if _, duplicate := seen[trimmedDep]; duplicate {
|
||||
continue
|
||||
}
|
||||
seen[trimmedDep] = struct{}{}
|
||||
graph[name] = append(graph[name], trimmedDep)
|
||||
}
|
||||
sort.Strings(graph[name])
|
||||
}
|
||||
|
||||
visiting := make(map[string]bool, len(artifacts))
|
||||
visited := make(map[string]bool, len(artifacts))
|
||||
|
||||
var visit func(node string) error
|
||||
visit = func(node string) error {
|
||||
if visiting[node] {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts dependencies must not contain cycles")
|
||||
}
|
||||
if visited[node] {
|
||||
return nil
|
||||
}
|
||||
visiting[node] = true
|
||||
for _, dep := range graph[node] {
|
||||
if err := visit(dep); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
visiting[node] = false
|
||||
visited[node] = true
|
||||
return nil
|
||||
}
|
||||
|
||||
nodes := make([]string, 0, len(artifacts))
|
||||
for node := range artifacts {
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
for _, node := range nodes {
|
||||
if err := visit(node); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
157
internal/config/validate_notarius.go
Normal file
157
internal/config/validate_notarius.go
Normal file
@@ -0,0 +1,157 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/notariusref"
|
||||
)
|
||||
|
||||
func validateNotarius(cfg *NotariusConfig, scriptorium *ScriptoriumConfig) error {
|
||||
if cfg == nil || !cfg.Enabled {
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(cfg.Binary) == "" {
|
||||
return fmt.Errorf("pipeline.notarius.binary is required when pipeline.notarius.enabled is true")
|
||||
}
|
||||
if strings.TrimSpace(cfg.ConfigPath) == "" {
|
||||
return fmt.Errorf("pipeline.notarius.config_path is required when pipeline.notarius.enabled is true")
|
||||
}
|
||||
if strings.TrimSpace(cfg.PipelineID) == "" {
|
||||
return fmt.Errorf("pipeline.notarius.pipeline_id is required when pipeline.notarius.enabled is true")
|
||||
}
|
||||
if len(cfg.Outputs) == 0 {
|
||||
return fmt.Errorf("pipeline.notarius.outputs must contain at least one output when pipeline.notarius.enabled is true")
|
||||
}
|
||||
duration, err := time.ParseDuration(strings.TrimSpace(cfg.Timeout))
|
||||
if err != nil {
|
||||
return fmt.Errorf("pipeline.notarius.timeout must be a valid duration: %w", err)
|
||||
}
|
||||
if duration <= 0 {
|
||||
return fmt.Errorf("pipeline.notarius.timeout must be positive")
|
||||
}
|
||||
if strings.TrimSpace(cfg.WorkingDirectory) == "" {
|
||||
return fmt.Errorf("pipeline.notarius.working_directory is required when pipeline.notarius.enabled is true")
|
||||
}
|
||||
if len(cfg.References) > MaxNotariusReferenceBindings {
|
||||
return fmt.Errorf("pipeline.notarius.references must contain at most %d bindings", MaxNotariusReferenceBindings)
|
||||
}
|
||||
|
||||
referenceKeys := make([]string, 0, len(cfg.References))
|
||||
for selector := range cfg.References {
|
||||
referenceKeys = append(referenceKeys, selector)
|
||||
}
|
||||
sort.Strings(referenceKeys)
|
||||
var normalizedReferences map[string]string
|
||||
if cfg.References != nil {
|
||||
normalizedReferences = make(map[string]string, len(cfg.References))
|
||||
}
|
||||
referenceOwners := make(map[string]string, len(cfg.References))
|
||||
for _, rawSelector := range referenceKeys {
|
||||
selector, err := notariusref.NormalizeSelector(rawSelector)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pipeline.notarius.references selector %q is invalid: %w", rawSelector, err)
|
||||
}
|
||||
if previous, ok := referenceOwners[selector]; ok {
|
||||
return fmt.Errorf("pipeline.notarius.references selectors %q and %q normalize to %q", previous, rawSelector, selector)
|
||||
}
|
||||
referenceOwners[selector] = rawSelector
|
||||
|
||||
source := strings.TrimSpace(cfg.References[rawSelector])
|
||||
if source == "" {
|
||||
return fmt.Errorf("pipeline.notarius.references.%s source is required", selector)
|
||||
}
|
||||
if _, ok := artifactpolicy.DescribePreparedInputSource(source); !ok {
|
||||
return fmt.Errorf("pipeline.notarius.references.%s source %q is not a supported prepared input source", selector, source)
|
||||
}
|
||||
normalizedReferences[selector] = source
|
||||
}
|
||||
|
||||
reservedSources := map[string]string{}
|
||||
for _, spec := range artifactmodel.RuntimeTranscriptArtifacts() {
|
||||
reservedSources[spec.SourceID] = "built-in source"
|
||||
}
|
||||
reservedSources[artifactpolicy.SourceBoundsSession] = "built-in source"
|
||||
if scriptorium != nil {
|
||||
for key := range scriptorium.Artifacts {
|
||||
normalizedKey := strings.TrimSpace(key)
|
||||
reservedSources[artifactpolicy.ConfiguredSourceID(normalizedKey)] = "configured Scriptorium source"
|
||||
reservedSources[artifactpolicy.PreviousSessionSourceID(normalizedKey)] = "previous-session source"
|
||||
}
|
||||
}
|
||||
|
||||
rawKeys := make([]string, 0, len(cfg.Outputs))
|
||||
for key := range cfg.Outputs {
|
||||
rawKeys = append(rawKeys, key)
|
||||
}
|
||||
sort.Strings(rawKeys)
|
||||
normalizedOutputs := make(map[string]NotariusOutputConfig, len(cfg.Outputs))
|
||||
sourceOwners := map[string]string{}
|
||||
laneOwners := map[string]string{}
|
||||
for _, rawKey := range rawKeys {
|
||||
output := cfg.Outputs[rawKey]
|
||||
key := strings.TrimSpace(rawKey)
|
||||
if !artifactpolicy.IsConfiguredKey(key) {
|
||||
return fmt.Errorf("pipeline.notarius.outputs keys must match ^[a-z][a-z0-9_]*$")
|
||||
}
|
||||
sourceID := artifactpolicy.ExtractionSourceID(key)
|
||||
if previousKey, ok := sourceOwners[sourceID]; ok {
|
||||
return fmt.Errorf("pipeline.notarius.outputs keys %q and %q normalize to duplicate source %q", previousKey, rawKey, sourceID)
|
||||
}
|
||||
if owner, ok := reservedSources[sourceID]; ok {
|
||||
return fmt.Errorf("pipeline.notarius.outputs.%s source %q collides with %s", key, sourceID, owner)
|
||||
}
|
||||
sourceOwners[sourceID] = rawKey
|
||||
|
||||
output.LaneID = strings.TrimSpace(output.LaneID)
|
||||
output.MediaType = strings.TrimSpace(output.MediaType)
|
||||
output.SchemaID = strings.TrimSpace(output.SchemaID)
|
||||
output.SchemaVersion = strings.TrimSpace(output.SchemaVersion)
|
||||
output.ModuleKey = strings.TrimSpace(output.ModuleKey)
|
||||
prefix := "pipeline.notarius.outputs." + key
|
||||
if output.LaneID == "" {
|
||||
return fmt.Errorf("%s.lane_id is required", prefix)
|
||||
}
|
||||
if previousKey, ok := laneOwners[output.LaneID]; ok {
|
||||
return fmt.Errorf("%s.lane_id %q duplicates pipeline.notarius.outputs.%s.lane_id", prefix, output.LaneID, previousKey)
|
||||
}
|
||||
laneOwners[output.LaneID] = key
|
||||
if output.MediaType == "" {
|
||||
return fmt.Errorf("%s.media_type is required", prefix)
|
||||
}
|
||||
if output.SchemaID == "" {
|
||||
return fmt.Errorf("%s.schema_id is required", prefix)
|
||||
}
|
||||
if output.SchemaVersion == "" {
|
||||
return fmt.Errorf("%s.schema_version is required", prefix)
|
||||
}
|
||||
normalizedOutputs[key] = output
|
||||
}
|
||||
cfg.Binary = strings.TrimSpace(cfg.Binary)
|
||||
cfg.ConfigPath = filepath.Clean(cfg.ConfigPath)
|
||||
cfg.PipelineID = strings.TrimSpace(cfg.PipelineID)
|
||||
cfg.Timeout = strings.TrimSpace(cfg.Timeout)
|
||||
cfg.WorkingDirectory = filepath.Clean(cfg.WorkingDirectory)
|
||||
cfg.References = normalizedReferences
|
||||
cfg.Outputs = normalizedOutputs
|
||||
return nil
|
||||
}
|
||||
|
||||
func notariusOutputKeySet(notarius *NotariusConfig) map[string]struct{} {
|
||||
out := map[string]struct{}{}
|
||||
if notarius == nil {
|
||||
return out
|
||||
}
|
||||
for key := range notarius.Outputs {
|
||||
trimmed := strings.TrimSpace(key)
|
||||
if artifactpolicy.IsConfiguredKey(trimmed) {
|
||||
out[trimmed] = struct{}{}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
178
internal/config/validate_publish.go
Normal file
178
internal/config/validate_publish.go
Normal file
@@ -0,0 +1,178 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
|
||||
)
|
||||
|
||||
func validateStorage(cfg StorageConfig) error {
|
||||
backend := strings.ToLower(strings.TrimSpace(cfg.Backend))
|
||||
switch backend {
|
||||
case "", StorageBackendLocal:
|
||||
if cfg.S3 != nil {
|
||||
return fmt.Errorf("pipeline.storage.s3 is only supported when pipeline.storage.backend is s3")
|
||||
}
|
||||
return nil
|
||||
case StorageBackendS3:
|
||||
if cfg.S3 == nil {
|
||||
return fmt.Errorf("pipeline.storage.s3 is required when pipeline.storage.backend is s3")
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("pipeline.storage.backend must be one of: local, s3")
|
||||
}
|
||||
if strings.TrimSpace(cfg.S3.RootPrefix) == "" {
|
||||
return fmt.Errorf("pipeline.storage.s3.root_prefix must be non-empty")
|
||||
}
|
||||
if err := validateRelativeSafePath("pipeline.storage.s3.root_prefix", cfg.S3.RootPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.S3.Endpoint != "" && strings.TrimSpace(cfg.S3.Endpoint) == "" {
|
||||
return fmt.Errorf("pipeline.storage.s3.endpoint must be non-empty when provided")
|
||||
}
|
||||
if err := validateEnvVarNameField("pipeline.storage.s3.access_key_id_env", cfg.S3.AccessKeyIDEnv); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateEnvVarNameField("pipeline.storage.s3.secret_access_key_env", cfg.S3.SecretKeyEnv); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig, notarius *NotariusConfig) error {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
configuredOutputs := scriptoriumOutputPathMap(scriptorium)
|
||||
extractionOutputs := notariusOutputKeySet(notarius)
|
||||
seenDest := map[string]struct{}{}
|
||||
for i, item := range cfg.Outputs {
|
||||
prefix := fmt.Sprintf("pipeline.publish.outputs[%d]", i)
|
||||
source := strings.TrimSpace(item.Source)
|
||||
if source == "" {
|
||||
return fmt.Errorf("%s.source is required", prefix)
|
||||
}
|
||||
if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil {
|
||||
return fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
||||
}
|
||||
dest := strings.TrimSpace(item.Dest)
|
||||
if dest == "" {
|
||||
derivedDest, err := artifactpolicy.ResolvePublishedDestinationWithExtractions(source, "", configuredOutputs, extractionOutputs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s.dest is required when destination cannot be derived from %q: %w", prefix, source, err)
|
||||
}
|
||||
dest = derivedDest
|
||||
cfg.Outputs[i].Dest = derivedDest
|
||||
}
|
||||
normalizedDest, err := pathsafe.NormalizeRelativeDestination(dest)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, pathsafe.ErrRelativePathAbsolute):
|
||||
return fmt.Errorf("%s.dest must be a relative path", prefix)
|
||||
case errors.Is(err, pathsafe.ErrRelativePathEscape):
|
||||
return fmt.Errorf("%s.dest must not contain path traversal", prefix)
|
||||
default:
|
||||
return fmt.Errorf("%s.dest must be non-empty", prefix)
|
||||
}
|
||||
}
|
||||
if _, ok := seenDest[normalizedDest]; ok {
|
||||
return fmt.Errorf("%s.dest %q duplicates another publish output destination", prefix, dest)
|
||||
}
|
||||
seenDest[normalizedDest] = struct{}{}
|
||||
}
|
||||
locks, err := ValidatePublishLockRules(cfg.Locks, scriptorium, notarius, "pipeline.publish.locks")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg.Locks = locks
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidatePublishLockRules validates and normalizes source-based publish locks.
|
||||
func ValidatePublishLockRules(locks []PublishLockRule, scriptorium *ScriptoriumConfig, notarius *NotariusConfig, label string) ([]PublishLockRule, error) {
|
||||
seenLocks := map[string]struct{}{}
|
||||
out := make([]PublishLockRule, 0, len(locks))
|
||||
configuredOutputs := scriptoriumOutputPathMap(scriptorium)
|
||||
extractionOutputs := notariusOutputKeySet(notarius)
|
||||
if strings.TrimSpace(label) == "" {
|
||||
label = "publish.locks"
|
||||
}
|
||||
for i, item := range locks {
|
||||
prefix := fmt.Sprintf("%s[%d]", label, i)
|
||||
source := strings.TrimSpace(item.Source)
|
||||
if source == "" {
|
||||
return nil, fmt.Errorf("%s.source is required", prefix)
|
||||
}
|
||||
if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil {
|
||||
return nil, fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
|
||||
}
|
||||
if _, ok := seenLocks[source]; ok {
|
||||
return nil, fmt.Errorf("%s.source %q duplicates another publish lock source", prefix, source)
|
||||
}
|
||||
seenLocks[source] = struct{}{}
|
||||
out = append(out, PublishLockRule{
|
||||
Source: source,
|
||||
Reason: strings.TrimSpace(item.Reason),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// MergePublishLockRules returns the union of static and remote locks. Static
|
||||
// locks win when both sources contain the same lock.
|
||||
func MergePublishLockRules(staticLocks, remoteLocks []PublishLockRule) []PublishLockRule {
|
||||
out := make([]PublishLockRule, 0, len(staticLocks)+len(remoteLocks))
|
||||
seen := map[string]struct{}{}
|
||||
for _, item := range staticLocks {
|
||||
source := strings.TrimSpace(item.Source)
|
||||
if source == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, PublishLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
|
||||
seen[source] = struct{}{}
|
||||
}
|
||||
for _, item := range remoteLocks {
|
||||
source := strings.TrimSpace(item.Source)
|
||||
if source == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[source]; ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, PublishLockRule{Source: source, Reason: strings.TrimSpace(item.Reason)})
|
||||
seen[source] = struct{}{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func scriptoriumOutputPathMap(scriptorium *ScriptoriumConfig) map[string]string {
|
||||
out := map[string]string{}
|
||||
if scriptorium == nil || len(scriptorium.Artifacts) == 0 {
|
||||
return out
|
||||
}
|
||||
for key, artifactCfg := range scriptorium.Artifacts {
|
||||
out[strings.TrimSpace(key)] = strings.TrimSpace(artifactCfg.OutputPath)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func publishUploadConfiguredForS3(pipeline *PipelineConfig) bool {
|
||||
if pipeline == nil || pipeline.Publish == nil {
|
||||
return false
|
||||
}
|
||||
if !strings.EqualFold(strings.TrimSpace(pipeline.Storage.Backend), StorageBackendS3) {
|
||||
return false
|
||||
}
|
||||
enabled := true
|
||||
if pipeline.Publish.Enabled != nil {
|
||||
enabled = *pipeline.Publish.Enabled
|
||||
}
|
||||
upload := true
|
||||
if pipeline.Publish.UploadRun != nil {
|
||||
upload = *pipeline.Publish.UploadRun
|
||||
}
|
||||
return enabled && upload
|
||||
}
|
||||
254
internal/config/validate_scriptorium.go
Normal file
254
internal/config/validate_scriptorium.go
Normal file
@@ -0,0 +1,254 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
|
||||
)
|
||||
|
||||
func validateScriptorium(cfg *ScriptoriumConfig, notarius *NotariusConfig) error {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(cfg.Binary) == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.binary is required when pipeline.scriptorium is configured")
|
||||
}
|
||||
if cfg.ConfigPath != "" && strings.TrimSpace(cfg.ConfigPath) == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.config_path must be non-empty when provided")
|
||||
}
|
||||
if err := validateDuration("pipeline.scriptorium.timeout", cfg.Timeout); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
configuredArtifacts := make(map[string]struct{}, len(cfg.Artifacts))
|
||||
referencedArtifacts := make(map[string]struct{})
|
||||
outputOwners := make(map[string]string, len(cfg.Artifacts))
|
||||
for artifactName := range cfg.Artifacts {
|
||||
if !artifactpolicy.IsConfiguredKey(artifactName) {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts keys must match ^[a-z][a-z0-9_]*$")
|
||||
}
|
||||
configuredArtifacts[artifactName] = struct{}{}
|
||||
}
|
||||
|
||||
for artifactName, artifactCfg := range cfg.Artifacts {
|
||||
if artifactCfg.Enabled && strings.TrimSpace(artifactCfg.PromptID) == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.prompt_id is required when enabled", artifactName)
|
||||
}
|
||||
if artifactCfg.Enabled && strings.TrimSpace(artifactCfg.OutputPath) == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.output_path is required when enabled", artifactName)
|
||||
}
|
||||
if strings.TrimSpace(artifactCfg.OutputPath) != "" {
|
||||
pathField := "pipeline.scriptorium.artifacts." + artifactName + ".output_path"
|
||||
if err := validateRelativeSafePath(pathField, artifactCfg.OutputPath); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validatePathWithinRoot(pathField, artifactCfg.OutputPath, DefaultScriptoriumArtifactOutputRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
if previous, exists := outputOwners[artifactCfg.OutputPath]; exists {
|
||||
return fmt.Errorf("%s.output_path %q duplicates artifact %q", pathField, artifactCfg.OutputPath, previous)
|
||||
}
|
||||
outputOwners[artifactCfg.OutputPath] = artifactName
|
||||
}
|
||||
if err := validateDuration("pipeline.scriptorium.artifacts."+artifactName+".timeout", artifactCfg.Timeout); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
depSet := make(map[string]struct{}, len(artifactCfg.DependsOn))
|
||||
for i, depName := range artifactCfg.DependsOn {
|
||||
trimmedDep := strings.TrimSpace(depName)
|
||||
field := fmt.Sprintf("pipeline.scriptorium.artifacts.%s.depends_on[%d]", artifactName, i)
|
||||
if trimmedDep == "" {
|
||||
return fmt.Errorf("%s must be non-empty", field)
|
||||
}
|
||||
if _, ok := configuredArtifacts[trimmedDep]; !ok {
|
||||
return fmt.Errorf("%s %q is not a configured artifact key", field, depName)
|
||||
}
|
||||
if trimmedDep == artifactName {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.depends_on must not include itself", artifactName)
|
||||
}
|
||||
depSet[trimmedDep] = struct{}{}
|
||||
referencedArtifacts[trimmedDep] = struct{}{}
|
||||
}
|
||||
|
||||
for inputName, inputCfg := range artifactCfg.Inputs {
|
||||
trimmedInputName := strings.TrimSpace(inputName)
|
||||
if trimmedInputName == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs keys must be non-empty", artifactName)
|
||||
}
|
||||
source := strings.TrimSpace(inputCfg.Source)
|
||||
if source == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs.%s.source is required", artifactName, inputName)
|
||||
}
|
||||
|
||||
referencedArtifact, err := validateScriptoriumInputSource(artifactName, inputName, source, configuredArtifacts, notariusOutputKeySet(notarius))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if referencedArtifact != "" {
|
||||
if _, ok := depSet[referencedArtifact]; !ok {
|
||||
return fmt.Errorf(
|
||||
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q requires depends_on entry %q",
|
||||
artifactName,
|
||||
inputName,
|
||||
source,
|
||||
referencedArtifact,
|
||||
)
|
||||
}
|
||||
referencedArtifacts[referencedArtifact] = struct{}{}
|
||||
}
|
||||
}
|
||||
for varName, varValue := range artifactCfg.Vars {
|
||||
if strings.TrimSpace(varName) == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.vars keys must be non-empty", artifactName)
|
||||
}
|
||||
switch varValue.(type) {
|
||||
case bool, string:
|
||||
default:
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.vars.%s must be a string or boolean", artifactName, varName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for artifactName := range referencedArtifacts {
|
||||
artifactCfg := cfg.Artifacts[artifactName]
|
||||
if strings.TrimSpace(artifactCfg.OutputPath) == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.output_path is required when artifact is referenced", artifactName)
|
||||
}
|
||||
}
|
||||
|
||||
if err := ValidateScriptoriumArtifactDependencies(cfg.Artifacts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateScriptoriumInputSource(
|
||||
artifactName, inputName, source string,
|
||||
configuredArtifacts map[string]struct{},
|
||||
extractionOutputs map[string]struct{},
|
||||
) (string, error) {
|
||||
trimmedSource := strings.TrimSpace(source)
|
||||
descriptor, err := artifactpolicy.DescribeScriptoriumInputSource(trimmedSource)
|
||||
if err != nil {
|
||||
if errors.Is(err, artifactpolicy.ErrInvalidPreviousSessionSource) {
|
||||
return "", fmt.Errorf(
|
||||
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q must reference configured artifact key matching ^[a-z][a-z0-9_]*$",
|
||||
artifactName,
|
||||
inputName,
|
||||
source,
|
||||
)
|
||||
}
|
||||
return "", fmt.Errorf(
|
||||
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported",
|
||||
artifactName,
|
||||
inputName,
|
||||
source,
|
||||
)
|
||||
}
|
||||
if err := artifactpolicy.ValidateInputReference(descriptor, configuredArtifacts, extractionOutputs); err != nil {
|
||||
var unknownConfigured *artifactpolicy.UnknownConfiguredArtifactError
|
||||
if errors.As(err, &unknownConfigured) {
|
||||
return "", fmt.Errorf(
|
||||
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q references unknown artifact %q",
|
||||
artifactName,
|
||||
inputName,
|
||||
source,
|
||||
unknownConfigured.ConfiguredKey,
|
||||
)
|
||||
}
|
||||
var unknownExtraction *artifactpolicy.UnknownExtractionArtifactError
|
||||
if errors.As(err, &unknownExtraction) {
|
||||
return "", fmt.Errorf(
|
||||
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q references unknown extraction output %q",
|
||||
artifactName,
|
||||
inputName,
|
||||
source,
|
||||
unknownExtraction.ConfiguredKey,
|
||||
)
|
||||
}
|
||||
return "", fmt.Errorf(
|
||||
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported",
|
||||
artifactName,
|
||||
inputName,
|
||||
source,
|
||||
)
|
||||
}
|
||||
if descriptor.Source.Kind == artifactpolicy.SourceKindConfiguredArtifact {
|
||||
return descriptor.Source.ConfiguredKey, nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// ValidateScriptoriumArtifactDependencies validates the configured dependency
|
||||
// graph independently of execution selection. Disabled artifacts remain valid
|
||||
// prerequisites for explicit selections and therefore participate in cycles.
|
||||
func ValidateScriptoriumArtifactDependencies(artifacts map[string]ScriptoriumArtifactConfig) error {
|
||||
if len(artifacts) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
graph := make(map[string][]string, len(artifacts))
|
||||
for name, cfg := range artifacts {
|
||||
if !artifactpolicy.IsConfiguredKey(name) {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts keys must match ^[a-z][a-z0-9_]*$")
|
||||
}
|
||||
seen := make(map[string]struct{}, len(cfg.DependsOn))
|
||||
for _, dep := range cfg.DependsOn {
|
||||
trimmedDep := strings.TrimSpace(dep)
|
||||
if trimmedDep == "" {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.depends_on entries must be non-empty", name)
|
||||
}
|
||||
if _, ok := artifacts[trimmedDep]; !ok {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s dependency %q is not configured", name, dep)
|
||||
}
|
||||
if trimmedDep == name {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.depends_on must not include itself", name)
|
||||
}
|
||||
if _, duplicate := seen[trimmedDep]; duplicate {
|
||||
continue
|
||||
}
|
||||
seen[trimmedDep] = struct{}{}
|
||||
graph[name] = append(graph[name], trimmedDep)
|
||||
}
|
||||
sort.Strings(graph[name])
|
||||
}
|
||||
|
||||
visiting := make(map[string]bool, len(artifacts))
|
||||
visited := make(map[string]bool, len(artifacts))
|
||||
|
||||
var visit func(node string) error
|
||||
visit = func(node string) error {
|
||||
if visiting[node] {
|
||||
return fmt.Errorf("pipeline.scriptorium.artifacts dependencies must not contain cycles")
|
||||
}
|
||||
if visited[node] {
|
||||
return nil
|
||||
}
|
||||
visiting[node] = true
|
||||
for _, dep := range graph[node] {
|
||||
if err := visit(dep); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
visiting[node] = false
|
||||
visited[node] = true
|
||||
return nil
|
||||
}
|
||||
|
||||
nodes := make([]string, 0, len(artifacts))
|
||||
for node := range artifacts {
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
for _, node := range nodes {
|
||||
if err := visit(node); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user