eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 15 Wiki Activity

Add Notarius configuration and extraction source policy

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-09 23:30:32 +00:00
parent 951383226c
commit 98734644d6
15 changed files with 678 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
internal/config/validate.go
View File

@@ -6,9 +6,11 @@ import (
"net/url"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
"gitea.maximumdirect.net/eric/narratio/internal/pathsafe"
)
@@ -85,7 +87,10 @@ func validatePipeline(cfg *PipelineConfig) error {
if err := validateCache(cfg.Cache); err != nil {
return err
}
if err := validatePublish(cfg.Publish, cfg.Scriptorium); err != nil {
if err := validateNotarius(cfg.Notarius, cfg.Scriptorium); err != nil {
return err
}
if err := validatePublish(cfg.Publish, cfg.Scriptorium, cfg.Notarius); err != nil {
return err
}
if err := validateWhisperX(cfg.WhisperX); err != nil {
@@ -106,7 +111,7 @@ func validatePipeline(cfg *PipelineConfig) error {
if err := validateRender(cfg.Render); err != nil {
return err
}
if err := validateScriptorium(cfg.Scriptorium); err != nil {
if err := validateScriptorium(cfg.Scriptorium, cfg.Notarius); err != nil {
return err
}
if err := validateDuration("pipeline.notification.timeout", cfg.Notification.Timeout); err != nil {
@@ -149,11 +154,12 @@ func validateCache(cfg CacheConfig) error {
return nil
}
func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig) error {
func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig, notarius *NotariusConfig) error {
if cfg == nil {
return nil
}
configuredOutputs := scriptoriumOutputPathMap(scriptorium)
extractionOutputs := notariusOutputKeySet(notarius)
seenDest := map[string]struct{}{}
for i, item := range cfg.Outputs {
prefix := fmt.Sprintf("pipeline.publish.outputs[%d]", i)
@@ -161,7 +167,7 @@ func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig) error {
if source == "" {
return fmt.Errorf("%s.source is required", prefix)
}
if _, err := artifactpolicy.ValidatePublishSource(source, configuredOutputs); err != nil {
if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil {
return fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
}
dest := strings.TrimSpace(item.Dest)
@@ -189,7 +195,7 @@ func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig) error {
}
seenDest[normalizedDest] = struct{}{}
}
locks, err := ValidatePublishLockRules(cfg.Locks, scriptorium, "pipeline.publish.locks")
locks, err := ValidatePublishLockRules(cfg.Locks, scriptorium, notarius, "pipeline.publish.locks")
if err != nil {
return err
}
@@ -198,10 +204,11 @@ func validatePublish(cfg *PublishConfig, scriptorium *ScriptoriumConfig) error {
}
// ValidatePublishLockRules validates and normalizes source-based publish locks.
func ValidatePublishLockRules(locks []PublishLockRule, scriptorium *ScriptoriumConfig, label string) ([]PublishLockRule, error) {
func ValidatePublishLockRules(locks []PublishLockRule, scriptorium *ScriptoriumConfig, notarius *NotariusConfig, label string) ([]PublishLockRule, error) {
seenLocks := map[string]struct{}{}
out := make([]PublishLockRule, 0, len(locks))
configuredOutputs := scriptoriumOutputPathMap(scriptorium)
extractionOutputs := notariusOutputKeySet(notarius)
if strings.TrimSpace(label) == "" {
label = "publish.locks"
}
@@ -211,7 +218,7 @@ func ValidatePublishLockRules(locks []PublishLockRule, scriptorium *ScriptoriumC
if source == "" {
return nil, fmt.Errorf("%s.source is required", prefix)
}
if _, err := artifactpolicy.ValidatePublishSource(source, configuredOutputs); err != nil {
if _, err := artifactpolicy.ValidatePublishSourceWithExtractions(source, configuredOutputs, extractionOutputs); err != nil {
return nil, fmt.Errorf("%s.source %q is unsupported: %w", prefix, item.Source, err)
}
if _, ok := seenLocks[source]; ok {
@@ -264,6 +271,20 @@ func scriptoriumOutputPathMap(scriptorium *ScriptoriumConfig) map[string]string
return out
}
func notariusOutputKeySet(notarius *NotariusConfig) map[string]struct{} {
out := map[string]struct{}{}
if notarius == nil {
return out
}
for key := range notarius.Outputs {
trimmed := strings.TrimSpace(key)
if artifactpolicy.IsConfiguredKey(trimmed) {
out[trimmed] = struct{}{}
}
}
return out
}
func validateSecrets(cfg *SecretsConfig) error {
if cfg == nil {
return nil
@@ -474,7 +495,103 @@ func validateAudita(cfg AuditaConfig) error {
return nil
}
func validateScriptorium(cfg *ScriptoriumConfig) error {
func validateNotarius(cfg *NotariusConfig, scriptorium *ScriptoriumConfig) error {
if cfg == nil || !cfg.Enabled {
return nil
}
if strings.TrimSpace(cfg.Binary) == "" {
return fmt.Errorf("pipeline.notarius.binary is required when pipeline.notarius.enabled is true")
}
if strings.TrimSpace(cfg.ConfigPath) == "" {
return fmt.Errorf("pipeline.notarius.config_path is required when pipeline.notarius.enabled is true")
}
if strings.TrimSpace(cfg.PipelineID) == "" {
return fmt.Errorf("pipeline.notarius.pipeline_id is required when pipeline.notarius.enabled is true")
}
if len(cfg.Outputs) == 0 {
return fmt.Errorf("pipeline.notarius.outputs must contain at least one output when pipeline.notarius.enabled is true")
}
duration, err := time.ParseDuration(strings.TrimSpace(cfg.Timeout))
if err != nil {
return fmt.Errorf("pipeline.notarius.timeout must be a valid duration: %w", err)
}
if duration <= 0 {
return fmt.Errorf("pipeline.notarius.timeout must be positive")
}
if strings.TrimSpace(cfg.WorkingDirectory) == "" {
return fmt.Errorf("pipeline.notarius.working_directory is required when pipeline.notarius.enabled is true")
}
reservedSources := map[string]string{}
for _, spec := range artifactmodel.RuntimeTranscriptArtifacts() {
reservedSources[spec.SourceID] = "built-in source"
}
reservedSources[artifactpolicy.SourceBoundsSession] = "built-in source"
if scriptorium != nil {
for key := range scriptorium.Artifacts {
normalizedKey := strings.TrimSpace(key)
reservedSources[artifactpolicy.ConfiguredSourceID(normalizedKey)] = "configured Scriptorium source"
reservedSources[artifactpolicy.PreviousSessionSourceID(normalizedKey)] = "previous-session source"
}
}
rawKeys := make([]string, 0, len(cfg.Outputs))
for key := range cfg.Outputs {
rawKeys = append(rawKeys, key)
}
sort.Strings(rawKeys)
normalizedOutputs := make(map[string]NotariusOutputConfig, len(cfg.Outputs))
sourceOwners := map[string]string{}
laneOwners := map[string]string{}
for _, rawKey := range rawKeys {
output := cfg.Outputs[rawKey]
key := strings.TrimSpace(rawKey)
if !artifactpolicy.IsConfiguredKey(key) {
return fmt.Errorf("pipeline.notarius.outputs keys must match ^[a-z][a-z0-9_]*$")
}
sourceID := artifactpolicy.ExtractionSourceID(key)
if previousKey, ok := sourceOwners[sourceID]; ok {
return fmt.Errorf("pipeline.notarius.outputs keys %q and %q normalize to duplicate source %q", previousKey, rawKey, sourceID)
}
if owner, ok := reservedSources[sourceID]; ok {
return fmt.Errorf("pipeline.notarius.outputs.%s source %q collides with %s", key, sourceID, owner)
}
sourceOwners[sourceID] = rawKey
output.LaneID = strings.TrimSpace(output.LaneID)
output.MediaType = strings.TrimSpace(output.MediaType)
output.SchemaID = strings.TrimSpace(output.SchemaID)
output.SchemaVersion = strings.TrimSpace(output.SchemaVersion)
output.ModuleKey = strings.TrimSpace(output.ModuleKey)
prefix := "pipeline.notarius.outputs." + key
if output.LaneID == "" {
return fmt.Errorf("%s.lane_id is required", prefix)
}
if previousKey, ok := laneOwners[output.LaneID]; ok {
return fmt.Errorf("%s.lane_id %q duplicates pipeline.notarius.outputs.%s.lane_id", prefix, output.LaneID, previousKey)
}
laneOwners[output.LaneID] = key
if output.MediaType == "" {
return fmt.Errorf("%s.media_type is required", prefix)
}
if output.SchemaID == "" {
return fmt.Errorf("%s.schema_id is required", prefix)
}
if output.SchemaVersion == "" {
return fmt.Errorf("%s.schema_version is required", prefix)
}
normalizedOutputs[key] = output
}
cfg.Binary = strings.TrimSpace(cfg.Binary)
cfg.ConfigPath = filepath.Clean(cfg.ConfigPath)
cfg.PipelineID = strings.TrimSpace(cfg.PipelineID)
cfg.Timeout = strings.TrimSpace(cfg.Timeout)
cfg.WorkingDirectory = filepath.Clean(cfg.WorkingDirectory)
cfg.Outputs = normalizedOutputs
return nil
}
func validateScriptorium(cfg *ScriptoriumConfig, notarius *NotariusConfig) error {
if cfg == nil {
return nil
}
@@ -491,7 +608,7 @@ func validateScriptorium(cfg *ScriptoriumConfig) error {
configuredArtifacts := make(map[string]struct{}, len(cfg.Artifacts))
referencedArtifacts := make(map[string]struct{})
for artifactName := range cfg.Artifacts {
if !scriptoriumArtifactKeyRE.MatchString(strings.TrimSpace(artifactName)) {
if !artifactpolicy.IsConfiguredKey(artifactName) {
return fmt.Errorf("pipeline.scriptorium.artifacts keys must match ^[a-z][a-z0-9_]*$")
}
configuredArtifacts[artifactName] = struct{}{}
@@ -544,7 +661,7 @@ func validateScriptorium(cfg *ScriptoriumConfig) error {
return fmt.Errorf("pipeline.scriptorium.artifacts.%s.inputs.%s.source is required", artifactName, inputName)
}
referencedArtifact, err := validateScriptoriumInputSource(artifactName, inputName, source, configuredArtifacts)
referencedArtifact, err := validateScriptoriumInputSource(artifactName, inputName, source, configuredArtifacts, notariusOutputKeySet(notarius))
if err != nil {
return err
}
@@ -684,9 +801,12 @@ func publishUploadConfiguredForS3(pipeline *PipelineConfig) bool {
var windowsAbsPathRE = regexp.MustCompile(`^[A-Za-z]:[\\/].*`)
var envVarNameRE = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
var scriptoriumArtifactKeyRE = regexp.MustCompile(`^[a-z][a-z0-9_]*$`)
func validateScriptoriumInputSource(artifactName, inputName, source string, configuredArtifacts map[string]struct{}) (string, error) {
func validateScriptoriumInputSource(
artifactName, inputName, source string,
configuredArtifacts map[string]struct{},
extractionOutputs map[string]struct{},
) (string, error) {
trimmedSource := strings.TrimSpace(source)
descriptor, err := artifactpolicy.DescribeScriptoriumInputSource(trimmedSource)
if err != nil {
@@ -705,7 +825,7 @@ func validateScriptoriumInputSource(artifactName, inputName, source string, conf
source,
)
}
if err := artifactpolicy.ValidateInputConfiguredReference(descriptor, configuredArtifacts); err != nil {
if err := artifactpolicy.ValidateInputReference(descriptor, configuredArtifacts, extractionOutputs); err != nil {
var unknownConfigured *artifactpolicy.UnknownConfiguredArtifactError
if errors.As(err, &unknownConfigured) {
return "", fmt.Errorf(
@@ -716,6 +836,16 @@ func validateScriptoriumInputSource(artifactName, inputName, source string, conf
unknownConfigured.ConfiguredKey,
)
}
var unknownExtraction *artifactpolicy.UnknownExtractionArtifactError
if errors.As(err, &unknownExtraction) {
return "", fmt.Errorf(
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q references unknown extraction output %q",
artifactName,
inputName,
source,
unknownExtraction.ConfiguredKey,
)
}
return "", fmt.Errorf(
"pipeline.scriptorium.artifacts.%s.inputs.%s.source %q is unsupported",
artifactName,