Add archive storage path configuration

This commit is contained in:
2026-05-16 14:11:59 +00:00
parent 58c6ab2d54
commit 0454296c81
23 changed files with 950 additions and 22 deletions

View File

@@ -3,6 +3,8 @@ package config
import (
"fmt"
"net/url"
"path/filepath"
"regexp"
"strings"
"time"
)
@@ -25,6 +27,9 @@ func Validate(cfg *Config) error {
if err := validateSession(cfg.Session); err != nil {
return fmt.Errorf("session config %q invalid: %w", shortName(cfg.SessionPath, "session.yml"), err)
}
if err := validateCrossConfig(cfg.Pipeline, cfg.Session); err != nil {
return fmt.Errorf("pipeline/session config invalid: %w", err)
}
return nil
}
@@ -36,6 +41,15 @@ func validatePipeline(cfg *PipelineConfig) error {
if err := validateSecrets(cfg.Secrets); err != nil {
return err
}
if err := validateStorage(cfg.Storage); err != nil {
return err
}
if err := validateSpool(cfg.Spool); err != nil {
return err
}
if err := validateArchive(cfg.Archive); err != nil {
return err
}
if err := validateWhisperX(cfg.WhisperX); err != nil {
return err
}
@@ -64,6 +78,48 @@ func validatePipeline(cfg *PipelineConfig) error {
return nil
}
func validateStorage(cfg StorageConfig) error {
if cfg.S3 == nil {
return nil
}
if strings.TrimSpace(cfg.S3.RootPrefix) == "" {
return fmt.Errorf("pipeline.storage.s3.root_prefix must be non-empty")
}
if err := validateRelativeSafePath("pipeline.storage.s3.root_prefix", cfg.S3.RootPrefix); err != nil {
return err
}
if cfg.S3.Endpoint != "" && strings.TrimSpace(cfg.S3.Endpoint) == "" {
return fmt.Errorf("pipeline.storage.s3.endpoint must be non-empty when provided")
}
return nil
}
func validateSpool(cfg SpoolConfig) error {
return nil
}
func validateArchive(cfg *ArchiveConfig) error {
if cfg == nil {
return nil
}
for i, item := range cfg.PromoteArtifacts {
prefix := fmt.Sprintf("pipeline.archive.promote_artifacts[%d]", i)
if strings.TrimSpace(item.From) == "" {
return fmt.Errorf("%s.from is required", prefix)
}
if strings.TrimSpace(item.To) == "" {
return fmt.Errorf("%s.to is required", prefix)
}
if err := validateRelativeSafePath(prefix+".from", item.From); err != nil {
return err
}
if err := validateRelativeSafePath(prefix+".to", item.To); err != nil {
return err
}
}
return nil
}
func validateSecrets(cfg *SecretsConfig) error {
if cfg == nil {
return nil
@@ -307,6 +363,9 @@ func validateSession(cfg *SessionConfig) error {
if strings.TrimSpace(cfg.SessionID) == "" {
return fmt.Errorf("session.session_id is required")
}
if strings.TrimSpace(cfg.Campaign) == "" {
return fmt.Errorf("session.campaign is required")
}
if strings.TrimSpace(cfg.Inputs.SpeakersFile) == "" {
return fmt.Errorf("session.inputs.speakers_file is required")
@@ -320,13 +379,79 @@ func validateSession(cfg *SessionConfig) error {
hasAudioDir := strings.TrimSpace(cfg.Inputs.AudioDir) != ""
hasAudioFiles := len(cfg.Inputs.AudioFiles) > 0
if !hasAudioDir && !hasAudioFiles {
return fmt.Errorf("session.inputs requires audio_dir or at least one audio_files entry")
hasAudioS3 := cfg.Inputs.AudioS3 != nil
if hasAudioS3 {
if strings.TrimSpace(cfg.Inputs.AudioS3.Prefix) == "" {
return fmt.Errorf("session.inputs.audio_s3.prefix is required when session.inputs.audio_s3 is configured")
}
if err := validateRelativeSafePath("session.inputs.audio_s3.prefix", cfg.Inputs.AudioS3.Prefix); err != nil {
return err
}
}
if hasAudioS3 && (hasAudioDir || hasAudioFiles) {
return fmt.Errorf("session.inputs.audio_dir/audio_files and session.inputs.audio_s3 are mutually exclusive")
}
if !hasAudioDir && !hasAudioFiles && !hasAudioS3 {
return fmt.Errorf("session.inputs requires audio_dir, at least one audio_files entry, or audio_s3")
}
return nil
}
func validateCrossConfig(pipeline *PipelineConfig, session *SessionConfig) error {
if pipeline == nil || session == nil {
return nil
}
if pipeline.Storage.S3 == nil {
return nil
}
audioS3Enabled := session.Inputs.AudioS3 != nil
archiveUploadEnabled := archiveUploadConfiguredForS3(pipeline)
if (audioS3Enabled || archiveUploadEnabled) && strings.TrimSpace(pipeline.Storage.S3.Bucket) == "" {
return fmt.Errorf("pipeline.storage.s3.bucket is required when S3 session audio or archive upload is enabled")
}
return nil
}
func archiveUploadConfiguredForS3(pipeline *PipelineConfig) bool {
if pipeline == nil || pipeline.Archive == nil {
return false
}
if !strings.EqualFold(strings.TrimSpace(pipeline.Storage.Backend), "s3") {
return false
}
enabled := true
if pipeline.Archive.Enabled != nil {
enabled = *pipeline.Archive.Enabled
}
upload := true
if pipeline.Archive.UploadRun != nil {
upload = *pipeline.Archive.UploadRun
}
return enabled && upload
}
var windowsAbsPathRE = regexp.MustCompile(`^[A-Za-z]:[\\/].*`)
func validateRelativeSafePath(fieldName, value string) error {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return fmt.Errorf("%s must be non-empty", fieldName)
}
if filepath.IsAbs(trimmed) || strings.HasPrefix(trimmed, "/") || strings.HasPrefix(trimmed, "\\") || windowsAbsPathRE.MatchString(trimmed) {
return fmt.Errorf("%s must be a relative path", fieldName)
}
normalized := strings.ReplaceAll(trimmed, "\\", "/")
for _, segment := range strings.Split(normalized, "/") {
if segment == ".." {
return fmt.Errorf("%s must not contain path traversal", fieldName)
}
}
return nil
}
func validateDuration(fieldName, value string) error {
trimmed := strings.TrimSpace(value)
if trimmed == "" {