Rename archive config and stage contract to publish
This commit is contained in:
@@ -21,7 +21,7 @@ type PipelineConfig struct {
|
||||
Storage StorageConfig `yaml:"storage"`
|
||||
Spool SpoolConfig `yaml:"spool"`
|
||||
Cache CacheConfig `yaml:"cache"`
|
||||
Archive *ArchiveConfig `yaml:"archive"`
|
||||
Publish *ArchiveConfig `yaml:"publish"`
|
||||
Secrets *SecretsConfig `yaml:"secrets"`
|
||||
WhisperX WhisperXConfig `yaml:"whisperx"`
|
||||
Seriatim SeriatimConfig `yaml:"seriatim"`
|
||||
@@ -65,7 +65,7 @@ type SessionConfig struct {
|
||||
// WorkspaceConfig configures local workspace behavior.
|
||||
type WorkspaceConfig struct {
|
||||
Root string `yaml:"root"`
|
||||
CleanupAfterArchive bool `yaml:"cleanup_after_archive"`
|
||||
CleanupAfterPublish bool `yaml:"cleanup_after_publish"`
|
||||
}
|
||||
|
||||
// SecretsConfig configures optional local filesystem secret loading.
|
||||
@@ -93,7 +93,7 @@ type StorageS3Config struct {
|
||||
// SpoolConfig configures local spool storage for staged data.
|
||||
type SpoolConfig struct {
|
||||
Root string `yaml:"root"`
|
||||
DeleteAudioAfterArchive bool `yaml:"delete_audio_after_archive"`
|
||||
DeleteAudioAfterPublish bool `yaml:"delete_audio_after_publish"`
|
||||
}
|
||||
|
||||
// CacheConfig configures durable local caches for reusable remote inputs.
|
||||
@@ -106,7 +106,7 @@ type CacheConfig struct {
|
||||
type ArchiveConfig struct {
|
||||
Enabled *bool `yaml:"enabled"`
|
||||
UploadRun *bool `yaml:"upload_run"`
|
||||
PromoteArtifacts []ArchivePromotionRule `yaml:"promote_artifacts"`
|
||||
PromoteArtifacts []ArchivePromotionRule `yaml:"outputs"`
|
||||
Locks []ArchiveLockRule `yaml:"locks"`
|
||||
}
|
||||
|
||||
|
||||
@@ -328,7 +328,7 @@ func applyPipelineDefaults(cfg *PipelineConfig) {
|
||||
applyStorageDefaults(&cfg.Storage)
|
||||
applySpoolDefaults(&cfg.Spool)
|
||||
applyCacheDefaults(&cfg.Cache)
|
||||
applyArchiveDefaults(&cfg.Archive)
|
||||
applyArchiveDefaults(&cfg.Publish)
|
||||
applyWhisperXDefaults(&cfg.WhisperX)
|
||||
applySeriatimDefaults(&cfg.Seriatim)
|
||||
applyAuditaDefaults(&cfg.Audita)
|
||||
|
||||
@@ -164,33 +164,33 @@ func TestSpoolAndArchiveDefaults(t *testing.T) {
|
||||
if cfg.Pipeline.Spool.Root != "/var/spool/narratio" {
|
||||
t.Fatalf("spool.root = %q, want /var/spool/narratio", cfg.Pipeline.Spool.Root)
|
||||
}
|
||||
if cfg.Pipeline.Spool.DeleteAudioAfterArchive {
|
||||
t.Fatalf("spool.delete_audio_after_archive = true, want false")
|
||||
if cfg.Pipeline.Spool.DeleteAudioAfterPublish {
|
||||
t.Fatalf("spool.delete_audio_after_publish = true, want false")
|
||||
}
|
||||
if cfg.Pipeline.Workspace.CleanupAfterArchive {
|
||||
t.Fatalf("workspace.cleanup_after_archive = true, want false")
|
||||
if cfg.Pipeline.Workspace.CleanupAfterPublish {
|
||||
t.Fatalf("workspace.cleanup_after_publish = true, want false")
|
||||
}
|
||||
if cfg.Pipeline.Archive == nil {
|
||||
if cfg.Pipeline.Publish == nil {
|
||||
t.Fatal("archive should be initialized by defaults")
|
||||
}
|
||||
if cfg.Pipeline.Archive.Enabled == nil || !*cfg.Pipeline.Archive.Enabled {
|
||||
t.Fatalf("archive.enabled = %#v, want true", cfg.Pipeline.Archive.Enabled)
|
||||
if cfg.Pipeline.Publish.Enabled == nil || !*cfg.Pipeline.Publish.Enabled {
|
||||
t.Fatalf("archive.enabled = %#v, want true", cfg.Pipeline.Publish.Enabled)
|
||||
}
|
||||
if cfg.Pipeline.Archive.UploadRun == nil || !*cfg.Pipeline.Archive.UploadRun {
|
||||
t.Fatalf("archive.upload_run = %#v, want true", cfg.Pipeline.Archive.UploadRun)
|
||||
if cfg.Pipeline.Publish.UploadRun == nil || !*cfg.Pipeline.Publish.UploadRun {
|
||||
t.Fatalf("archive.upload_run = %#v, want true", cfg.Pipeline.Publish.UploadRun)
|
||||
}
|
||||
if len(cfg.Pipeline.Archive.PromoteArtifacts) != 1 {
|
||||
t.Fatalf("archive.promote_artifacts len = %d, want 1 default", len(cfg.Pipeline.Archive.PromoteArtifacts))
|
||||
if len(cfg.Pipeline.Publish.PromoteArtifacts) != 1 {
|
||||
t.Fatalf("publish.outputs len = %d, want 1 default", len(cfg.Pipeline.Publish.PromoteArtifacts))
|
||||
}
|
||||
item := cfg.Pipeline.Archive.PromoteArtifacts[0]
|
||||
item := cfg.Pipeline.Publish.PromoteArtifacts[0]
|
||||
if item.Required == nil || !*item.Required {
|
||||
t.Fatalf("archive.promote_artifacts[0].required = %#v, want true", item.Required)
|
||||
t.Fatalf("publish.outputs[0].required = %#v, want true", item.Required)
|
||||
}
|
||||
if item.Source != "narratio.transcript.final_trimmed" {
|
||||
t.Fatalf("archive.promote_artifacts[0].source = %q, want narratio.transcript.final_trimmed", item.Source)
|
||||
t.Fatalf("publish.outputs[0].source = %q, want narratio.transcript.final_trimmed", item.Source)
|
||||
}
|
||||
if item.Dest != "transcripts/final.trimmed.json" {
|
||||
t.Fatalf("archive.promote_artifacts[0].dest = %q, want transcripts/final.trimmed.json", item.Dest)
|
||||
t.Fatalf("publish.outputs[0].dest = %q, want transcripts/final.trimmed.json", item.Dest)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -202,8 +202,8 @@ func TestArchivePromotionValidation(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
name: "absolute dest path rejected",
|
||||
ruleYML: `archive:
|
||||
promote_artifacts:
|
||||
ruleYML: `publish:
|
||||
outputs:
|
||||
- source: "narratio.transcript.final_trimmed"
|
||||
dest: "/transcripts/final.trimmed.json"
|
||||
`,
|
||||
@@ -211,8 +211,8 @@ func TestArchivePromotionValidation(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "traversal dest path rejected",
|
||||
ruleYML: `archive:
|
||||
promote_artifacts:
|
||||
ruleYML: `publish:
|
||||
outputs:
|
||||
- source: "narratio.transcript.final_trimmed"
|
||||
dest: "../trimmed.json"
|
||||
`,
|
||||
@@ -220,8 +220,8 @@ func TestArchivePromotionValidation(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "invalid source rejected",
|
||||
ruleYML: `archive:
|
||||
promote_artifacts:
|
||||
ruleYML: `publish:
|
||||
outputs:
|
||||
- source: "narratio.unknown"
|
||||
dest: "transcripts/final.trimmed.json"
|
||||
`,
|
||||
@@ -229,8 +229,8 @@ func TestArchivePromotionValidation(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "duplicate destination rejected",
|
||||
ruleYML: `archive:
|
||||
promote_artifacts:
|
||||
ruleYML: `publish:
|
||||
outputs:
|
||||
- source: "narratio.transcript.final_trimmed"
|
||||
dest: "artifacts/shared.md"
|
||||
- source: "narratio.transcript.final"
|
||||
@@ -240,8 +240,8 @@ func TestArchivePromotionValidation(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "configured source requires configured artifact key",
|
||||
ruleYML: `archive:
|
||||
promote_artifacts:
|
||||
ruleYML: `publish:
|
||||
outputs:
|
||||
- source: "narratio.artifact.session_recap"
|
||||
dest: "artifacts/session_recap.md"
|
||||
`,
|
||||
@@ -253,8 +253,8 @@ func TestArchivePromotionValidation(t *testing.T) {
|
||||
artifacts:
|
||||
session_recap:
|
||||
enabled: false
|
||||
archive:
|
||||
promote_artifacts:
|
||||
publish:
|
||||
outputs:
|
||||
- source: "narratio.artifact.session_recap"
|
||||
`,
|
||||
wantErr: "destination cannot be derived",
|
||||
@@ -288,8 +288,8 @@ func TestArchivePromotionLegacyTranscriptSourcesRejected(t *testing.T) {
|
||||
for _, source := range legacyTranscriptSources {
|
||||
t.Run(source, func(t *testing.T) {
|
||||
pipelineYAML := testPipelineBaseYAML + `
|
||||
archive:
|
||||
promote_artifacts:
|
||||
publish:
|
||||
outputs:
|
||||
- source: ` + source + `
|
||||
dest: transcripts/final.trimmed.json
|
||||
`
|
||||
@@ -316,8 +316,8 @@ func TestArchivePromotionDerivesDestinationWhenOmitted(t *testing.T) {
|
||||
{
|
||||
name: "built in source derives canonical destination",
|
||||
pipelineYML: testPipelineBaseYAML + `
|
||||
archive:
|
||||
promote_artifacts:
|
||||
publish:
|
||||
outputs:
|
||||
- source: narratio.transcript.final
|
||||
`,
|
||||
wantDest: "transcripts/final.json",
|
||||
@@ -331,8 +331,8 @@ scriptorium:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
output_path: artifacts/session_recap.md
|
||||
archive:
|
||||
promote_artifacts:
|
||||
publish:
|
||||
outputs:
|
||||
- source: narratio.artifact.session_recap
|
||||
`,
|
||||
wantDest: "artifacts/session_recap.md",
|
||||
@@ -349,11 +349,11 @@ archive:
|
||||
if err := Validate(cfg); err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
if len(cfg.Pipeline.Archive.PromoteArtifacts) != 1 {
|
||||
t.Fatalf("archive.promote_artifacts len = %d, want 1", len(cfg.Pipeline.Archive.PromoteArtifacts))
|
||||
if len(cfg.Pipeline.Publish.PromoteArtifacts) != 1 {
|
||||
t.Fatalf("publish.outputs len = %d, want 1", len(cfg.Pipeline.Publish.PromoteArtifacts))
|
||||
}
|
||||
if cfg.Pipeline.Archive.PromoteArtifacts[0].Dest != tt.wantDest {
|
||||
t.Fatalf("archive.promote_artifacts[0].dest = %q, want %q", cfg.Pipeline.Archive.PromoteArtifacts[0].Dest, tt.wantDest)
|
||||
if cfg.Pipeline.Publish.PromoteArtifacts[0].Dest != tt.wantDest {
|
||||
t.Fatalf("publish.outputs[0].dest = %q, want %q", cfg.Pipeline.Publish.PromoteArtifacts[0].Dest, tt.wantDest)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -368,7 +368,7 @@ func TestArchiveLockValidation(t *testing.T) {
|
||||
{
|
||||
name: "valid built in source",
|
||||
pipelineYML: testPipelineBaseYAML + `
|
||||
archive:
|
||||
publish:
|
||||
locks:
|
||||
- source: narratio.transcript.final_trimmed
|
||||
reason: reviewed transcript
|
||||
@@ -383,7 +383,7 @@ scriptorium:
|
||||
enabled: true
|
||||
prompt_id: dnd.session_recap
|
||||
output_path: artifacts/session_recap.md
|
||||
archive:
|
||||
publish:
|
||||
locks:
|
||||
- source: narratio.artifact.session_recap
|
||||
`,
|
||||
@@ -391,25 +391,25 @@ archive:
|
||||
{
|
||||
name: "missing source rejected",
|
||||
pipelineYML: testPipelineBaseYAML + `
|
||||
archive:
|
||||
publish:
|
||||
locks:
|
||||
- reason: no source
|
||||
`,
|
||||
wantErr: "pipeline.archive.locks[0].source is required",
|
||||
wantErr: "pipeline.publish.locks[0].source is required",
|
||||
},
|
||||
{
|
||||
name: "invalid source rejected",
|
||||
pipelineYML: testPipelineBaseYAML + `
|
||||
archive:
|
||||
publish:
|
||||
locks:
|
||||
- source: narratio.unknown
|
||||
`,
|
||||
wantErr: "pipeline.archive.locks[0].source \"narratio.unknown\" is unsupported",
|
||||
wantErr: "pipeline.publish.locks[0].source \"narratio.unknown\" is unsupported",
|
||||
},
|
||||
{
|
||||
name: "duplicate source rejected",
|
||||
pipelineYML: testPipelineBaseYAML + `
|
||||
archive:
|
||||
publish:
|
||||
locks:
|
||||
- source: narratio.transcript.final_trimmed
|
||||
- source: " narratio.transcript.final_trimmed "
|
||||
@@ -449,7 +449,7 @@ func TestArchiveLockLegacyTranscriptSourcesRejected(t *testing.T) {
|
||||
for _, source := range legacyTranscriptSources {
|
||||
t.Run(source, func(t *testing.T) {
|
||||
pipelineYAML := testPipelineBaseYAML + `
|
||||
archive:
|
||||
publish:
|
||||
locks:
|
||||
- source: ` + source + `
|
||||
`
|
||||
@@ -469,7 +469,7 @@ archive:
|
||||
|
||||
func TestArchiveLockUnknownFieldFailsStrictDecode(t *testing.T) {
|
||||
pipelineYAML := testPipelineBaseYAML + `
|
||||
archive:
|
||||
publish:
|
||||
locks:
|
||||
- source: narratio.transcript.final_trimmed
|
||||
dest: transcripts/final.trimmed.json
|
||||
@@ -483,8 +483,8 @@ archive:
|
||||
|
||||
func TestArchiveLegacyFromToFailsStrictDecode(t *testing.T) {
|
||||
pipelineYAML := testPipelineBaseYAML + `
|
||||
archive:
|
||||
promote_artifacts:
|
||||
publish:
|
||||
outputs:
|
||||
- from: transcripts/final.trimmed.json
|
||||
to: transcripts/final.trimmed.json
|
||||
`
|
||||
|
||||
@@ -68,7 +68,7 @@ func validatePipeline(cfg *PipelineConfig) error {
|
||||
if err := validateCache(cfg.Cache); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateArchive(cfg.Archive, cfg.Scriptorium); err != nil {
|
||||
if err := validateArchive(cfg.Publish, cfg.Scriptorium); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateWhisperX(cfg.WhisperX); err != nil {
|
||||
@@ -135,7 +135,7 @@ func validateArchive(cfg *ArchiveConfig, scriptorium *ScriptoriumConfig) error {
|
||||
}
|
||||
seenDest := map[string]struct{}{}
|
||||
for i, item := range cfg.PromoteArtifacts {
|
||||
prefix := fmt.Sprintf("pipeline.archive.promote_artifacts[%d]", i)
|
||||
prefix := fmt.Sprintf("pipeline.publish.outputs[%d]", i)
|
||||
source := strings.TrimSpace(item.Source)
|
||||
if source == "" {
|
||||
return fmt.Errorf("%s.source is required", prefix)
|
||||
@@ -161,7 +161,7 @@ func validateArchive(cfg *ArchiveConfig, scriptorium *ScriptoriumConfig) error {
|
||||
}
|
||||
seenDest[normalizedDest] = struct{}{}
|
||||
}
|
||||
locks, err := ValidateArchiveLockRules(cfg.Locks, scriptorium, "pipeline.archive.locks")
|
||||
locks, err := ValidateArchiveLockRules(cfg.Locks, scriptorium, "pipeline.publish.locks")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -174,7 +174,7 @@ func ValidateArchiveLockRules(locks []ArchiveLockRule, scriptorium *ScriptoriumC
|
||||
seenLocks := map[string]struct{}{}
|
||||
out := make([]ArchiveLockRule, 0, len(locks))
|
||||
if strings.TrimSpace(label) == "" {
|
||||
label = "archive.locks"
|
||||
label = "publish.locks"
|
||||
}
|
||||
for i, item := range locks {
|
||||
prefix := fmt.Sprintf("%s[%d]", label, i)
|
||||
@@ -637,19 +637,19 @@ func validateCrossConfig(pipeline *PipelineConfig, session *SessionConfig) error
|
||||
}
|
||||
|
||||
func archiveUploadConfiguredForS3(pipeline *PipelineConfig) bool {
|
||||
if pipeline == nil || pipeline.Archive == nil {
|
||||
if pipeline == nil || pipeline.Publish == nil {
|
||||
return false
|
||||
}
|
||||
if !strings.EqualFold(strings.TrimSpace(pipeline.Storage.Backend), "s3") {
|
||||
return false
|
||||
}
|
||||
enabled := true
|
||||
if pipeline.Archive.Enabled != nil {
|
||||
enabled = *pipeline.Archive.Enabled
|
||||
if pipeline.Publish.Enabled != nil {
|
||||
enabled = *pipeline.Publish.Enabled
|
||||
}
|
||||
upload := true
|
||||
if pipeline.Archive.UploadRun != nil {
|
||||
upload = *pipeline.Archive.UploadRun
|
||||
if pipeline.Publish.UploadRun != nil {
|
||||
upload = *pipeline.Publish.UploadRun
|
||||
}
|
||||
return enabled && upload
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user