Files
narratio/internal/config/storage_publish_test.go

690 lines
19 KiB
Go

package config
import (
"strings"
"testing"
)
func TestStorageS3DefaultsAndValidation(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Pipeline.Storage.S3 == nil {
t.Fatal("storage.s3 should be initialized")
}
if cfg.Pipeline.Storage.S3.RootPrefix != "dnd" {
t.Fatalf("storage.s3.root_prefix = %q, want dnd", cfg.Pipeline.Storage.S3.RootPrefix)
}
if cfg.Pipeline.Storage.S3.AccessKeyIDEnv != DefaultS3AccessKeyIDEnv {
t.Fatalf("storage.s3.access_key_id_env = %q, want %q", cfg.Pipeline.Storage.S3.AccessKeyIDEnv, DefaultS3AccessKeyIDEnv)
}
if cfg.Pipeline.Storage.S3.SecretKeyEnv != DefaultS3SecretAccessKeyEnv {
t.Fatalf("storage.s3.secret_access_key_env = %q, want %q", cfg.Pipeline.Storage.S3.SecretKeyEnv, DefaultS3SecretAccessKeyEnv)
}
if cfg.Pipeline.Storage.S3.ForcePathStyle {
t.Fatalf("storage.s3.force_path_style = true, want false default")
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
}
func TestStorageLegacyTopLevelFieldsFailStrictDecode(t *testing.T) {
tests := []struct {
name string
storageYAML string
wantField string
}{
{
name: "bucket",
storageYAML: `
storage:
bucket: my-dnd-archive
`,
wantField: "bucket",
},
{
name: "prefix",
storageYAML: `
storage:
prefix: dnd
`,
wantField: "prefix",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + tt.storageYAML
pipelinePath, _ := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
_, err := LoadPipeline(pipelinePath)
if err == nil {
t.Fatal("LoadPipeline() error = nil, want strict decode error")
}
if !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("LoadPipeline() error = %v, want strict decode failed", err)
}
if !strings.Contains(err.Error(), tt.wantField) {
t.Fatalf("LoadPipeline() error = %v, want field %q", err, tt.wantField)
}
})
}
}
func TestStorageS3CredentialEnvNamesLoadAndValidate(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
access_key_id_env: CUSTOM_KEY_ID
secret_access_key_env: CUSTOM_SECRET
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Pipeline.Storage.S3.AccessKeyIDEnv != "CUSTOM_KEY_ID" {
t.Fatalf("storage.s3.access_key_id_env = %q, want CUSTOM_KEY_ID", cfg.Pipeline.Storage.S3.AccessKeyIDEnv)
}
if cfg.Pipeline.Storage.S3.SecretKeyEnv != "CUSTOM_SECRET" {
t.Fatalf("storage.s3.secret_access_key_env = %q, want CUSTOM_SECRET", cfg.Pipeline.Storage.S3.SecretKeyEnv)
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
}
func TestStorageS3CredentialEnvValidation(t *testing.T) {
tests := []struct {
name string
pipelineYML string
wantErr string
}{
{
name: "invalid access key env name",
pipelineYML: testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
access_key_id_env: "123BAD"
`,
wantErr: "pipeline.storage.s3.access_key_id_env must be a valid environment variable name",
},
{
name: "invalid secret key env name",
pipelineYML: testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
secret_access_key_env: "bad-name"
`,
wantErr: "pipeline.storage.s3.secret_access_key_env must be a valid environment variable name",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, tt.pipelineYML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
}
})
}
}
func TestSpoolAndPublishDefaults(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, testPipelineBaseYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Pipeline.Spool.Root != "/var/spool/narratio" {
t.Fatalf("spool.root = %q, want /var/spool/narratio", cfg.Pipeline.Spool.Root)
}
if cfg.Pipeline.Spool.DeleteAudioAfterPublish {
t.Fatalf("spool.delete_audio_after_publish = true, want false")
}
if cfg.Pipeline.Workspace.CleanupAfterPublish {
t.Fatalf("workspace.cleanup_after_publish = true, want false")
}
if cfg.Pipeline.Publish == nil {
t.Fatal("publish should be initialized by defaults")
}
if cfg.Pipeline.Publish.Enabled == nil || !*cfg.Pipeline.Publish.Enabled {
t.Fatalf("publish.enabled = %#v, want true", cfg.Pipeline.Publish.Enabled)
}
if cfg.Pipeline.Publish.UploadRun == nil || !*cfg.Pipeline.Publish.UploadRun {
t.Fatalf("publish.upload_run = %#v, want true", cfg.Pipeline.Publish.UploadRun)
}
if len(cfg.Pipeline.Publish.Outputs) != 3 {
t.Fatalf("publish.outputs len = %d, want 3 defaults", len(cfg.Pipeline.Publish.Outputs))
}
wantBySource := map[string]string{
"narratio.transcript.final_trimmed": "transcripts/final.trimmed.json",
"narratio.transcript.final_markdown": "transcripts/final.md",
"narratio.transcript.final_trimmed_markdown": "transcripts/final.trimmed.md",
}
for i, item := range cfg.Pipeline.Publish.Outputs {
if item.Required == nil || !*item.Required {
t.Fatalf("publish.outputs[%d].required = %#v, want true", i, item.Required)
}
wantDest, ok := wantBySource[item.Source]
if !ok {
t.Fatalf("publish.outputs[%d].source = %q, want known default source", i, item.Source)
}
if item.Dest != wantDest {
t.Fatalf("publish.outputs[%d].dest = %q, want %q", i, item.Dest, wantDest)
}
delete(wantBySource, item.Source)
}
if len(wantBySource) != 0 {
t.Fatalf("missing default publish outputs for sources: %#v", wantBySource)
}
}
func TestPublishOutputValidation(t *testing.T) {
tests := []struct {
name string
ruleYML string
wantErr string
}{
{
name: "absolute dest path rejected",
ruleYML: `publish:
outputs:
- source: "narratio.transcript.final_trimmed"
dest: "/transcripts/final.trimmed.json"
`,
wantErr: "must be a relative path",
},
{
name: "traversal dest path rejected",
ruleYML: `publish:
outputs:
- source: "narratio.transcript.final_trimmed"
dest: "../trimmed.json"
`,
wantErr: "must not contain path traversal",
},
{
name: "invalid source rejected",
ruleYML: `publish:
outputs:
- source: "narratio.unknown"
dest: "transcripts/final.trimmed.json"
`,
wantErr: "source \"narratio.unknown\" is unsupported",
},
{
name: "duplicate destination rejected",
ruleYML: `publish:
outputs:
- source: "narratio.transcript.final_trimmed"
dest: "artifacts/shared.md"
- source: "narratio.transcript.final"
dest: "artifacts/shared.md"
`,
wantErr: "duplicates another publish output destination",
},
{
name: "configured source requires configured artifact key",
ruleYML: `publish:
outputs:
- source: "narratio.artifact.session_recap"
dest: "artifacts/session_recap.md"
`,
wantErr: "configured artifact \"session_recap\" is not defined in pipeline.scriptorium.artifacts",
},
{
name: "configured source without output path fails when dest omitted",
ruleYML: `scriptorium:
artifacts:
session_recap:
enabled: false
publish:
outputs:
- source: "narratio.artifact.session_recap"
`,
wantErr: "destination cannot be derived",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + "\n" + tt.ruleYML
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
}
})
}
}
func TestPublishOutputLegacyTranscriptSourcesRejected(t *testing.T) {
legacyTranscriptSources := []string{
"narratio.transcript." + "merged",
"narratio.transcript." + "full",
"narratio.transcript." + "trimmed",
}
for _, source := range legacyTranscriptSources {
t.Run(source, func(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
publish:
outputs:
- source: ` + source + `
dest: transcripts/final.trimmed.json
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
wantErr := `source "` + source + `" is unsupported`
if err == nil || !strings.Contains(err.Error(), wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, wantErr)
}
})
}
}
func TestPublishOutputDerivesDestinationWhenOmitted(t *testing.T) {
tests := []struct {
name string
pipelineYML string
wantDest string
}{
{
name: "built in source derives canonical destination",
pipelineYML: testPipelineBaseYAML + `
publish:
outputs:
- source: narratio.transcript.final
`,
wantDest: "transcripts/final.json",
},
{
name: "configured source derives configured output path",
pipelineYML: testPipelineBaseYAML + `
scriptorium:
artifacts:
session_recap:
enabled: true
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
publish:
outputs:
- source: narratio.artifact.session_recap
`,
wantDest: "artifacts/session_recap.md",
},
{
name: "markdown built in derives canonical destination",
pipelineYML: testPipelineBaseYAML + `
publish:
outputs:
- source: narratio.transcript.final_markdown
`,
wantDest: "transcripts/final.md",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, tt.pipelineYML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
if len(cfg.Pipeline.Publish.Outputs) != 1 {
t.Fatalf("publish.outputs len = %d, want 1", len(cfg.Pipeline.Publish.Outputs))
}
if cfg.Pipeline.Publish.Outputs[0].Dest != tt.wantDest {
t.Fatalf("publish.outputs[0].dest = %q, want %q", cfg.Pipeline.Publish.Outputs[0].Dest, tt.wantDest)
}
})
}
}
func TestPublishLockValidation(t *testing.T) {
tests := []struct {
name string
pipelineYML string
wantErr string
}{
{
name: "valid built in source",
pipelineYML: testPipelineBaseYAML + `
publish:
locks:
- source: narratio.transcript.final_trimmed
reason: reviewed transcript
`,
},
{
name: "valid configured source",
pipelineYML: testPipelineBaseYAML + `
scriptorium:
artifacts:
session_recap:
enabled: true
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
publish:
locks:
- source: narratio.artifact.session_recap
`,
},
{
name: "missing source rejected",
pipelineYML: testPipelineBaseYAML + `
publish:
locks:
- reason: no source
`,
wantErr: "pipeline.publish.locks[0].source is required",
},
{
name: "invalid source rejected",
pipelineYML: testPipelineBaseYAML + `
publish:
locks:
- source: narratio.unknown
`,
wantErr: "pipeline.publish.locks[0].source \"narratio.unknown\" is unsupported",
},
{
name: "duplicate source rejected",
pipelineYML: testPipelineBaseYAML + `
publish:
locks:
- source: narratio.transcript.final_trimmed
- source: " narratio.transcript.final_trimmed "
`,
wantErr: "duplicates another publish lock source",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, tt.pipelineYML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if tt.wantErr == "" {
if err != nil {
t.Fatalf("Validate() error = %v", err)
}
return
}
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
}
})
}
}
func TestPublishLockLegacyTranscriptSourcesRejected(t *testing.T) {
legacyTranscriptSources := []string{
"narratio.transcript." + "merged",
"narratio.transcript." + "full",
"narratio.transcript." + "trimmed",
}
for _, source := range legacyTranscriptSources {
t.Run(source, func(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
publish:
locks:
- source: ` + source + `
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
wantErr := `source "` + source + `" is unsupported`
if err == nil || !strings.Contains(err.Error(), wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, wantErr)
}
})
}
}
func TestPublishLockUnknownFieldFailsStrictDecode(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
publish:
locks:
- source: narratio.transcript.final_trimmed
dest: transcripts/final.trimmed.json
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
_, err := Load(pipelinePath, sessionPath)
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("Load() error = %v, want strict decode failed", err)
}
}
func TestPublishLegacyFromToFailsStrictDecode(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
publish:
outputs:
- from: transcripts/final.trimmed.json
to: transcripts/final.trimmed.json
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
_, err := Load(pipelinePath, sessionPath)
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("Load() error = %v, want strict decode failed", err)
}
}
func TestPublishLockStoreBytesStrictDecodeAndValidation(t *testing.T) {
store, err := LoadPublishLockStoreBytes("locks.yml", []byte(`locks:
- source: narratio.transcript.final_trimmed
reason: reviewed
`), nil)
if err != nil {
t.Fatalf("LoadPublishLockStoreBytes() error = %v", err)
}
if len(store.Locks) != 1 || store.Locks[0].Source != "narratio.transcript.final_trimmed" || store.Locks[0].Reason != "reviewed" {
t.Fatalf("locks = %#v", store.Locks)
}
_, err = LoadPublishLockStoreBytes("locks.yml", []byte(`locks:
- source: narratio.transcript.final_trimmed
dest: transcripts/final.trimmed.json
`), nil)
if err == nil || !strings.Contains(err.Error(), "strict decode failed") {
t.Fatalf("unknown field error = %v, want strict decode failed", err)
}
_, err = LoadPublishLockStoreBytes("locks.yml", []byte(`locks:
- source: narratio.transcript.final_trimmed
- source: narratio.transcript.final_trimmed
`), nil)
if err == nil || !strings.Contains(err.Error(), "duplicates another publish lock source") {
t.Fatalf("duplicate error = %v", err)
}
}
func TestMergePublishLockRulesStaticWins(t *testing.T) {
merged := MergePublishLockRules(
[]PublishLockRule{{Source: "narratio.transcript.final_trimmed", Reason: "static"}},
[]PublishLockRule{
{Source: "narratio.transcript.final_trimmed", Reason: "remote"},
{Source: "narratio.transcript.final", Reason: "remote full"},
},
)
if len(merged) != 2 {
t.Fatalf("merged len = %d, want 2: %#v", len(merged), merged)
}
if merged[0].Source != "narratio.transcript.final_trimmed" || merged[0].Reason != "static" {
t.Fatalf("merged[0] = %#v, want static lock", merged[0])
}
if merged[1].Source != "narratio.transcript.final" {
t.Fatalf("merged[1] = %#v, want remote full lock", merged[1])
}
}
func TestSessionAudioS3Validation(t *testing.T) {
tests := []struct {
name string
sessionYAML string
wantErr string
}{
{
name: "valid audio_s3 prefix",
sessionYAML: `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_s3:
prefix: audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
},
{
name: "invalid audio_s3 absolute prefix",
sessionYAML: `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_s3:
prefix: /audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantErr: "session.inputs.audio_s3.prefix must be a relative path",
},
{
name: "invalid audio_s3 traversal prefix",
sessionYAML: `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_s3:
prefix: ../audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantErr: "session.inputs.audio_s3.prefix must not contain path traversal",
},
{
name: "local and s3 audio conflict",
sessionYAML: `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_dir: ./audio
audio_s3:
prefix: audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`,
wantErr: "mutually exclusive",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
storage:
backend: s3
s3:
bucket: my-dnd-archive
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, tt.sessionYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if tt.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, tt.wantErr)
}
return
}
if err != nil {
t.Fatalf("Validate() error = %v", err)
}
})
}
}
func TestStorageS3BucketRequiredWhenS3DependentFeatureEnabled(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
storage:
backend: s3
`
sessionYAML := `session_id: 2026-05-03
campaign: forsaken
inputs:
audio_s3:
prefix: audio/
speakers_file: ./speakers.yml
autocorrect_file: ./autocorrect.yml
glossary_file: ./glossary.yml
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, sessionYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if err == nil || !strings.Contains(err.Error(), "pipeline.storage.s3.bucket is required") {
t.Fatalf("Validate() error = %v, want bucket requirement", err)
}
}
func TestLocalAudioConfigStillValid(t *testing.T) {
pipelinePath, sessionPath := writeConfigFiles(t, testPipelineBaseYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if err := Validate(cfg); err != nil {
t.Fatalf("Validate() error = %v", err)
}
}