eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 27 Wiki Activity

Complete configuration diagnostics workstream

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-05 00:42:47 +00:00
parent 979d8049b1
commit 0d4d71b9dc
6 changed files with 133 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
internal/config/notarius_test.go
View File

@@ -192,20 +192,22 @@ func TestNotariusReferenceValidationRejectsInvalidBindings(t *testing.T) {
tests := []struct {
name string
references map[string]string
wantErr string
wantField string
wantReason string
}{
{name: "empty selector", references: map[string]string{" ": "narratio.input.party"}, wantErr: "selector is required"},
{name: "equals in selector", references: map[string]string{"party=x": "narratio.input.party"}, wantErr: "must not contain"},
{name: "invalid stage", references: map[string]string{"lane.prepare.party": "narratio.input.party"}, wantErr: "middle component"},
{name: "empty source", references: map[string]string{"party": " "}, wantErr: "source is required"},
{name: "unsupported source", references: map[string]string{"party": "narratio.input.unknown"}, wantErr: "not a supported prepared input source"},
{name: "empty selector", references: map[string]string{" ": "narratio.input.party"}, wantField: "pipeline.notarius.references", wantReason: "selector is required"},
{name: "equals in selector", references: map[string]string{"party=x": "narratio.input.party"}, wantField: "pipeline.notarius.references", wantReason: "must not contain"},
{name: "invalid stage", references: map[string]string{"lane.prepare.party": "narratio.input.party"}, wantField: "pipeline.notarius.references", wantReason: "middle component"},
{name: "empty source", references: map[string]string{"party": " "}, wantField: "pipeline.notarius.references.party", wantReason: "source is required"},
{name: "unsupported source", references: map[string]string{"party": "narratio.input.unknown"}, wantField: "pipeline.notarius.references.party", wantReason: "not a supported prepared input source"},
{
name: "normalized collision",
references: map[string]string{
"chunk.party": "narratio.input.party",
" chunk . party ": "narratio.input.players",
},
wantErr: "normalize to",
wantField: "pipeline.notarius.references",
wantReason: "normalize to",
},
}
@@ -214,8 +216,8 @@ func TestNotariusReferenceValidationRejectsInvalidBindings(t *testing.T) {
cfg := validNotariusConfig()
cfg.References = tt.references
err := validateNotarius(cfg, nil)
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("validateNotarius() error = %v, want containing %q", err, tt.wantErr)
if err == nil || !strings.Contains(err.Error(), tt.wantField) || !strings.Contains(err.Error(), tt.wantReason) {
t.Fatalf("validateNotarius() error = %v, want field %q and reason %q", err, tt.wantField, tt.wantReason)
}
})
}
@@ -398,21 +400,21 @@ func TestNotariusEnabledValidation(t *testing.T) {
tests := []struct {
name string
section string
wantErr string
wantErr []string
}{
{name: "valid", section: valid},
{name: "blank binary", section: strings.Replace(valid, " enabled: true\n", " enabled: true\n binary: \" \"\n", 1), wantErr: "pipeline.notarius.binary is required"},
{name: "missing config path", section: strings.Replace(valid, " config_path: ./notarius.yml\n", "", 1), wantErr: "pipeline.notarius.config_path is required"},
{name: "missing pipeline id", section: strings.Replace(valid, " pipeline_id: dnd-session\n", "", 1), wantErr: "pipeline.notarius.pipeline_id is required"},
{name: "missing outputs", section: strings.Split(valid, " outputs:\n")[0], wantErr: "pipeline.notarius.outputs must contain at least one output"},
{name: "zero timeout", section: strings.Replace(valid, " timeout: 45m", " timeout: 0s", 1), wantErr: "pipeline.notarius.timeout must be positive"},
{name: "negative timeout", section: strings.Replace(valid, " timeout: 45m", " timeout: -1s", 1), wantErr: "pipeline.notarius.timeout must be positive"},
{name: "invalid timeout", section: strings.Replace(valid, " timeout: 45m", " timeout: later", 1), wantErr: "pipeline.notarius.timeout must be a valid duration"},
{name: "invalid output key", section: strings.Replace(valid, " npc_registry:", " npc-registry:", 1), wantErr: "outputs keys must match"},
{name: "missing lane", section: strings.Replace(valid, " lane_id: npc-registry\n", "", 1), wantErr: "lane_id is required"},
{name: "missing media type", section: strings.Replace(valid, " media_type: application/json\n", "", 1), wantErr: "media_type is required"},
{name: "missing schema id", section: strings.Replace(valid, " schema_id: notarius.dnd.npc_registry\n", "", 1), wantErr: "schema_id is required"},
{name: "missing schema version", section: strings.Replace(valid, " schema_version: v1\n", "", 1), wantErr: "schema_version is required"},
{name: "blank binary", section: strings.Replace(valid, " enabled: true\n", " enabled: true\n binary: \" \"\n", 1), wantErr: []string{"pipeline.notarius.binary", "required"}},
{name: "missing config path", section: strings.Replace(valid, " config_path: ./notarius.yml\n", "", 1), wantErr: []string{"pipeline.notarius.config_path", "required"}},
{name: "missing pipeline id", section: strings.Replace(valid, " pipeline_id: dnd-session\n", "", 1), wantErr: []string{"pipeline.notarius.pipeline_id", "required"}},
{name: "missing outputs", section: strings.Split(valid, " outputs:\n")[0], wantErr: []string{"pipeline.notarius.outputs", "at least one"}},
{name: "zero timeout", section: strings.Replace(valid, " timeout: 45m", " timeout: 0s", 1), wantErr: []string{"pipeline.notarius.timeout", "positive"}},
{name: "negative timeout", section: strings.Replace(valid, " timeout: 45m", " timeout: -1s", 1), wantErr: []string{"pipeline.notarius.timeout", "positive"}},
{name: "invalid timeout", section: strings.Replace(valid, " timeout: 45m", " timeout: later", 1), wantErr: []string{"pipeline.notarius.timeout", "valid duration"}},
{name: "invalid output key", section: strings.Replace(valid, " npc_registry:", " npc-registry:", 1), wantErr: []string{"pipeline.notarius.outputs keys", "must match"}},
{name: "missing lane", section: strings.Replace(valid, " lane_id: npc-registry\n", "", 1), wantErr: []string{"pipeline.notarius.outputs.npc_registry.lane_id", "required"}},
{name: "missing media type", section: strings.Replace(valid, " media_type: application/json\n", "", 1), wantErr: []string{"pipeline.notarius.outputs.npc_registry.media_type", "required"}},
{name: "missing schema id", section: strings.Replace(valid, " schema_id: notarius.dnd.npc_registry\n", "", 1), wantErr: []string{"pipeline.notarius.outputs.npc_registry.schema_id", "required"}},
{name: "missing schema version", section: strings.Replace(valid, " schema_version: v1\n", "", 1), wantErr: []string{"pipeline.notarius.outputs.npc_registry.schema_version", "required"}},
{
name: "normalized key collision",
section: strings.Replace(valid, " module_key: dnd/npc-registry\n", ` module_key: dnd/npc-registry
@@ -422,7 +424,7 @@ func TestNotariusEnabledValidation(t *testing.T) {
schema_id: two
schema_version: v1
`, 1),
wantErr: "normalize to duplicate source",
wantErr: []string{"pipeline.notarius.outputs keys", "duplicate source"},
},
{
name: "duplicate normalized lane",
@@ -433,7 +435,7 @@ func TestNotariusEnabledValidation(t *testing.T) {
schema_id: two
schema_version: v1
`, 1),
wantErr: "duplicates pipeline.notarius.outputs.npc_registry.lane_id",
wantErr: []string{"pipeline.notarius.outputs.spells.lane_id", "duplicates"},
},
}
@@ -445,7 +447,7 @@ func TestNotariusEnabledValidation(t *testing.T) {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if tt.wantErr == "" {
if len(tt.wantErr) == 0 {
if err != nil {
t.Fatalf("Validate() error = %v", err)
}
@@ -455,8 +457,13 @@ func TestNotariusEnabledValidation(t *testing.T) {
}
return
}
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want containing %q", err, tt.wantErr)
if err == nil {
t.Fatalf("Validate() error = nil, want containing %q", tt.wantErr)
}
for _, fragment := range tt.wantErr {
if !strings.Contains(err.Error(), fragment) {
t.Fatalf("Validate() error = %v, want containing %q", err, fragment)
}
}
})
}
@@ -471,7 +478,7 @@ func TestExtractionReferencesRequireDeclaredOutput(t *testing.T) {
tests := []struct {
name string
body string
wantErr string
wantErr []string
}{
{
name: "scriptorium declared extraction",
@@ -492,7 +499,7 @@ func TestExtractionReferencesRequireDeclaredOutput(t *testing.T) {
npcs:
source: narratio.extraction.unknown
`,
wantErr: `references unknown extraction output "unknown"`,
wantErr: []string{"pipeline.scriptorium.artifacts.recap.inputs.npcs.source", "unknown extraction output"},
},
{
name: "publish declared extraction",
@@ -509,7 +516,7 @@ func TestExtractionReferencesRequireDeclaredOutput(t *testing.T) {
- source: narratio.extraction.unknown
dest: artifacts/unknown.json
`,
wantErr: `extraction output "unknown" is not defined`,
wantErr: []string{"pipeline.publish.outputs[0].source", "not defined"},
},
{
name: "publish lock declared extraction",
@@ -528,14 +535,19 @@ func TestExtractionReferencesRequireDeclaredOutput(t *testing.T) {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if tt.wantErr == "" {
if len(tt.wantErr) == 0 {
if err != nil {
t.Fatalf("Validate() error = %v", err)
}
return
}
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("Validate() error = %v, want containing %q", err, tt.wantErr)
if err == nil {
t.Fatalf("Validate() error = nil, want containing %q", tt.wantErr)
}
for _, fragment := range tt.wantErr {
if !strings.Contains(err.Error(), fragment) {
t.Fatalf("Validate() error = %v, want containing %q", err, fragment)
}
}
})
}

76
internal/config/scriptorium_test.go
View File

@@ -11,7 +11,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
name string
scriptoriumYAML string
wantLoadErr string
wantValidateErr string
wantValidateErr []string
assert func(t *testing.T, cfg *Config)
}{
{
@@ -73,7 +73,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
enabled: true
output_path: artifacts/session_recap.md
`,
wantValidateErr: "pipeline.scriptorium.artifacts.session_recap.prompt_id is required when enabled",
wantValidateErr: []string{"pipeline.scriptorium.artifacts.session_recap.prompt_id", "required"},
},
{
name: "enabled artifact missing output path fails",
@@ -84,7 +84,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
enabled: true
prompt_id: dnd.session_recap
`,
wantValidateErr: "pipeline.scriptorium.artifacts.session_recap.output_path is required when enabled",
wantValidateErr: []string{"pipeline.scriptorium.artifacts.session_recap.output_path", "required"},
},
{
name: "invalid timeout fails",
@@ -92,7 +92,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
binary: scriptorium
timeout: definitely-not-a-duration
`,
wantValidateErr: "pipeline.scriptorium.timeout must be a valid duration",
wantValidateErr: []string{"pipeline.scriptorium.timeout", "valid duration"},
},
{
name: "legacy previous session source fails validation",
@@ -114,7 +114,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
session_id: true
output_kind: session_recap
`,
wantValidateErr: `pipeline.scriptorium.artifacts.session_recap.inputs.previous_recap.source "` + legacyPreviousSource + `" is unsupported`,
wantValidateErr: []string{"pipeline.scriptorium.artifacts.session_recap.inputs.previous_recap.source", "unsupported"},
},
{
name: "obsolete input passthrough fields fail strict decoding",
@@ -166,7 +166,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
source: narratio.previous_session.artifact.
required: false
`,
wantValidateErr: `pipeline.scriptorium.artifacts.session_recap.inputs.previous_recap.source "narratio.previous_session.artifact." must reference configured artifact key matching ^[a-z][a-z0-9_]*$`,
wantValidateErr: []string{"pipeline.scriptorium.artifacts.session_recap.inputs.previous_recap.source", "configured artifact key"},
},
{
name: "canonical previous-session source invalid artifact key fails validation",
@@ -182,7 +182,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
source: narratio.previous_session.artifact.session-recap
required: false
`,
wantValidateErr: `pipeline.scriptorium.artifacts.session_recap.inputs.previous_recap.source "narratio.previous_session.artifact.session-recap" must reference configured artifact key matching ^[a-z][a-z0-9_]*$`,
wantValidateErr: []string{"pipeline.scriptorium.artifacts.session_recap.inputs.previous_recap.source", "configured artifact key"},
},
{
name: "canonical previous-session source unknown artifact fails validation",
@@ -198,7 +198,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
source: narratio.previous_session.artifact.quest_log
required: false
`,
wantValidateErr: `pipeline.scriptorium.artifacts.session_recap.inputs.previous_recap.source "narratio.previous_session.artifact.quest_log" references unknown artifact "quest_log"`,
wantValidateErr: []string{"pipeline.scriptorium.artifacts.session_recap.inputs.previous_recap.source", "unknown artifact"},
},
{
name: "canonical artifact source is accepted",
@@ -265,7 +265,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
source: narratio.unknown
required: true
`,
wantValidateErr: `pipeline.scriptorium.artifacts.session_recap.inputs.transcript.source "narratio.unknown" is unsupported`,
wantValidateErr: []string{"pipeline.scriptorium.artifacts.session_recap.inputs.transcript.source", "unsupported"},
},
{
name: "artifact render_debug override is accepted",
@@ -380,7 +380,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
`,
wantValidateErr: "pipeline.scriptorium.artifacts keys must match ^[a-z][a-z0-9_]*$",
wantValidateErr: []string{"pipeline.scriptorium.artifacts keys", "must match"},
},
{
name: "artifact output path outside artifacts root fails validation",
@@ -392,7 +392,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
prompt_id: dnd.session_recap
output_path: transcripts/session_recap.md
`,
wantValidateErr: "pipeline.scriptorium.artifacts.session_recap.output_path must be under artifacts/",
wantValidateErr: []string{"pipeline.scriptorium.artifacts.session_recap.output_path", "under artifacts/"},
},
{
name: "missing depends_on for artifact source fails validation",
@@ -416,7 +416,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
source: narratio.artifact.session_recap
required: true
`,
wantValidateErr: `pipeline.scriptorium.artifacts.player_handout.inputs.recap.source "narratio.artifact.session_recap" requires depends_on entry "session_recap"`,
wantValidateErr: []string{"pipeline.scriptorium.artifacts.player_handout.inputs.recap.source", "requires depends_on"},
},
{
name: "dependency on unknown artifact fails validation",
@@ -434,7 +434,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
source: narratio.artifact.session_recap
required: true
`,
wantValidateErr: `pipeline.scriptorium.artifacts.player_handout.depends_on[0] "session_recap" is not a configured artifact key`,
wantValidateErr: []string{"pipeline.scriptorium.artifacts.player_handout.depends_on[0]", "not a configured artifact"},
},
{
name: "self dependency fails validation",
@@ -448,7 +448,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
prompt_id: dnd.session_recap
output_path: artifacts/session_recap.md
`,
wantValidateErr: "pipeline.scriptorium.artifacts.session_recap.depends_on must not include itself",
wantValidateErr: []string{"pipeline.scriptorium.artifacts.session_recap.depends_on", "itself"},
},
{
name: "configured dependency cycle fails validation",
@@ -476,7 +476,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
source: narratio.artifact.artifact_a
required: true
`,
wantValidateErr: "pipeline.scriptorium.artifacts dependencies must not contain cycles",
wantValidateErr: []string{"pipeline.scriptorium.artifacts dependencies", "cycles"},
},
{
name: "disabled dependency cycle fails validation",
@@ -494,7 +494,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
- artifact_a
output_path: artifacts/b.md
`,
wantValidateErr: "pipeline.scriptorium.artifacts dependencies must not contain cycles",
wantValidateErr: []string{"pipeline.scriptorium.artifacts dependencies", "cycles"},
},
{
name: "artifact source typo fails validation",
@@ -510,7 +510,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
source: narratio.artifact.session-recap
required: true
`,
wantValidateErr: `pipeline.scriptorium.artifacts.player_handout.inputs.recap.source "narratio.artifact.session-recap" is unsupported`,
wantValidateErr: []string{"pipeline.scriptorium.artifacts.player_handout.inputs.recap.source", "unsupported"},
},
{
name: "referenced disabled artifact missing output path fails validation",
@@ -530,7 +530,7 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
source: narratio.artifact.session_recap
required: true
`,
wantValidateErr: "pipeline.scriptorium.artifacts.session_recap.output_path is required when artifact is referenced",
wantValidateErr: []string{"pipeline.scriptorium.artifacts.session_recap.output_path", "referenced"},
},
}
@@ -558,12 +558,14 @@ func TestScriptoriumLoadAndValidate(t *testing.T) {
}
err = Validate(cfg)
if tt.wantValidateErr != "" {
if len(tt.wantValidateErr) > 0 {
if err == nil {
t.Fatalf("expected validation error containing %q, got nil", tt.wantValidateErr)
}
if !strings.Contains(err.Error(), tt.wantValidateErr) {
t.Fatalf("validation error = %q, want to contain %q", err.Error(), tt.wantValidateErr)
for _, fragment := range tt.wantValidateErr {
if !strings.Contains(err.Error(), fragment) {
t.Fatalf("validation error = %q, want to contain %q", err.Error(), fragment)
}
}
return
}
@@ -596,16 +598,9 @@ func TestScriptoriumDiagnosticsAreIndependentOfMapInsertionOrder(t *testing.T) {
}
}
func TestScriptoriumLegacyTranscriptSourcesRejected(t *testing.T) {
legacyTranscriptSources := []string{
"narratio.transcript." + "merged",
"narratio.transcript." + "full",
"narratio.transcript." + "trimmed",
}
for _, source := range legacyTranscriptSources {
t.Run(source, func(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
func TestScriptoriumUnsupportedTranscriptSourceRejected(t *testing.T) {
source := "narratio.transcript." + "merged"
pipelineYAML := testPipelineBaseYAML + `
scriptorium:
binary: scriptorium
artifacts:
@@ -618,17 +613,14 @@ scriptorium:
source: ` + source + `
required: true
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
wantErr := `pipeline.scriptorium.artifacts.session_recap.inputs.transcript.source "` + source + `" is unsupported`
if err == nil || !strings.Contains(err.Error(), wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, wantErr)
}
})
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if err == nil || !strings.Contains(err.Error(), "pipeline.scriptorium.artifacts.session_recap.inputs.transcript.source") || !strings.Contains(err.Error(), "unsupported") {
t.Fatalf("Validate() error = %v, want source field and unsupported category", err)
}
}

64
internal/config/storage_publish_test.go
View File

@@ -400,32 +400,22 @@ publish:
}
}
func TestPublishOutputLegacyTranscriptSourcesRejected(t *testing.T) {
legacyTranscriptSources := []string{
"narratio.transcript." + "merged",
"narratio.transcript." + "full",
"narratio.transcript." + "trimmed",
}
for _, source := range legacyTranscriptSources {
t.Run(source, func(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
func TestPublishOutputUnsupportedTranscriptSourceRejected(t *testing.T) {
source := "narratio.transcript." + "merged"
pipelineYAML := testPipelineBaseYAML + `
publish:
outputs:
- source: ` + source + `
dest: transcripts/final.trimmed.json
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
wantErr := `source "` + source + `" is unsupported`
if err == nil || !strings.Contains(err.Error(), wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, wantErr)
}
})
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if err == nil || !strings.Contains(err.Error(), "pipeline.publish.outputs[0].source") || !strings.Contains(err.Error(), "unsupported") {
t.Fatalf("Validate() error = %v, want source field and unsupported category", err)
}
}
@@ -579,31 +569,21 @@ publish:
}
}
func TestPublishLockLegacyTranscriptSourcesRejected(t *testing.T) {
legacyTranscriptSources := []string{
"narratio.transcript." + "merged",
"narratio.transcript." + "full",
"narratio.transcript." + "trimmed",
}
for _, source := range legacyTranscriptSources {
t.Run(source, func(t *testing.T) {
pipelineYAML := testPipelineBaseYAML + `
func TestPublishLockUnsupportedTranscriptSourceRejected(t *testing.T) {
source := "narratio.transcript." + "merged"
pipelineYAML := testPipelineBaseYAML + `
publish:
locks:
- source: ` + source + `
`
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
wantErr := `source "` + source + `" is unsupported`
if err == nil || !strings.Contains(err.Error(), wantErr) {
t.Fatalf("Validate() error = %v, want to contain %q", err, wantErr)
}
})
pipelinePath, sessionPath := writeConfigFiles(t, pipelineYAML, testSessionBaseYAML)
cfg, err := Load(pipelinePath, sessionPath)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
err = Validate(cfg)
if err == nil || !strings.Contains(err.Error(), "pipeline.publish.locks[0].source") || !strings.Contains(err.Error(), "unsupported") {
t.Fatalf("Validate() error = %v, want source field and unsupported category", err)
}
}