Verify Notarius profile lifecycle behavior
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# Implementation Plan: Notarius Configuration Profile Selection
|
||||
|
||||
Status: Implementation in progress; Stages 1-2 complete.
|
||||
Status: Implementation in progress; Stages 1-3 complete.
|
||||
|
||||
## Scope and Working Instructions
|
||||
|
||||
@@ -159,7 +159,14 @@ changing the manifest schema.
|
||||
|
||||
## Stage 3: Verify Application Lifecycle Behavior
|
||||
|
||||
Status: Pending.
|
||||
Status: Complete.
|
||||
|
||||
Completion marker (2026-09-21): Extended the existing application lifecycle
|
||||
fixtures to cover same-selection reuse, named and base selection changes,
|
||||
dependent invalidation/rerun, sibling preservation, ordinary downstream
|
||||
failure, and real ConfigKit profile-to-execution composition without implicit
|
||||
name forwarding or digest-wide invalidation. Verified with
|
||||
`go test ./internal/app ./internal/stage`.
|
||||
|
||||
Preparation:
|
||||
|
||||
|
||||
@@ -691,6 +691,187 @@ func TestExtractLifecycleCredentialChangesRemainResumable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleConfigProfileChangesRerunExtractionAndDependents(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
selected := "inexpensive"
|
||||
cfg.Pipeline.Notarius.ConfigProfile = &selected
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||
t.Fatalf("initial executeStages() error = %v", err)
|
||||
}
|
||||
if len(runner.requests) != 1 || runner.requests[0].ConfigProfile == nil || *runner.requests[0].ConfigProfile != selected || analyzeRuns != 1 {
|
||||
t.Fatalf("initial requests=%#v analyze=%d, want selected extraction and analysis", runner.requests, analyzeRuns)
|
||||
}
|
||||
reused, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("reuse executeStages() error = %v", err)
|
||||
}
|
||||
if len(reused.Executed) != 0 || len(runner.requests) != 1 || analyzeRuns != 1 {
|
||||
t.Fatalf("same-profile reuse summary=%#v requests=%d analyze=%d", reused, len(runner.requests), analyzeRuns)
|
||||
}
|
||||
|
||||
markLifecycleStageSucceeded(t, cfg, "render")
|
||||
markLifecycleStageSucceeded(t, cfg, "publish")
|
||||
before := loadLifecycleManifest(t, cfg)
|
||||
trimBefore, err := json.Marshal(before.Stages["trim"])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
changed := "experimental"
|
||||
cfg.Pipeline.Notarius.ConfigProfile = &changed
|
||||
rerun, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("changed-profile executeStages() error = %v", err)
|
||||
}
|
||||
if len(rerun.Executed) != 2 || len(runner.requests) != 2 || runner.requests[1].ConfigProfile == nil || *runner.requests[1].ConfigProfile != changed || analyzeRuns != 2 {
|
||||
t.Fatalf("changed-profile summary=%#v requests=%#v analyze=%d", rerun, runner.requests, analyzeRuns)
|
||||
}
|
||||
after := loadLifecycleManifest(t, cfg)
|
||||
trimAfter, err := json.Marshal(after.Stages["trim"])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(trimAfter) != string(trimBefore) {
|
||||
t.Fatalf("profile change altered upstream trim record:\nbefore=%s\nafter=%s", trimBefore, trimAfter)
|
||||
}
|
||||
if after.Stages["render"].Status != manifest.StatusSucceeded || after.Stages["analyze"].Status != manifest.StatusSucceeded || after.Stages["publish"].Status != manifest.StatusStale {
|
||||
t.Fatalf("changed-profile lifecycle render=%#v analyze=%#v publish=%#v", after.Stages["render"], after.Stages["analyze"], after.Stages["publish"])
|
||||
}
|
||||
|
||||
cfg.Pipeline.Notarius.ConfigProfile = nil
|
||||
returned, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("return-to-base executeStages() error = %v", err)
|
||||
}
|
||||
if len(returned.Executed) != 2 || len(runner.requests) != 3 || runner.requests[2].ConfigProfile != nil || analyzeRuns != 3 {
|
||||
t.Fatalf("return-to-base summary=%#v requests=%#v analyze=%d", returned, runner.requests, analyzeRuns)
|
||||
}
|
||||
final := loadLifecycleManifest(t, cfg)
|
||||
if final.Stages["trim"].Status != manifest.StatusSucceeded || final.Stages["render"].Status != manifest.StatusSucceeded {
|
||||
t.Fatalf("return-to-base invalidated upstream siblings: trim=%#v render=%#v", final.Stages["trim"], final.Stages["render"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleConfigProfileFailureClearsCurrentResultAndDependents(t *testing.T) {
|
||||
cfg, env, runner := extractionLifecycleFixture(t, true)
|
||||
selected := "inexpensive"
|
||||
cfg.Pipeline.Notarius.ConfigProfile = &selected
|
||||
analyzeRuns := 0
|
||||
plan := extractionLifecyclePlan(t, &analyzeRuns)
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
|
||||
t.Fatalf("initial executeStages() error = %v", err)
|
||||
}
|
||||
markLifecycleStageSucceeded(t, cfg, "render")
|
||||
markLifecycleStageSucceeded(t, cfg, "publish")
|
||||
|
||||
changed := "missing-profile"
|
||||
cfg.Pipeline.Notarius.ConfigProfile = &changed
|
||||
runner.failuresRemaining = 1
|
||||
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err == nil {
|
||||
t.Fatal("changed-profile executeStages() error = nil, want downstream failure")
|
||||
}
|
||||
failed := loadLifecycleManifest(t, cfg)
|
||||
extract := failed.Stages["extract"]
|
||||
if extract.Status != manifest.StatusFailed || len(extract.Outputs) != 0 || len(extract.Logs) != 0 || len(extract.GeneratedConfigs) != 0 || len(extract.Metadata) != 0 {
|
||||
t.Fatalf("failed changed-profile extraction advertised prior success: %#v", extract)
|
||||
}
|
||||
if failed.Stages["trim"].Status != manifest.StatusSucceeded || failed.Stages["render"].Status != manifest.StatusSucceeded ||
|
||||
failed.Stages["analyze"].Status != manifest.StatusStale || failed.Stages["publish"].Status != manifest.StatusStale || analyzeRuns != 1 {
|
||||
t.Fatalf("failed changed-profile lifecycle trim=%#v render=%#v analyze=%#v publish=%#v analyzeRuns=%d",
|
||||
failed.Stages["trim"], failed.Stages["render"], failed.Stages["analyze"], failed.Stages["publish"], analyzeRuns)
|
||||
}
|
||||
if len(runner.requests) != 2 || runner.requests[1].ConfigProfile == nil || *runner.requests[1].ConfigProfile != changed {
|
||||
t.Fatalf("failed request = %#v, want changed selector", runner.requests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractLifecycleRealConfigProfilesSelectNotariusExplicitly(t *testing.T) {
|
||||
seed, env, runner := extractionLifecycleFixture(t, true)
|
||||
workspace := seed.Pipeline.Workspace.Root
|
||||
pipelinePath, campaignPath, sessionPath := writeValidConfigFiles(t, workspace)
|
||||
pipelineFile, err := os.OpenFile(pipelinePath, os.O_APPEND|os.O_WRONLY, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, writeErr := pipelineFile.WriteString(`notarius:
|
||||
enabled: true
|
||||
binary: ` + filepath.Join(workspace, "notarius") + `
|
||||
config_path: notarius.yml
|
||||
pipeline_id: dnd-session
|
||||
timeout: 45m
|
||||
outputs:
|
||||
npc_registry:
|
||||
lane_id: npc-registry
|
||||
media_type: application/json
|
||||
schema_id: notarius.dnd.npc_registry
|
||||
schema_version: v1
|
||||
module_key: dnd/npc-registry
|
||||
`)
|
||||
closeErr := pipelineFile.Close()
|
||||
if writeErr != nil || closeErr != nil {
|
||||
t.Fatalf("write Notarius configuration: %v / %v", writeErr, closeErr)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(filepath.Dir(pipelinePath), "notarius.yml"), []byte("pipelines: {}\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profilesDir := filepath.Join(filepath.Dir(pipelinePath), "profiles")
|
||||
if err := os.MkdirAll(profilesDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, contents := range map[string]string{
|
||||
"testing": "whisperx:\n language: fr\nnotarius:\n config_profile: downstream-economy\n",
|
||||
"narratio-only": "whisperx:\n language: de\n",
|
||||
"narratio-only-two": "whisperx:\n language: es\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(profilesDir, name+".yml"), []byte(contents), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
load := func(profile string) *config.Config {
|
||||
t.Helper()
|
||||
loaded, err := config.LoadWithSessionOptions(pipelinePath, campaignPath, sessionPath, config.SessionLoadOptions{Profile: &profile})
|
||||
if err != nil {
|
||||
t.Fatalf("load profile %q: %v", profile, err)
|
||||
}
|
||||
return loaded
|
||||
}
|
||||
plan, err := BuildSingleStagePlan("extract")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
testingConfig := load("testing")
|
||||
runner.cfg = testingConfig.Pipeline.Notarius
|
||||
if _, err := executeStages(context.Background(), testingConfig, plan, RunOptions{Env: env}); err != nil {
|
||||
t.Fatalf("execute testing profile: %v", err)
|
||||
}
|
||||
if len(runner.requests) != 1 || runner.requests[0].ConfigProfile == nil || *runner.requests[0].ConfigProfile != "downstream-economy" {
|
||||
t.Fatalf("testing profile request = %#v, want explicit differently named downstream selector", runner.requests)
|
||||
}
|
||||
|
||||
narratioOnly := load("narratio-only")
|
||||
runner.cfg = narratioOnly.Pipeline.Notarius
|
||||
if _, err := executeStages(context.Background(), narratioOnly, plan, RunOptions{Env: env}); err != nil {
|
||||
t.Fatalf("execute Narratio-only profile: %v", err)
|
||||
}
|
||||
if len(runner.requests) != 2 || runner.requests[1].ConfigProfile != nil {
|
||||
t.Fatalf("Narratio-only profile request = %#v, want no automatic downstream selector", runner.requests)
|
||||
}
|
||||
|
||||
narratioOnlyTwo := load("narratio-only-two")
|
||||
runner.cfg = narratioOnlyTwo.Pipeline.Notarius
|
||||
reused, err := executeStages(context.Background(), narratioOnlyTwo, plan, RunOptions{Env: env})
|
||||
if err != nil {
|
||||
t.Fatalf("execute second Narratio-only profile: %v", err)
|
||||
}
|
||||
if len(reused.Executed) != 0 || len(runner.requests) != 2 {
|
||||
t.Fatalf("Narratio profile label/effective digest broadened invalidation: summary=%#v requests=%#v", reused, runner.requests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedPipelineAndManifestExcludeNotariusCredentialValue(t *testing.T) {
|
||||
const name = "NARRATIO_LIFECYCLE_PERSISTED_KEY"
|
||||
const canary = "notarius-persisted-credential-canary"
|
||||
|
||||
Reference in New Issue
Block a user