From cc43906ae92b529c68f509a9bef28f772f48090f Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 16 Sep 2026 14:31:54 +0000 Subject: [PATCH] Verify Notarius credential lifecycle --- docs/roadmap/implementation.md | 20 +++- internal/app/config_commands_test.go | 48 +++++++-- internal/app/extract_lifecycle_test.go | 101 ++++++++++++++++++ internal/app/secrets_bootstrap_test.go | 49 ++++++++- .../stage/semantic_contracts_delivery_test.go | 51 +++++++++ 5 files changed, 260 insertions(+), 9 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index e97e681..7fbe6cd 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -178,7 +178,25 @@ Acceptance: ## Stage 3 — Verify command lifecycle, inspection, and resume invariants -Status: Not started +Status: Complete — 2026-09-16 + +Completion: Extended command-boundary fixtures to cover Notarius credentials +from conventional and explicit secret directories, environment precedence, and +the existing-empty failure rule. Inspection coverage now proves validation, +safe show/sources/diff output, ownership, profile replacement and null removal, +and unavailable-reference rejection. Lifecycle tests verify manifests and the +prepared effective pipeline omit credential bytes, while adding, removing, +renaming, or rotating a valid loaded credential leaves extract's semantic +fingerprint, resume eligibility, and downstream success unchanged without a +second Notarius invocation. No fingerprint version, schema, or invalidation +code changed. + +Checks: + +- `GOWORK=off go test ./internal/app ./internal/config ./internal/stage` — pass. +- `GOWORK=off go test -race ./internal/app ./internal/config ./internal/stage` — pass. +- `GOWORK=off go test ./...` — pass. +- `git diff --check` — pass. Read: `internal/app/secrets_bootstrap.go`, `secrets_bootstrap_test.go`, `config_commands_test.go`, `config_loader.go`, `extract_lifecycle_test.go`, diff --git a/internal/app/config_commands_test.go b/internal/app/config_commands_test.go index 92b4185..bbe54c9 100644 --- a/internal/app/config_commands_test.go +++ b/internal/app/config_commands_test.go @@ -367,10 +367,12 @@ func TestConfigInspectionOutputsNeverExposeResolvedCredentialValues(t *testing.T const firstCanary = "inspection-credential-canary-one" const secondCanary = "inspection-credential-canary-two" for name, value := range map[string]string{ - "NARRATIO_INSPECT_ACCESS": firstCanary, - "NARRATIO_INSPECT_SECRET": firstCanary, - "NARRATIO_INSPECT_LLM_ONE": firstCanary, - "NARRATIO_INSPECT_LLM_TWO": secondCanary, + "NARRATIO_INSPECT_ACCESS": firstCanary, + "NARRATIO_INSPECT_SECRET": firstCanary, + "NARRATIO_INSPECT_LLM_ONE": firstCanary, + "NARRATIO_INSPECT_LLM_TWO": secondCanary, + "NARRATIO_INSPECT_NOTARIUS_ONE": firstCanary, + "NARRATIO_INSPECT_NOTARIUS_TWO": secondCanary, } { t.Setenv(name, value) } @@ -384,13 +386,23 @@ func TestConfigInspectionOutputsNeverExposeResolvedCredentialValues(t *testing.T access_key_id: NARRATIO_INSPECT_ACCESS secret_access_key: NARRATIO_INSPECT_SECRET `, 1) + pipelineYAML += `notarius: + enabled: false + llm_api_key: NARRATIO_INSPECT_NOTARIUS_ONE +` if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil { t.Fatal(err) } writeInspectionProfiles(t, pipelinePath, "audita:\n model: production\n llm_api_key: NARRATIO_INSPECT_LLM_ONE\n", - "audita:\n model: testing\n llm_api_key: NARRATIO_INSPECT_LLM_TWO\n", + "audita:\n model: testing\n llm_api_key: NARRATIO_INSPECT_LLM_TWO\nnotarius:\n llm_api_key: NARRATIO_INSPECT_NOTARIUS_TWO\n", ) + mustWriteTestFile(t, filepath.Join(filepath.Dir(pipelinePath), "profiles", "without-notarius-key.yml"), "notarius:\n llm_api_key: null\n") + + var validate bytes.Buffer + if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--profile", "production"}, &validate); err != nil { + t.Fatalf("config validate: %v", err) + } var show bytes.Buffer if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--profile", "production"}, &show); err != nil { @@ -408,19 +420,41 @@ func TestConfigInspectionOutputsNeverExposeResolvedCredentialValues(t *testing.T if err := ConfigDiff(context.Background(), []string{"production", "testing", "--config", pipelinePath}, &diff); err != nil { t.Fatalf("config diff: %v", err) } + var removed bytes.Buffer + if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--profile", "without-notarius-key"}, &removed); err != nil { + t.Fatalf("config show with removed Notarius credential: %v", err) + } for label, output := range map[string]string{"show": show.String(), "sources": sources.String(), "artifacts": catalog.String(), "diff": diff.String()} { if strings.Contains(output, firstCanary) || strings.Contains(output, secondCanary) { t.Fatalf("%s output exposed credential value: %q", label, output) } } - for _, reference := range []string{"NARRATIO_INSPECT_ACCESS", "NARRATIO_INSPECT_SECRET", "NARRATIO_INSPECT_LLM_ONE"} { + for _, reference := range []string{"NARRATIO_INSPECT_ACCESS", "NARRATIO_INSPECT_SECRET", "NARRATIO_INSPECT_LLM_ONE", "NARRATIO_INSPECT_NOTARIUS_ONE"} { if !strings.Contains(show.String(), reference) { t.Fatalf("show omitted safe credential reference %q: %q", reference, show.String()) } } - if !strings.Contains(diff.String(), "NARRATIO_INSPECT_LLM_ONE") || !strings.Contains(diff.String(), "NARRATIO_INSPECT_LLM_TWO") { + if !strings.Contains(diff.String(), "NARRATIO_INSPECT_LLM_ONE") || !strings.Contains(diff.String(), "NARRATIO_INSPECT_LLM_TWO") || + !strings.Contains(diff.String(), "NARRATIO_INSPECT_NOTARIUS_ONE") || !strings.Contains(diff.String(), "NARRATIO_INSPECT_NOTARIUS_TWO") { t.Fatalf("diff omitted safe credential references: %q", diff.String()) } + if !strings.Contains(sources.String(), "notarius.llm_api_key") { + t.Fatalf("sources omitted Notarius credential ownership: %q", sources.String()) + } + if strings.Contains(removed.String(), "llm_api_key") || strings.Contains(removed.String(), "NARRATIO_INSPECT_NOTARIUS_ONE") { + t.Fatalf("profile null did not remove Notarius credential: %q", removed.String()) + } +} + +func TestConfigValidateRejectsUnavailableNotariusCredential(t *testing.T) { + const envName = "NARRATIO_INSPECT_NOTARIUS_MISSING" + restoreEnvAfterTest(t, envName) + path := filepath.Join(t.TempDir(), "pipeline.yml") + mustWriteTestFile(t, path, "whisperx:\n transcribe_url: https://transcription.example.test/transcribe\nnotarius:\n enabled: false\n llm_api_key: "+envName+"\n") + err := ConfigValidate(context.Background(), []string{"--config", path}, io.Discard) + if err == nil || !strings.Contains(err.Error(), "secret.unset") { + t.Fatalf("ConfigValidate() error = %v, want unavailable credential failure", err) + } } func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testing.T) { diff --git a/internal/app/extract_lifecycle_test.go b/internal/app/extract_lifecycle_test.go index b7774c2..6e3f085 100644 --- a/internal/app/extract_lifecycle_test.go +++ b/internal/app/extract_lifecycle_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + configkit "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius" "gitea.maximumdirect.net/eric/narratio/internal/artifactmodel" "gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy" @@ -485,6 +486,92 @@ func TestExtractLifecycleSkipsCurrentResumableResult(t *testing.T) { } } +func TestExtractLifecycleCredentialChangesRemainResumable(t *testing.T) { + for _, test := range []struct { + name string + before func(*testing.T) *configkit.Secret + after func(*testing.T) *configkit.Secret + }{ + { + name: "added", + after: func(t *testing.T) *configkit.Secret { + return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_ADDED_KEY", "added-value") + }, + }, + { + name: "removed", + before: func(t *testing.T) *configkit.Secret { + return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_REMOVED_KEY", "removed-value") + }, + }, + { + name: "renamed", + before: func(t *testing.T) *configkit.Secret { + return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_FIRST_KEY", "first-value") + }, + after: func(t *testing.T) *configkit.Secret { + return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_SECOND_KEY", "second-value") + }, + }, + { + name: "value rotated", + before: func(t *testing.T) *configkit.Secret { + return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_ROTATED_KEY", "first-value") + }, + after: func(t *testing.T) *configkit.Secret { + return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_ROTATED_KEY", "second-value") + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + cfg, env, runner := extractionLifecycleFixture(t, true) + if test.before != nil { + cfg.Pipeline.Notarius.LLMAPIKey = test.before(t) + } + analyzeRuns := 0 + plan := extractionLifecyclePlan(t, &analyzeRuns) + if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil { + t.Fatalf("initial executeStages() error = %v", err) + } + cfg.Pipeline.Notarius.LLMAPIKey = nil + if test.after != nil { + cfg.Pipeline.Notarius.LLMAPIKey = test.after(t) + } + resumed, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}) + if err != nil { + t.Fatalf("resume executeStages() error = %v", err) + } + if len(resumed.Executed) != 0 || len(resumed.Skipped) != 2 || len(runner.requests) != 1 || analyzeRuns != 1 { + t.Fatalf("credential-only change reexecuted work: summary=%#v requests=%d analyze=%d", resumed, len(runner.requests), analyzeRuns) + } + loaded := loadLifecycleManifest(t, cfg) + if loaded.Stages["extract"].Status != manifest.StatusSucceeded || loaded.Stages["analyze"].Status != manifest.StatusSucceeded { + t.Fatalf("credential-only change invalidated stages: extract=%q analyze=%q", loaded.Stages["extract"].Status, loaded.Stages["analyze"].Status) + } + }) + } +} + +func TestPreparedPipelineAndManifestExcludeNotariusCredentialValue(t *testing.T) { + const name = "NARRATIO_LIFECYCLE_PERSISTED_KEY" + const canary = "notarius-persisted-credential-canary" + cfg, env, _ := extractionLifecycleFixture(t, true) + cfg.Pipeline.Notarius.LLMAPIKey = loadLifecycleNotariusCredential(t, name, canary) + if _, err := executeStages(context.Background(), cfg, prepareExtractLifecyclePlan(t), RunOptions{Env: env}); err != nil { + t.Fatalf("executeStages() error = %v", err) + } + paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID) + for _, path := range []string{paths.ManifestPath, filepath.Join(paths.InputsDir, "pipeline.resolved.yml")} { + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile(%q) error = %v", path, err) + } + if strings.Contains(string(data), canary) { + t.Fatalf("persisted Narratio file %q contains the credential value", path) + } + } +} + func TestExtractLifecycleRerunsAfterDirectTranscriptChange(t *testing.T) { cfg, env, runner := extractionLifecycleFixture(t, true) analyzeRuns := 0 @@ -761,3 +848,17 @@ func extractionLifecycleFixture(t *testing.T, enabled bool) (*config.Config, *st runner := &materializingNotariusRunner{cfg: cfg.Pipeline.Notarius} return cfg, &stage.Env{Notarius: runner}, runner } + +func loadLifecycleNotariusCredential(t *testing.T, name, value string) *configkit.Secret { + t.Helper() + t.Setenv(name, value) + path := filepath.Join(t.TempDir(), "pipeline.yml") + if err := os.WriteFile(path, []byte("whisperx:\n transcribe_url: https://transcription.example.test/transcribe\nnotarius:\n enabled: false\n llm_api_key: "+name+"\n"), 0o644); err != nil { + t.Fatal(err) + } + pipeline, err := config.LoadPipeline(path) + if err != nil { + t.Fatal(err) + } + return pipeline.Notarius.LLMAPIKey +} diff --git a/internal/app/secrets_bootstrap_test.go b/internal/app/secrets_bootstrap_test.go index e4501a3..909a9c9 100644 --- a/internal/app/secrets_bootstrap_test.go +++ b/internal/app/secrets_bootstrap_test.go @@ -2,6 +2,7 @@ package app import ( "bytes" + "fmt" "io" "os" "path/filepath" @@ -119,6 +120,52 @@ func TestSecretsBootstrapPreservesExistingEnvironmentValues(t *testing.T) { } } +func TestNotariusCredentialLoadsFromCommandSecretBootstrap(t *testing.T) { + tests := []struct { + name string + explicit bool + environment string + fileValue string + wantValue string + }{ + {name: "conventional", fileValue: "conventional-value", wantValue: "conventional-value"}, + {name: "explicit", explicit: true, fileValue: "explicit-value", wantValue: "explicit-value"}, + {name: "environment precedence", explicit: true, environment: "environment-value", fileValue: "file-value", wantValue: "environment-value"}, + } + for index, test := range tests { + t.Run(test.name, func(t *testing.T) { + envName := fmt.Sprintf("NARRATIO_NOTARIUS_BOOTSTRAP_%d", index) + restoreEnvAfterTest(t, envName) + root := t.TempDir() + pipelinePath := filepath.Join(root, "pipeline.yml") + if err := os.WriteFile(pipelinePath, []byte("whisperx:\n transcribe_url: https://transcription.example.test/transcribe\nnotarius:\n enabled: false\n llm_api_key: "+envName+"\n"), 0o644); err != nil { + t.Fatal(err) + } + secretsDir := filepath.Join(root, "secrets") + var selection *string + if test.explicit { + secretsDir = filepath.Join(t.TempDir(), "selected-secrets") + selection = &secretsDir + } + mustWriteSecretFile(t, filepath.Join(secretsDir, envName), test.fileValue+"\n") + if test.environment != "" { + if err := os.Setenv(envName, test.environment); err != nil { + t.Fatal(err) + } + } + + _, pipeline, err := loadPipelineConfigWithDependencies(pipelinePath, config.PipelineLoadOptions{}, selection, commandDependencies{diagnostics: io.Discard}) + if err != nil { + t.Fatalf("load pipeline: %v", err) + } + if pipeline.Notarius == nil || pipeline.Notarius.LLMAPIKey == nil || !pipeline.Notarius.LLMAPIKey.IsResolved() || + pipeline.Notarius.LLMAPIKey.EnvName() != envName || pipeline.Notarius.LLMAPIKey.Value() != test.wantValue { + t.Fatal("Notarius credential did not resolve from the selected command bootstrap source") + } + }) + } +} + func TestSecretsBootstrapPreservesSafePartialReportOnFailure(t *testing.T) { directory := t.TempDir() mustWriteSecretFile(t, filepath.Join(directory, "INVALID-NAME"), "must-not-appear\n") @@ -173,7 +220,7 @@ func TestExecuteConfigValidateReportsSkippedEmptyBeforeTypedLoadFailure(t *testi if err != nil { t.Fatal(err) } - pipelineYAML = []byte(strings.Replace(string(pipelineYAML), "audita:\n binary:", "audita:\n llm_api_key: "+name+"\n binary:", 1)) + pipelineYAML = append(pipelineYAML, []byte("notarius:\n enabled: false\n llm_api_key: "+name+"\n")...) if err := os.WriteFile(pipelinePath, pipelineYAML, 0o644); err != nil { t.Fatal(err) } diff --git a/internal/stage/semantic_contracts_delivery_test.go b/internal/stage/semantic_contracts_delivery_test.go index f45b466..f78b254 100644 --- a/internal/stage/semantic_contracts_delivery_test.go +++ b/internal/stage/semantic_contracts_delivery_test.go @@ -2,6 +2,8 @@ package stage import ( "context" + "os" + "path/filepath" "testing" configkit "gitea.maximumdirect.net/eric/ConfigKit" @@ -93,6 +95,55 @@ func TestExtractResumeValidationExcludesOperationalSettings(t *testing.T) { } } +func TestExtractSemanticConfigExcludesLoadedCredentialChanges(t *testing.T) { + const firstName = "NARRATIO_SEMANTIC_NOTARIUS_KEY" + const secondName = "NARRATIO_SEMANTIC_NOTARIUS_KEY_TWO" + first := loadNotariusCredentialForSemanticTest(t, firstName, "first-value") + rotated := loadNotariusCredentialForSemanticTest(t, firstName, "rotated-value") + renamed := loadNotariusCredentialForSemanticTest(t, secondName, "second-value") + + baseline := downstreamSemanticEnv() + baseline.Config.Pipeline.Notarius.LLMAPIKey = first + want, err := (extractStage{}).SemanticConfigFingerprint(baseline) + if err != nil { + t.Fatal(err) + } + for _, test := range []struct { + name string + credential *configkit.Secret + }{ + {name: "removed", credential: nil}, + {name: "renamed", credential: renamed}, + {name: "value rotated", credential: rotated}, + } { + t.Run(test.name, func(t *testing.T) { + env := downstreamSemanticEnv() + env.Config.Pipeline.Notarius.LLMAPIKey = test.credential + got, err := (extractStage{}).SemanticConfigFingerprint(env) + if err != nil { + t.Fatal(err) + } + if !want.Equal(got) { + t.Fatalf("credential-only change altered extract semantic fingerprint") + } + }) + } +} + +func loadNotariusCredentialForSemanticTest(t *testing.T, name, value string) *configkit.Secret { + t.Helper() + t.Setenv(name, value) + path := filepath.Join(t.TempDir(), "pipeline.yml") + if err := os.WriteFile(path, []byte("whisperx:\n transcribe_url: https://transcription.example.test/transcribe\nnotarius:\n enabled: false\n llm_api_key: "+name+"\n"), 0o644); err != nil { + t.Fatal(err) + } + pipeline, err := config.LoadPipeline(path) + if err != nil { + t.Fatal(err) + } + return pipeline.Notarius.LLMAPIKey +} + func TestPublishSemanticConfigSensitivityAndOperationalExclusions(t *testing.T) { want, err := (publishStage{}).SemanticConfigFingerprint(downstreamSemanticEnv()) if err != nil {