eric/narratio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 27 Wiki Activity

Verify Notarius credential lifecycle

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-16 14:31:54 +00:00
parent dcaa2ad9c6
commit cc43906ae9
5 changed files with 260 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
docs/roadmap/implementation.md
View File

@@ -178,7 +178,25 @@ Acceptance:
## Stage 3 — Verify command lifecycle, inspection, and resume invariants
Status: Not started
Status: Complete — 2026-09-16
Completion: Extended command-boundary fixtures to cover Notarius credentials
from conventional and explicit secret directories, environment precedence, and
the existing-empty failure rule. Inspection coverage now proves validation,
safe show/sources/diff output, ownership, profile replacement and null removal,
and unavailable-reference rejection. Lifecycle tests verify manifests and the
prepared effective pipeline omit credential bytes, while adding, removing,
renaming, or rotating a valid loaded credential leaves extract's semantic
fingerprint, resume eligibility, and downstream success unchanged without a
second Notarius invocation. No fingerprint version, schema, or invalidation
code changed.
Checks:
- `GOWORK=off go test ./internal/app ./internal/config ./internal/stage` — pass.
- `GOWORK=off go test -race ./internal/app ./internal/config ./internal/stage` — pass.
- `GOWORK=off go test ./...` — pass.
- `git diff --check` — pass.
Read: `internal/app/secrets_bootstrap.go`, `secrets_bootstrap_test.go`,
`config_commands_test.go`, `config_loader.go`, `extract_lifecycle_test.go`,

48
internal/app/config_commands_test.go
View File

@@ -367,10 +367,12 @@ func TestConfigInspectionOutputsNeverExposeResolvedCredentialValues(t *testing.T
const firstCanary = "inspection-credential-canary-one"
const secondCanary = "inspection-credential-canary-two"
for name, value := range map[string]string{
"NARRATIO_INSPECT_ACCESS": firstCanary,
"NARRATIO_INSPECT_SECRET": firstCanary,
"NARRATIO_INSPECT_LLM_ONE": firstCanary,
"NARRATIO_INSPECT_LLM_TWO": secondCanary,
"NARRATIO_INSPECT_ACCESS": firstCanary,
"NARRATIO_INSPECT_SECRET": firstCanary,
"NARRATIO_INSPECT_LLM_ONE": firstCanary,
"NARRATIO_INSPECT_LLM_TWO": secondCanary,
"NARRATIO_INSPECT_NOTARIUS_ONE": firstCanary,
"NARRATIO_INSPECT_NOTARIUS_TWO": secondCanary,
} {
t.Setenv(name, value)
}
@@ -384,13 +386,23 @@ func TestConfigInspectionOutputsNeverExposeResolvedCredentialValues(t *testing.T
access_key_id: NARRATIO_INSPECT_ACCESS
secret_access_key: NARRATIO_INSPECT_SECRET
`, 1)
pipelineYAML += `notarius:
enabled: false
llm_api_key: NARRATIO_INSPECT_NOTARIUS_ONE
`
if err := os.WriteFile(pipelinePath, []byte(pipelineYAML), 0o644); err != nil {
t.Fatal(err)
}
writeInspectionProfiles(t, pipelinePath,
"audita:\n model: production\n llm_api_key: NARRATIO_INSPECT_LLM_ONE\n",
"audita:\n model: testing\n llm_api_key: NARRATIO_INSPECT_LLM_TWO\n",
"audita:\n model: testing\n llm_api_key: NARRATIO_INSPECT_LLM_TWO\nnotarius:\n llm_api_key: NARRATIO_INSPECT_NOTARIUS_TWO\n",
)
mustWriteTestFile(t, filepath.Join(filepath.Dir(pipelinePath), "profiles", "without-notarius-key.yml"), "notarius:\n llm_api_key: null\n")
var validate bytes.Buffer
if err := ConfigValidate(context.Background(), []string{"--config", pipelinePath, "--profile", "production"}, &validate); err != nil {
t.Fatalf("config validate: %v", err)
}
var show bytes.Buffer
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--profile", "production"}, &show); err != nil {
@@ -408,19 +420,41 @@ func TestConfigInspectionOutputsNeverExposeResolvedCredentialValues(t *testing.T
if err := ConfigDiff(context.Background(), []string{"production", "testing", "--config", pipelinePath}, &diff); err != nil {
t.Fatalf("config diff: %v", err)
}
var removed bytes.Buffer
if err := ConfigShow(context.Background(), []string{"--config", pipelinePath, "--profile", "without-notarius-key"}, &removed); err != nil {
t.Fatalf("config show with removed Notarius credential: %v", err)
}
for label, output := range map[string]string{"show": show.String(), "sources": sources.String(), "artifacts": catalog.String(), "diff": diff.String()} {
if strings.Contains(output, firstCanary) || strings.Contains(output, secondCanary) {
t.Fatalf("%s output exposed credential value: %q", label, output)
}
}
for _, reference := range []string{"NARRATIO_INSPECT_ACCESS", "NARRATIO_INSPECT_SECRET", "NARRATIO_INSPECT_LLM_ONE"} {
for _, reference := range []string{"NARRATIO_INSPECT_ACCESS", "NARRATIO_INSPECT_SECRET", "NARRATIO_INSPECT_LLM_ONE", "NARRATIO_INSPECT_NOTARIUS_ONE"} {
if !strings.Contains(show.String(), reference) {
t.Fatalf("show omitted safe credential reference %q: %q", reference, show.String())
}
}
if !strings.Contains(diff.String(), "NARRATIO_INSPECT_LLM_ONE") || !strings.Contains(diff.String(), "NARRATIO_INSPECT_LLM_TWO") {
if !strings.Contains(diff.String(), "NARRATIO_INSPECT_LLM_ONE") || !strings.Contains(diff.String(), "NARRATIO_INSPECT_LLM_TWO") ||
!strings.Contains(diff.String(), "NARRATIO_INSPECT_NOTARIUS_ONE") || !strings.Contains(diff.String(), "NARRATIO_INSPECT_NOTARIUS_TWO") {
t.Fatalf("diff omitted safe credential references: %q", diff.String())
}
if !strings.Contains(sources.String(), "notarius.llm_api_key") {
t.Fatalf("sources omitted Notarius credential ownership: %q", sources.String())
}
if strings.Contains(removed.String(), "llm_api_key") || strings.Contains(removed.String(), "NARRATIO_INSPECT_NOTARIUS_ONE") {
t.Fatalf("profile null did not remove Notarius credential: %q", removed.String())
}
}
func TestConfigValidateRejectsUnavailableNotariusCredential(t *testing.T) {
const envName = "NARRATIO_INSPECT_NOTARIUS_MISSING"
restoreEnvAfterTest(t, envName)
path := filepath.Join(t.TempDir(), "pipeline.yml")
mustWriteTestFile(t, path, "whisperx:\n transcribe_url: https://transcription.example.test/transcribe\nnotarius:\n enabled: false\n llm_api_key: "+envName+"\n")
err := ConfigValidate(context.Background(), []string{"--config", path}, io.Discard)
if err == nil || !strings.Contains(err.Error(), "secret.unset") {
t.Fatalf("ConfigValidate() error = %v, want unavailable credential failure", err)
}
}
func TestConfigCommandsInspectMaintainedSplitBundleWithoutRuntimeState(t *testing.T) {

101
internal/app/extract_lifecycle_test.go
View File

@@ -12,6 +12,7 @@ import (
"testing"
"time"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/notarius"
"gitea.maximumdirect.net/eric/narratio/internal/artifactmodel"
"gitea.maximumdirect.net/eric/narratio/internal/artifactpolicy"
@@ -485,6 +486,92 @@ func TestExtractLifecycleSkipsCurrentResumableResult(t *testing.T) {
}
}
func TestExtractLifecycleCredentialChangesRemainResumable(t *testing.T) {
for _, test := range []struct {
name string
before func(*testing.T) *configkit.Secret
after func(*testing.T) *configkit.Secret
}{
{
name: "added",
after: func(t *testing.T) *configkit.Secret {
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_ADDED_KEY", "added-value")
},
},
{
name: "removed",
before: func(t *testing.T) *configkit.Secret {
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_REMOVED_KEY", "removed-value")
},
},
{
name: "renamed",
before: func(t *testing.T) *configkit.Secret {
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_FIRST_KEY", "first-value")
},
after: func(t *testing.T) *configkit.Secret {
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_SECOND_KEY", "second-value")
},
},
{
name: "value rotated",
before: func(t *testing.T) *configkit.Secret {
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_ROTATED_KEY", "first-value")
},
after: func(t *testing.T) *configkit.Secret {
return loadLifecycleNotariusCredential(t, "NARRATIO_LIFECYCLE_ROTATED_KEY", "second-value")
},
},
} {
t.Run(test.name, func(t *testing.T) {
cfg, env, runner := extractionLifecycleFixture(t, true)
if test.before != nil {
cfg.Pipeline.Notarius.LLMAPIKey = test.before(t)
}
analyzeRuns := 0
plan := extractionLifecyclePlan(t, &analyzeRuns)
if _, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env}); err != nil {
t.Fatalf("initial executeStages() error = %v", err)
}
cfg.Pipeline.Notarius.LLMAPIKey = nil
if test.after != nil {
cfg.Pipeline.Notarius.LLMAPIKey = test.after(t)
}
resumed, err := executeStages(context.Background(), cfg, plan, RunOptions{Env: env})
if err != nil {
t.Fatalf("resume executeStages() error = %v", err)
}
if len(resumed.Executed) != 0 || len(resumed.Skipped) != 2 || len(runner.requests) != 1 || analyzeRuns != 1 {
t.Fatalf("credential-only change reexecuted work: summary=%#v requests=%d analyze=%d", resumed, len(runner.requests), analyzeRuns)
}
loaded := loadLifecycleManifest(t, cfg)
if loaded.Stages["extract"].Status != manifest.StatusSucceeded || loaded.Stages["analyze"].Status != manifest.StatusSucceeded {
t.Fatalf("credential-only change invalidated stages: extract=%q analyze=%q", loaded.Stages["extract"].Status, loaded.Stages["analyze"].Status)
}
})
}
}
func TestPreparedPipelineAndManifestExcludeNotariusCredentialValue(t *testing.T) {
const name = "NARRATIO_LIFECYCLE_PERSISTED_KEY"
const canary = "notarius-persisted-credential-canary"
cfg, env, _ := extractionLifecycleFixture(t, true)
cfg.Pipeline.Notarius.LLMAPIKey = loadLifecycleNotariusCredential(t, name, canary)
if _, err := executeStages(context.Background(), cfg, prepareExtractLifecyclePlan(t), RunOptions{Env: env}); err != nil {
t.Fatalf("executeStages() error = %v", err)
}
paths := artifacts.NewLocalStore(cfg.Pipeline.Workspace.Root).SessionPathsFor(cfg.Session.Campaign, cfg.Session.SessionID)
for _, path := range []string{paths.ManifestPath, filepath.Join(paths.InputsDir, "pipeline.resolved.yml")} {
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%q) error = %v", path, err)
}
if strings.Contains(string(data), canary) {
t.Fatalf("persisted Narratio file %q contains the credential value", path)
}
}
}
func TestExtractLifecycleRerunsAfterDirectTranscriptChange(t *testing.T) {
cfg, env, runner := extractionLifecycleFixture(t, true)
analyzeRuns := 0
@@ -761,3 +848,17 @@ func extractionLifecycleFixture(t *testing.T, enabled bool) (*config.Config, *st
runner := &materializingNotariusRunner{cfg: cfg.Pipeline.Notarius}
return cfg, &stage.Env{Notarius: runner}, runner
}
func loadLifecycleNotariusCredential(t *testing.T, name, value string) *configkit.Secret {
t.Helper()
t.Setenv(name, value)
path := filepath.Join(t.TempDir(), "pipeline.yml")
if err := os.WriteFile(path, []byte("whisperx:\n transcribe_url: https://transcription.example.test/transcribe\nnotarius:\n enabled: false\n llm_api_key: "+name+"\n"), 0o644); err != nil {
t.Fatal(err)
}
pipeline, err := config.LoadPipeline(path)
if err != nil {
t.Fatal(err)
}
return pipeline.Notarius.LLMAPIKey
}

49
internal/app/secrets_bootstrap_test.go
View File

@@ -2,6 +2,7 @@ package app
import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"
@@ -119,6 +120,52 @@ func TestSecretsBootstrapPreservesExistingEnvironmentValues(t *testing.T) {
}
}
func TestNotariusCredentialLoadsFromCommandSecretBootstrap(t *testing.T) {
tests := []struct {
name string
explicit bool
environment string
fileValue string
wantValue string
}{
{name: "conventional", fileValue: "conventional-value", wantValue: "conventional-value"},
{name: "explicit", explicit: true, fileValue: "explicit-value", wantValue: "explicit-value"},
{name: "environment precedence", explicit: true, environment: "environment-value", fileValue: "file-value", wantValue: "environment-value"},
}
for index, test := range tests {
t.Run(test.name, func(t *testing.T) {
envName := fmt.Sprintf("NARRATIO_NOTARIUS_BOOTSTRAP_%d", index)
restoreEnvAfterTest(t, envName)
root := t.TempDir()
pipelinePath := filepath.Join(root, "pipeline.yml")
if err := os.WriteFile(pipelinePath, []byte("whisperx:\n transcribe_url: https://transcription.example.test/transcribe\nnotarius:\n enabled: false\n llm_api_key: "+envName+"\n"), 0o644); err != nil {
t.Fatal(err)
}
secretsDir := filepath.Join(root, "secrets")
var selection *string
if test.explicit {
secretsDir = filepath.Join(t.TempDir(), "selected-secrets")
selection = &secretsDir
}
mustWriteSecretFile(t, filepath.Join(secretsDir, envName), test.fileValue+"\n")
if test.environment != "" {
if err := os.Setenv(envName, test.environment); err != nil {
t.Fatal(err)
}
}
_, pipeline, err := loadPipelineConfigWithDependencies(pipelinePath, config.PipelineLoadOptions{}, selection, commandDependencies{diagnostics: io.Discard})
if err != nil {
t.Fatalf("load pipeline: %v", err)
}
if pipeline.Notarius == nil || pipeline.Notarius.LLMAPIKey == nil || !pipeline.Notarius.LLMAPIKey.IsResolved() ||
pipeline.Notarius.LLMAPIKey.EnvName() != envName || pipeline.Notarius.LLMAPIKey.Value() != test.wantValue {
t.Fatal("Notarius credential did not resolve from the selected command bootstrap source")
}
})
}
}
func TestSecretsBootstrapPreservesSafePartialReportOnFailure(t *testing.T) {
directory := t.TempDir()
mustWriteSecretFile(t, filepath.Join(directory, "INVALID-NAME"), "must-not-appear\n")
@@ -173,7 +220,7 @@ func TestExecuteConfigValidateReportsSkippedEmptyBeforeTypedLoadFailure(t *testi
if err != nil {
t.Fatal(err)
}
pipelineYAML = []byte(strings.Replace(string(pipelineYAML), "audita:\n binary:", "audita:\n llm_api_key: "+name+"\n binary:", 1))
pipelineYAML = append(pipelineYAML, []byte("notarius:\n enabled: false\n llm_api_key: "+name+"\n")...)
if err := os.WriteFile(pipelinePath, pipelineYAML, 0o644); err != nil {
t.Fatal(err)
}

51
internal/stage/semantic_contracts_delivery_test.go
View File

@@ -2,6 +2,8 @@ package stage
import (
"context"
"os"
"path/filepath"
"testing"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
@@ -93,6 +95,55 @@ func TestExtractResumeValidationExcludesOperationalSettings(t *testing.T) {
}
}
func TestExtractSemanticConfigExcludesLoadedCredentialChanges(t *testing.T) {
const firstName = "NARRATIO_SEMANTIC_NOTARIUS_KEY"
const secondName = "NARRATIO_SEMANTIC_NOTARIUS_KEY_TWO"
first := loadNotariusCredentialForSemanticTest(t, firstName, "first-value")
rotated := loadNotariusCredentialForSemanticTest(t, firstName, "rotated-value")
renamed := loadNotariusCredentialForSemanticTest(t, secondName, "second-value")
baseline := downstreamSemanticEnv()
baseline.Config.Pipeline.Notarius.LLMAPIKey = first
want, err := (extractStage{}).SemanticConfigFingerprint(baseline)
if err != nil {
t.Fatal(err)
}
for _, test := range []struct {
name string
credential *configkit.Secret
}{
{name: "removed", credential: nil},
{name: "renamed", credential: renamed},
{name: "value rotated", credential: rotated},
} {
t.Run(test.name, func(t *testing.T) {
env := downstreamSemanticEnv()
env.Config.Pipeline.Notarius.LLMAPIKey = test.credential
got, err := (extractStage{}).SemanticConfigFingerprint(env)
if err != nil {
t.Fatal(err)
}
if !want.Equal(got) {
t.Fatalf("credential-only change altered extract semantic fingerprint")
}
})
}
}
func loadNotariusCredentialForSemanticTest(t *testing.T, name, value string) *configkit.Secret {
t.Helper()
t.Setenv(name, value)
path := filepath.Join(t.TempDir(), "pipeline.yml")
if err := os.WriteFile(path, []byte("whisperx:\n transcribe_url: https://transcription.example.test/transcribe\nnotarius:\n enabled: false\n llm_api_key: "+name+"\n"), 0o644); err != nil {
t.Fatal(err)
}
pipeline, err := config.LoadPipeline(path)
if err != nil {
t.Fatal(err)
}
return pipeline.Notarius.LLMAPIKey
}
func TestPublishSemanticConfigSensitivityAndOperationalExclusions(t *testing.T) {
want, err := (publishStage{}).SemanticConfigFingerprint(downstreamSemanticEnv())
if err != nil {