From c7b3af82b492a7c687374aacea09ff79104b6fc0 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 18 Jul 2026 15:59:57 +0000 Subject: [PATCH] Add CLI cache and resume contract tests --- internal/cli/cache_contract_test.go | 364 +++++++++++++++++++++++++++ internal/cli/state_hardening_test.go | 2 +- 2 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 internal/cli/cache_contract_test.go diff --git a/internal/cli/cache_contract_test.go b/internal/cli/cache_contract_test.go new file mode 100644 index 0000000..841b9a6 --- /dev/null +++ b/internal/cli/cache_contract_test.go @@ -0,0 +1,364 @@ +package cli + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" +) + +func TestRunChunkPlanModePrecedenceAndValidation(t *testing.T) { + tests := []struct { + name string + fileMode string + envMode string + cliMode string + wantStores int + }{ + {name: "default", wantStores: 1}, + {name: "file", fileMode: "bypass"}, + {name: "environment", envMode: "bypass"}, + {name: "cli", envMode: "refresh", cliMode: "bypass"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + roots := newStateTestRoots(t) + if tt.name == "default" { + removeStateTestConfigLine(t, roots.config, " mode: auto\n") + } else if tt.fileMode != "" { + replaceStateTestConfigLine(t, roots.config, " mode: auto\n", " mode: "+tt.fileMode+"\n") + } + + var stores []string + opts := newStateTestHarness().options() + opts.LookupEnv = func(name string) (string, bool) { + if name == "NOTARIUS_CACHE_CHUNK_PLANS_MODE" && tt.envMode != "" { + return tt.envMode, true + } + return "", false + } + opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) { + stores = append(stores, root) + return chunkplan.NewFilesystemStore(root) + } + + args := []string{"run", "sample", "--config", roots.config, "--input", roots.input} + if tt.cliMode != "" { + args = append(args, "--chunk_cache", tt.cliMode) + } + var stdout, stderr bytes.Buffer + if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + assertStateTestOutput(t, roots.output) + if len(stores) != tt.wantStores { + t.Fatalf("chunk plan store roots = %v, want %d stores", stores, tt.wantStores) + } + if tt.wantStores == 1 && stores[0] != roots.plans { + t.Fatalf("chunk plan store root = %q, want %q", stores[0], roots.plans) + } + if tt.wantStores == 0 { + assertAbsent(t, roots.plans) + } + }) + } + + t.Run("invalid cli syntax is a usage error", func(t *testing.T) { + roots := newStateTestRoots(t) + var stdout, stderr bytes.Buffer + code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "invalid"}, &stdout, &stderr, newStateTestHarness().options()) + if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + assertNoRunState(t, roots) + }) + + for _, tt := range []struct { + name string + fileConfig bool + }{ + {name: "invalid environment mode"}, + {name: "invalid file mode", fileConfig: true}, + } { + t.Run(tt.name, func(t *testing.T) { + roots := newStateTestRoots(t) + opts := newStateTestHarness().options() + if tt.fileConfig { + replaceStateTestConfigLine(t, roots.config, " mode: auto\n", " mode: invalid\n") + } else { + opts.LookupEnv = func(name string) (string, bool) { + if name == "NOTARIUS_CACHE_CHUNK_PLANS_MODE" { + return "invalid", true + } + return "", false + } + } + var stdout, stderr bytes.Buffer + code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts) + if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + assertNoRunState(t, roots) + }) + } +} + +func TestRunChunkPlanRootSelectionAndFailures(t *testing.T) { + t.Run("empty configured root uses the per-user cache root", func(t *testing.T) { + roots := newStateTestRoots(t) + removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans)) + userCache := filepath.Join(t.TempDir(), "user-cache") + var stores []string + opts := newStateTestHarness().options() + opts.UserCacheDir = func() (string, error) { return userCache, nil } + opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) { + stores = append(stores, root) + return chunkplan.NewFilesystemStore(root) + } + var stdout, stderr bytes.Buffer + code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts) + if code != 0 { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + wantRoot := filepath.Join(userCache, "notarius", "chunk-plans") + if len(stores) != 1 || stores[0] != wantRoot { + t.Fatalf("chunk plan store roots = %v, want [%q]", stores, wantRoot) + } + assertFile(t, filepath.Join(wantRoot, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json")) + assertAbsent(t, roots.plans) + }) + + t.Run("bypass avoids default cache dependencies", func(t *testing.T) { + roots := newStateTestRoots(t) + removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans)) + userCacheCalls := 0 + storeCalls := 0 + opts := newStateTestHarness().options() + opts.UserCacheDir = func() (string, error) { + userCacheCalls++ + return "", errors.New("user cache must not be resolved") + } + opts.ChunkPlanStoreFactory = func(string) (pipeline.ChunkPlanStore, error) { + storeCalls++ + return nil, errors.New("chunk plan store must not be constructed") + } + var stdout, stderr bytes.Buffer + code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts) + if code != 0 { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + if userCacheCalls != 0 || storeCalls != 0 { + t.Fatalf("user cache calls=%d store calls=%d, want none", userCacheCalls, storeCalls) + } + assertStateTestOutput(t, roots.output) + assertAbsent(t, roots.plans) + }) + + t.Run("user cache resolution failure has context and no output", func(t *testing.T) { + roots := newStateTestRoots(t) + removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans)) + opts := newStateTestHarness().options() + opts.UserCacheDir = func() (string, error) { return "", errors.New("cache home unavailable") } + var stdout, stderr bytes.Buffer + code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts) + if code != 1 || !strings.Contains(stderr.String(), "resolve chunk plan root") || !strings.Contains(stderr.String(), "cache home unavailable") { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + assertNoRunState(t, roots) + }) + + t.Run("store construction failure has context and no output", func(t *testing.T) { + roots := newStateTestRoots(t) + opts := newStateTestHarness().options() + opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) { + return nil, fmt.Errorf("store unavailable") + } + var stdout, stderr bytes.Buffer + code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts) + want := fmt.Sprintf("create chunk plan store at %q", roots.plans) + if code != 1 || !strings.Contains(stderr.String(), want) || !strings.Contains(stderr.String(), "store unavailable") { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + assertNoRunState(t, roots) + }) + + t.Run("checkpoint root resolution failure has context and no output", func(t *testing.T) { + roots := newStateTestRoots(t) + removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints)) + opts := newStateTestHarness().options() + opts.UserCacheDir = func() (string, error) { return "", errors.New("checkpoint cache unavailable") } + result := runStateTest(t, roots, opts, false, true, "bypass") + if result.code != 1 || !strings.Contains(result.stderr, "resolve checkpoint root") || !strings.Contains(result.stderr, "checkpoint cache unavailable") { + t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr) + } + assertNoRunState(t, roots) + }) +} + +func TestRunAutoReusesPlanWhenRunInputsChange(t *testing.T) { + roots := newStateTestRoots(t) + data, err := os.ReadFile(roots.config) + if err != nil { + t.Fatal(err) + } + configText := strings.Replace(string(data), " chunk: test/chunk\n", ` chunk: + module: test/chunk + options: + strategy: first +`, 1) + configText = strings.Replace(configText, " output: test/output\n", ` other: + extract: test/extract + merge: test/merge + normalize: test/normalize + output: test/output +`, 1) + if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil { + t.Fatal(err) + } + referencePath := filepath.Join(filepath.Dir(roots.input), "reference.txt") + if err := os.WriteFile(referencePath, []byte("reference content"), 0o600); err != nil { + t.Fatal(err) + } + harness := newStateTestHarness() + var firstStdout, firstStderr bytes.Buffer + first := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &firstStdout, &firstStderr, harness.options()) + if first != 0 { + t.Fatalf("first run code=%d stdout=%q stderr=%q", first, firstStdout.String(), firstStderr.String()) + } + configData, err := os.ReadFile(roots.config) + if err != nil { + t.Fatal(err) + } + configText = strings.Replace(string(configData), "strategy: first", "strategy: second", 1) + if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil { + t.Fatal(err) + } + var stdout, stderr bytes.Buffer + second := RunWithOptions([]string{ + "run", "sample", "--config", roots.config, "--input", roots.input, + "--only", "items", "--reference", "chunk.cache-reference=" + referencePath, + }, &stdout, &stderr, harness.options()) + if second != 0 { + t.Fatalf("second run code=%d stdout=%q stderr=%q", second, stdout.String(), stderr.String()) + } + harness.mu.Lock() + chunkCalls := harness.chunkCalls + harness.mu.Unlock() + if chunkCalls != 1 { + t.Fatalf("chunk calls across changed run inputs = %d, want 1", chunkCalls) + } + assertFile(t, filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json")) + assertAnyFile(t, roots.output) +} + +func TestRunResumeSelectsConfiguredOrPerUserCheckpointRoot(t *testing.T) { + for _, configured := range []bool{true, false} { + name := "per-user root" + if configured { + name = "configured root" + } + t.Run(name, func(t *testing.T) { + roots := newStateTestRoots(t) + if !configured { + removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints)) + } + userCache := filepath.Join(t.TempDir(), "user-cache") + userCacheCalls := 0 + opts := newStateTestHarness().options() + opts.UserCacheDir = func() (string, error) { + userCacheCalls++ + return userCache, nil + } + result := runStateTest(t, roots, opts, false, true, "bypass") + if result.code != 0 { + t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr) + } + wantRoot := roots.checkpoints + wantCalls := 0 + if !configured { + wantRoot = filepath.Join(userCache, "notarius", "checkpoints") + wantCalls = 1 + } + if userCacheCalls != wantCalls { + t.Fatalf("user cache calls = %d, want %d", userCacheCalls, wantCalls) + } + assertAnyFile(t, wantRoot) + if !configured { + assertAbsent(t, roots.checkpoints) + } + assertStateTestOutput(t, roots.output) + }) + } + + t.Run("without resume avoids checkpoint root resolution", func(t *testing.T) { + roots := newStateTestRoots(t) + removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints)) + opts := newStateTestHarness().options() + opts.UserCacheDir = func() (string, error) { return "", errors.New("checkpoint cache must not be resolved") } + result := runStateTest(t, roots, opts, false, false, "bypass") + if result.code != 0 { + t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr) + } + assertStateTestOutput(t, roots.output) + assertAbsent(t, roots.checkpoints) + }) +} + +func TestConfigCommandsDoNotResolveRunState(t *testing.T) { + for _, args := range [][]string{ + {"config", "validate", "--config"}, + {"pipelines", "list", "--config"}, + } { + name := strings.Join(args[:2], "-") + t.Run(name, func(t *testing.T) { + roots := newStateTestRoots(t) + opts := newStateTestHarness().options() + opts.UserCacheDir = func() (string, error) { return "", errors.New("state root must not be resolved") } + opts.ChunkPlanStoreFactory = func(string) (pipeline.ChunkPlanStore, error) { + return nil, errors.New("chunk plan store must not be constructed") + } + command := append([]string(nil), args...) + command = append(command, roots.config) + var stdout, stderr bytes.Buffer + code := RunWithOptions(command, &stdout, &stderr, opts) + if code != 0 { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + assertNoRunState(t, roots) + }) + } +} + +func replaceStateTestConfigLine(t *testing.T, path, old, new string) { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + text := string(data) + if !strings.Contains(text, old) { + t.Fatalf("config %q does not contain %q", path, old) + } + text = strings.Replace(text, old, new, 1) + if err := os.WriteFile(path, []byte(text), 0o600); err != nil { + t.Fatal(err) + } +} + +func removeStateTestConfigLine(t *testing.T, path, line string) { + replaceStateTestConfigLine(t, path, line, "") +} + +func assertNoRunState(t *testing.T, roots stateTestRoots) { + t.Helper() + assertAbsent(t, roots.output) + assertAbsent(t, roots.plans) + assertAbsent(t, roots.checkpoints) + assertAbsent(t, roots.debug) +} diff --git a/internal/cli/state_hardening_test.go b/internal/cli/state_hardening_test.go index 5315be6..0dd7399 100644 --- a/internal/cli/state_hardening_test.go +++ b/internal/cli/state_hardening_test.go @@ -823,7 +823,7 @@ func (h *stateTestHarness) options() Options { if err := registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "test/input", Stage: pipeline.StageInput, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) { return stateTestInput{}, nil }); err != nil { panic(err) } - if err := registries.Chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/chunk", Stage: pipeline.StageChunk, Requires: []string{"source"}, Provides: []string{"chunks"}}, func() (contracts.Chunker, error) { return stateTestChunker{h}, nil }); err != nil { + if err := registries.Chunkers.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "test/chunk", Stage: pipeline.StageChunk, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "cache-reference"}}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.Chunker, error) { return stateTestChunker{h}, nil }); err != nil { panic(err) } if err := pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "test/extract", Stage: pipeline.StageExtract, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{h}, nil }); err != nil {