386 lines
14 KiB
Go
386 lines
14 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
)
|
|
|
|
func TestRunChunkPlanModePrecedenceAndValidation(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
fileMode string
|
|
envMode string
|
|
cliMode string
|
|
wantStores int
|
|
}{
|
|
{name: "default", wantStores: 1},
|
|
{name: "file", fileMode: "bypass"},
|
|
{name: "environment", envMode: "bypass"},
|
|
{name: "cli", envMode: "refresh", cliMode: "bypass"},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
if tt.name == "default" {
|
|
removeStateTestConfigLine(t, roots.config, " mode: auto\n")
|
|
} else if tt.fileMode != "" {
|
|
replaceStateTestConfigLine(t, roots.config, " mode: auto\n", " mode: "+tt.fileMode+"\n")
|
|
}
|
|
|
|
var stores []string
|
|
opts := newStateTestHarness().options()
|
|
opts.LookupEnv = func(name string) (string, bool) {
|
|
if name == "NOTARIUS_CACHE_CHUNK_PLANS_MODE" && tt.envMode != "" {
|
|
return tt.envMode, true
|
|
}
|
|
return "", false
|
|
}
|
|
opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) {
|
|
stores = append(stores, root)
|
|
return chunkplan.NewFilesystemStore(root)
|
|
}
|
|
|
|
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input}
|
|
if tt.cliMode != "" {
|
|
args = append(args, "--chunk_cache", tt.cliMode)
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
assertStateTestOutput(t, roots.output)
|
|
if len(stores) != tt.wantStores {
|
|
t.Fatalf("chunk plan store roots = %v, want %d stores", stores, tt.wantStores)
|
|
}
|
|
if tt.wantStores == 1 && stores[0] != roots.plans {
|
|
t.Fatalf("chunk plan store root = %q, want %q", stores[0], roots.plans)
|
|
}
|
|
if tt.wantStores == 0 {
|
|
assertAbsent(t, roots.plans)
|
|
}
|
|
})
|
|
}
|
|
|
|
t.Run("invalid cli syntax is a usage error", func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "invalid"}, &stdout, &stderr, newStateTestHarness().options())
|
|
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
assertNoRunState(t, roots)
|
|
})
|
|
|
|
for _, tt := range []struct {
|
|
name string
|
|
fileConfig bool
|
|
}{
|
|
{name: "invalid environment mode"},
|
|
{name: "invalid file mode", fileConfig: true},
|
|
} {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
opts := newStateTestHarness().options()
|
|
if tt.fileConfig {
|
|
replaceStateTestConfigLine(t, roots.config, " mode: auto\n", " mode: invalid\n")
|
|
} else {
|
|
opts.LookupEnv = func(name string) (string, bool) {
|
|
if name == "NOTARIUS_CACHE_CHUNK_PLANS_MODE" {
|
|
return "invalid", true
|
|
}
|
|
return "", false
|
|
}
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
|
|
if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
assertNoRunState(t, roots)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunChunkPlanRootSelectionAndFailures(t *testing.T) {
|
|
t.Run("empty configured root uses the per-user cache root", func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans))
|
|
userCache := filepath.Join(t.TempDir(), "user-cache")
|
|
var stores []string
|
|
opts := newStateTestHarness().options()
|
|
opts.UserCacheDir = func() (string, error) { return userCache, nil }
|
|
opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) {
|
|
stores = append(stores, root)
|
|
return chunkplan.NewFilesystemStore(root)
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
|
|
if code != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
wantRoot := filepath.Join(userCache, "notarius", "chunk-plans")
|
|
if len(stores) != 1 || stores[0] != wantRoot {
|
|
t.Fatalf("chunk plan store roots = %v, want [%q]", stores, wantRoot)
|
|
}
|
|
assertFile(t, filepath.Join(wantRoot, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json"))
|
|
assertAbsent(t, roots.plans)
|
|
})
|
|
|
|
t.Run("bypass avoids default cache dependencies", func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans))
|
|
userCacheCalls := 0
|
|
storeCalls := 0
|
|
opts := newStateTestHarness().options()
|
|
opts.UserCacheDir = func() (string, error) {
|
|
userCacheCalls++
|
|
return "", errors.New("user cache must not be resolved")
|
|
}
|
|
opts.ChunkPlanStoreFactory = func(string) (pipeline.ChunkPlanStore, error) {
|
|
storeCalls++
|
|
return nil, errors.New("chunk plan store must not be constructed")
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
|
|
if code != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
if userCacheCalls != 0 || storeCalls != 0 {
|
|
t.Fatalf("user cache calls=%d store calls=%d, want none", userCacheCalls, storeCalls)
|
|
}
|
|
assertStateTestOutput(t, roots.output)
|
|
assertAbsent(t, roots.plans)
|
|
})
|
|
|
|
t.Run("user cache resolution failure has context and no output", func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.plans))
|
|
opts := newStateTestHarness().options()
|
|
opts.UserCacheDir = func() (string, error) { return "", errors.New("cache home unavailable") }
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
|
|
if code != 1 || !strings.Contains(stderr.String(), "resolve chunk plan root") || !strings.Contains(stderr.String(), "cache home unavailable") {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
assertNoRunState(t, roots)
|
|
})
|
|
|
|
t.Run("store construction failure has context and no output", func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
opts := newStateTestHarness().options()
|
|
opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) {
|
|
return nil, fmt.Errorf("store unavailable")
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &stdout, &stderr, opts)
|
|
want := fmt.Sprintf("create chunk plan store at %q", roots.plans)
|
|
if code != 1 || !strings.Contains(stderr.String(), want) || !strings.Contains(stderr.String(), "store unavailable") {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
assertNoRunState(t, roots)
|
|
})
|
|
|
|
t.Run("checkpoint root resolution failure has context and no output", func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints))
|
|
opts := newStateTestHarness().options()
|
|
opts.UserCacheDir = func() (string, error) { return "", errors.New("checkpoint cache unavailable") }
|
|
result := runStateTest(t, roots, opts, false, true, "bypass")
|
|
if result.code != 1 || !strings.Contains(result.stderr, "resolve checkpoint root") || !strings.Contains(result.stderr, "checkpoint cache unavailable") {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
|
}
|
|
assertNoRunState(t, roots)
|
|
})
|
|
}
|
|
|
|
func TestRunAutoReusesPlanWhenRunInputsChange(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
data, err := os.ReadFile(roots.config)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
configText := replaceRequiredOnce(t, string(data), " chunk: test/chunk\n", ` chunk:
|
|
module: test/chunk
|
|
options:
|
|
strategy: first
|
|
`)
|
|
configText = replaceRequiredOnce(t, configText, " output: test/output\n", ` other:
|
|
extract: test/extract
|
|
merge: test/merge
|
|
normalize: test/normalize
|
|
output: test/output
|
|
`)
|
|
if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
referencePath := filepath.Join(filepath.Dir(roots.input), "reference.txt")
|
|
if err := os.WriteFile(referencePath, []byte("reference content"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
harness := newStateTestHarness()
|
|
var firstStdout, firstStderr bytes.Buffer
|
|
first := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, &firstStdout, &firstStderr, harness.options())
|
|
if first != 0 {
|
|
t.Fatalf("first run code=%d stdout=%q stderr=%q", first, firstStdout.String(), firstStderr.String())
|
|
}
|
|
configData, err := os.ReadFile(roots.config)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
configText = replaceRequiredOnce(t, string(configData), "strategy: first", "strategy: second")
|
|
if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
second := RunWithOptions([]string{
|
|
"run", "sample", "--config", roots.config, "--input", roots.input,
|
|
"--only", "items", "--reference", "chunk.cache-reference=" + referencePath,
|
|
}, &stdout, &stderr, harness.options())
|
|
if second != 0 {
|
|
t.Fatalf("second run code=%d stdout=%q stderr=%q", second, stdout.String(), stderr.String())
|
|
}
|
|
harness.mu.Lock()
|
|
chunkCalls := harness.chunkCalls
|
|
sessions := append([]string(nil), harness.sessionIDs...)
|
|
harness.mu.Unlock()
|
|
if chunkCalls != 1 {
|
|
t.Fatalf("chunk calls across changed run inputs = %d, want 1", chunkCalls)
|
|
}
|
|
rawInput, err := os.ReadFile(roots.input)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wantSessionID, err := resolvePromptSessionID("", "test/input", rawInput)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, sessionID := range sessions {
|
|
if sessionID != wantSessionID {
|
|
t.Fatalf("session IDs across reference changes = %#v, want %q", sessions, wantSessionID)
|
|
}
|
|
}
|
|
assertFile(t, filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json"))
|
|
assertAnyFile(t, roots.output)
|
|
}
|
|
|
|
func TestRunResumeSelectsConfiguredOrPerUserCheckpointRoot(t *testing.T) {
|
|
for _, configured := range []bool{true, false} {
|
|
name := "per-user root"
|
|
if configured {
|
|
name = "configured root"
|
|
}
|
|
t.Run(name, func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
if !configured {
|
|
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints))
|
|
}
|
|
userCache := filepath.Join(t.TempDir(), "user-cache")
|
|
userCacheCalls := 0
|
|
opts := newStateTestHarness().options()
|
|
opts.UserCacheDir = func() (string, error) {
|
|
userCacheCalls++
|
|
return userCache, nil
|
|
}
|
|
result := runStateTest(t, roots, opts, false, true, "bypass")
|
|
if result.code != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
|
}
|
|
wantRoot := roots.checkpoints
|
|
wantCalls := 0
|
|
if !configured {
|
|
wantRoot = filepath.Join(userCache, "notarius", "checkpoints")
|
|
wantCalls = 1
|
|
}
|
|
if userCacheCalls != wantCalls {
|
|
t.Fatalf("user cache calls = %d, want %d", userCacheCalls, wantCalls)
|
|
}
|
|
assertAnyFile(t, wantRoot)
|
|
if !configured {
|
|
assertAbsent(t, roots.checkpoints)
|
|
}
|
|
assertStateTestOutput(t, roots.output)
|
|
})
|
|
}
|
|
|
|
t.Run("disabled avoids checkpoint root resolution", func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
replaceStateTestConfigLine(t, roots.config, " enabled: true\n", " enabled: false\n")
|
|
removeStateTestConfigLine(t, roots.config, fmt.Sprintf(" directory: %q\n", roots.checkpoints))
|
|
opts := newStateTestHarness().options()
|
|
opts.UserCacheDir = func() (string, error) { return "", errors.New("checkpoint cache must not be resolved") }
|
|
result := runStateTest(t, roots, opts, false, false, "bypass")
|
|
if result.code != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
|
}
|
|
assertStateTestOutput(t, roots.output)
|
|
assertAbsent(t, roots.checkpoints)
|
|
})
|
|
|
|
t.Run("resume requires enabled checkpoint recording", func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
replaceStateTestConfigLine(t, roots.config, " enabled: true\n", " enabled: false\n")
|
|
result := runStateTest(t, roots, newStateTestHarness().options(), true, true, "bypass")
|
|
if result.code != 1 || !strings.Contains(result.stderr, "--resume requires cache.checkpoints.enabled: true") {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
|
}
|
|
assertNoRunState(t, roots)
|
|
})
|
|
}
|
|
|
|
func TestConfigCommandsDoNotResolveRunState(t *testing.T) {
|
|
for _, args := range [][]string{
|
|
{"config", "validate", "--config"},
|
|
{"pipelines", "list", "--config"},
|
|
} {
|
|
name := strings.Join(args[:2], "-")
|
|
t.Run(name, func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
opts := newStateTestHarness().options()
|
|
opts.UserCacheDir = func() (string, error) { return "", errors.New("state root must not be resolved") }
|
|
opts.ChunkPlanStoreFactory = func(string) (pipeline.ChunkPlanStore, error) {
|
|
return nil, errors.New("chunk plan store must not be constructed")
|
|
}
|
|
command := append([]string(nil), args...)
|
|
command = append(command, roots.config)
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions(command, &stdout, &stderr, opts)
|
|
if code != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
assertNoRunState(t, roots)
|
|
})
|
|
}
|
|
}
|
|
|
|
func replaceStateTestConfigLine(t *testing.T, path, old, new string) {
|
|
t.Helper()
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
text := replaceRequiredOnce(t, string(data), old, new)
|
|
if err := os.WriteFile(path, []byte(text), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func removeStateTestConfigLine(t *testing.T, path, line string) {
|
|
replaceStateTestConfigLine(t, path, line, "")
|
|
}
|
|
|
|
func assertNoRunState(t *testing.T, roots stateTestRoots) {
|
|
t.Helper()
|
|
assertAbsent(t, roots.output)
|
|
assertAbsent(t, roots.plans)
|
|
assertAbsent(t, roots.checkpoints)
|
|
assertAbsent(t, roots.debug)
|
|
}
|