From 26142f0e058f321a4b1480de1e567e4202e70374 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 18 Jul 2026 12:48:21 +0000 Subject: [PATCH] Remove legacy workspace and diagnostics implementation --- docs/roadmap/implementation.md | 2 +- internal/cli/chunk_cache_test.go | 498 -- internal/cli/compatibility_test.go | 602 --- internal/cli/run_test.go | 4258 ----------------- internal/cli/test_main_test.go | 10 - internal/core/config/chunk_cache_test.go | 128 - internal/core/config/config_test.go | 85 - internal/core/config/effective_config_test.go | 243 - internal/core/config/env_test.go | 185 - internal/core/config/file_config_test.go | 725 --- internal/core/config/redaction_test.go | 174 - internal/core/config/validation_test.go | 666 --- internal/core/diagnostics/artifacts.go | 15 - internal/core/diagnostics/artifacts_test.go | 24 - internal/core/diagnostics/run_dir.go | 293 -- internal/core/diagnostics/run_dir_test.go | 383 -- internal/core/workspace/files.go | 114 - internal/core/workspace/files_test.go | 148 - internal/core/workspace/settings.go | 54 - internal/core/workspace/settings_test.go | 120 - internal/framework/checkpoint/manifest.go | 2 + 21 files changed, 3 insertions(+), 8726 deletions(-) delete mode 100644 internal/cli/chunk_cache_test.go delete mode 100644 internal/cli/compatibility_test.go delete mode 100644 internal/cli/run_test.go delete mode 100644 internal/cli/test_main_test.go delete mode 100644 internal/core/config/chunk_cache_test.go delete mode 100644 internal/core/config/config_test.go delete mode 100644 internal/core/config/effective_config_test.go delete mode 100644 internal/core/config/env_test.go delete mode 100644 internal/core/config/file_config_test.go delete mode 100644 internal/core/config/redaction_test.go delete mode 100644 internal/core/config/validation_test.go delete mode 100644 internal/core/diagnostics/artifacts.go delete mode 100644 internal/core/diagnostics/artifacts_test.go delete mode 100644 internal/core/diagnostics/run_dir.go delete mode 100644 internal/core/diagnostics/run_dir_test.go delete mode 100644 internal/core/workspace/files.go delete mode 100644 internal/core/workspace/files_test.go delete mode 100644 internal/core/workspace/settings.go delete mode 100644 internal/core/workspace/settings_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 22d0e12..5da8b3f 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -469,7 +469,7 @@ and CLI tests use the new public contract; and the repository-wide checks pass. ## Stage 4: Remove legacy workspace and diagnostics implementation -**Status:** Not started +**Status:** Complete ### Objective diff --git a/internal/cli/chunk_cache_test.go b/internal/cli/chunk_cache_test.go deleted file mode 100644 index 5deac5c..0000000 --- a/internal/cli/chunk_cache_test.go +++ /dev/null @@ -1,498 +0,0 @@ -//go:build legacy - -package cli - -import ( - "bytes" - "errors" - "os" - "path/filepath" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" - "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" -) - -type recordingChunkPlanStore struct { - record pipeline.ChunkPlanRecord - decision pipeline.ChunkPlanDecision - loadErr error - saveErr error - loads int - saves int -} - -func (s *recordingChunkPlanStore) Load(string) (pipeline.ChunkPlanRecord, pipeline.ChunkPlanDecision, error) { - s.loads++ - return s.record, s.decision, s.loadErr -} - -func (s *recordingChunkPlanStore) Save(record pipeline.ChunkPlanRecord) error { - s.saves++ - s.record = record - return s.saveErr -} - -type recordingChunkPlanFactory struct { - roots []string - store *recordingChunkPlanStore - err error -} - -func (f *recordingChunkPlanFactory) build(root string) (pipeline.ChunkPlanStore, error) { - f.roots = append(f.roots, root) - if f.err != nil { - return nil, f.err - } - if f.store == nil { - f.store = &recordingChunkPlanStore{decision: pipeline.ChunkPlanDecision{Status: pipeline.ChunkPlanMissing}} - } - return f.store, nil -} - -func TestRunChunkCachePrecedence(t *testing.T) { - tests := []struct { - name string - fileMode string - envMode string - flagMode string - wantMode pipeline.ChunkCacheMode - wantBuild bool - }{ - {name: "file refresh", fileMode: "refresh", wantMode: pipeline.ChunkCacheRefresh, wantBuild: true}, - {name: "environment over file", fileMode: "bypass", envMode: "refresh", wantMode: pipeline.ChunkCacheRefresh, wantBuild: true}, - {name: "flag over environment", fileMode: "refresh", envMode: "auto", flagMode: "bypass", wantMode: pipeline.ChunkCacheBypass}, - {name: "explicit auto", fileMode: "bypass", envMode: "refresh", flagMode: "auto", wantMode: pipeline.ChunkCacheAuto, wantBuild: true}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - root := filepath.Join(t.TempDir(), "plans") - configPath := writeTestConfig(t, cacheTestConfig(tc.fileMode, root, "")) - inputPath := writeFile(t, "input.txt", "input") - values := map[string]string{} - if tc.envMode != "" { - values["NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE"] = tc.envMode - } - factory := &recordingChunkPlanFactory{} - args := []string{"run", "example", "--config", configPath, "--input", inputPath, "--output-dir", t.TempDir()} - if tc.flagMode != "" { - args = append(args, "--chunk_cache", tc.flagMode) - } - code, stderr := runCacheCommand(t, args, cacheTestOptions(t, values, factory)) - if code != 0 { - t.Fatalf("code = %d stderr = %q", code, stderr) - } - if got := len(factory.roots) > 0; got != tc.wantBuild { - t.Fatalf("store built = %t roots = %#v, want %t", got, factory.roots, tc.wantBuild) - } - if !tc.wantBuild { - return - } - if tc.wantMode == pipeline.ChunkCacheAuto && (factory.store.loads != 1 || factory.store.saves != 1) { - t.Fatalf("auto calls = load %d save %d", factory.store.loads, factory.store.saves) - } - if tc.wantMode == pipeline.ChunkCacheRefresh && (factory.store.loads != 0 || factory.store.saves != 1) { - t.Fatalf("refresh calls = load %d save %d", factory.store.loads, factory.store.saves) - } - }) - } -} - -func TestRunChunkCacheInvalidValuesHaveEstablishedExitCodes(t *testing.T) { - validConfig := writeTestConfig(t, cacheTestConfig("bypass", "", "")) - invalidFile := writeTestConfig(t, cacheTestConfig("sometimes", "", "")) - inputPath := writeFile(t, "input.txt", "input") - tests := []struct { - name string - config string - env map[string]string - flag string - want int - }{ - {name: "flag", config: validConfig, flag: "sometimes", want: 2}, - {name: "environment", config: validConfig, env: map[string]string{"NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE": "sometimes"}, want: 1}, - {name: "file", config: invalidFile, want: 1}, - {name: "flag does not mask invalid file", config: invalidFile, flag: "bypass", want: 1}, - {name: "flag does not mask invalid environment", config: validConfig, env: map[string]string{"NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE": "sometimes"}, flag: "bypass", want: 1}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - args := []string{"run", "example", "--config", tc.config, "--input", inputPath, "--output-dir", t.TempDir()} - if tc.flag != "" { - args = append(args, "--chunk_cache", tc.flag) - } - code, _ := runCacheCommand(t, args, cacheTestOptions(t, tc.env, &recordingChunkPlanFactory{})) - if code != tc.want { - t.Fatalf("code = %d, want %d", code, tc.want) - } - }) - } -} - -func TestRunChunkPlanRootResolution(t *testing.T) { - t.Run("explicit file root", func(t *testing.T) { - factory := &recordingChunkPlanFactory{} - resolverCalls := 0 - opts := cacheTestOptions(t, nil, factory) - opts.UserCacheDir = func() (string, error) { resolverCalls++; return "", errors.New("must not be called") } - configPath := writeTestConfig(t, cacheTestConfig("auto", "/var/cache/notarius/chunk-plans", "")) - code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), opts) - if code != 0 || stderr != "" || resolverCalls != 0 || len(factory.roots) != 1 || factory.roots[0] != "/var/cache/notarius/chunk-plans" { - t.Fatalf("code=%d stderr=%q resolver=%d roots=%#v", code, stderr, resolverCalls, factory.roots) - } - }) - - t.Run("environment root", func(t *testing.T) { - factory := &recordingChunkPlanFactory{} - environmentRoot := filepath.Join(t.TempDir(), "environment-plans") - configPath := writeTestConfig(t, cacheTestConfig("auto", filepath.Join(t.TempDir(), "file-plans"), "")) - code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), cacheTestOptions(t, map[string]string{"NOTARIUS_WORKSPACE_CHUNK_CACHE_DIR": environmentRoot}, factory)) - if code != 0 || stderr != "" || len(factory.roots) != 1 || factory.roots[0] != environmentRoot { - t.Fatalf("code=%d stderr=%q roots=%#v", code, stderr, factory.roots) - } - }) - - t.Run("per-user default", func(t *testing.T) { - factory := &recordingChunkPlanFactory{} - cacheDir := filepath.Join(t.TempDir(), "user-cache") - opts := cacheTestOptions(t, nil, factory) - opts.UserCacheDir = func() (string, error) { return cacheDir, nil } - configPath := writeTestConfig(t, cacheTestConfig("auto", "", "")) - code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), opts) - want := filepath.Join(cacheDir, "notarius", "chunk-plans") - if code != 0 || stderr != "" || len(factory.roots) != 1 || factory.roots[0] != want { - t.Fatalf("code=%d stderr=%q roots=%#v want=%q", code, stderr, factory.roots, want) - } - }) -} - -func TestRunBypassSkipsRootAndStore(t *testing.T) { - factory := &recordingChunkPlanFactory{err: errors.New("must not build")} - resolverCalls := 0 - opts := cacheTestOptions(t, nil, factory) - opts.UserCacheDir = func() (string, error) { resolverCalls++; return "", errors.New("must not resolve") } - configPath := writeTestConfig(t, cacheTestConfig("bypass", "", "")) - code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), opts) - if code != 0 || stderr != "" || resolverCalls != 0 || len(factory.roots) != 0 { - t.Fatalf("code=%d stderr=%q resolver=%d roots=%#v", code, stderr, resolverCalls, factory.roots) - } -} - -func TestRunChunkPlanSetupFailures(t *testing.T) { - for _, mode := range []string{"auto", "refresh"} { - t.Run(mode+" resolver", func(t *testing.T) { - factory := &recordingChunkPlanFactory{} - opts := cacheTestOptions(t, nil, factory) - opts.UserCacheDir = func() (string, error) { return "", errors.New("cache unavailable") } - configPath := writeTestConfig(t, cacheTestConfig(mode, "", "")) - code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), opts) - if code != 1 || !strings.Contains(stderr, "cache unavailable") || len(factory.roots) != 0 { - t.Fatalf("code=%d stderr=%q roots=%#v", code, stderr, factory.roots) - } - }) - } - - t.Run("store", func(t *testing.T) { - factory := &recordingChunkPlanFactory{err: errors.New("unwritable store")} - configPath := writeTestConfig(t, cacheTestConfig("auto", t.TempDir(), "")) - code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), cacheTestOptions(t, nil, factory)) - if code != 1 || !strings.Contains(stderr, "unwritable store") { - t.Fatalf("code=%d stderr=%q", code, stderr) - } - }) - - t.Run("unwritable filesystem store", func(t *testing.T) { - root := writeFile(t, "not-a-directory", "occupied") - configPath := writeTestConfig(t, `version: 2 -workspace: - chunk_cache: - mode: auto - directory: `+root+` -pipelines: - dnd-session: - input: seriatim - artifacts: - spells: - extract: dnd/spells -`) - code, stderr := runCacheCommand(t, []string{"run", "dnd-session", "--config", configPath, "--input", writeSeriatimInput(t), "--output-dir", t.TempDir()}, Options{LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), LookupEnv: mapLookup(nil)}) - if code != 1 || (!strings.Contains(stderr, "load chunk plan") && !strings.Contains(stderr, "save chunk plan")) { - t.Fatalf("code=%d stderr=%q", code, stderr) - } - }) -} - -func TestConfigCommandsDoNotResolveOrCreateChunkPlanState(t *testing.T) { - configPath := writeTestConfig(t, cacheTestConfig("auto", "", "")) - for _, args := range [][]string{ - {"config", "validate", "--config", configPath}, - {"pipelines", "list", "--config", configPath}, - } { - factory := &recordingChunkPlanFactory{err: errors.New("must not build")} - resolverCalls := 0 - opts := cacheTestOptions(t, nil, factory) - opts.UserCacheDir = func() (string, error) { resolverCalls++; return "", errors.New("must not resolve") } - code, stderr := runCacheCommand(t, args, opts) - if code != 0 || stderr != "" || resolverCalls != 0 || len(factory.roots) != 0 { - t.Fatalf("args=%v code=%d stderr=%q resolver=%d roots=%#v", args, code, stderr, resolverCalls, factory.roots) - } - } -} - -func TestRunRecordsExplicitChunkCacheOverrideAndEffectiveMode(t *testing.T) { - diagnosticsDir := t.TempDir() - configPath := writeTestConfig(t, cacheTestConfig("bypass", "", diagnosticsDir)) - factory := &recordingChunkPlanFactory{} - args := append(cacheRunArgs(t, configPath), "--chunk_cache", "refresh") - code, stderr := runCacheCommand(t, args, cacheTestOptions(t, nil, factory)) - if code != 0 || stderr != "" { - t.Fatalf("code=%d stderr=%q", code, stderr) - } - runDir := onlyChildDir(t, diagnosticsDir) - invocation := string(readFile(t, filepath.Join(runDir, diagnostics.ArtifactInvocationMetadata))) - effective := string(readFile(t, filepath.Join(runDir, diagnostics.ArtifactEffectiveConfig))) - if !strings.Contains(invocation, `"chunk_cache_override": "refresh"`) || !strings.Contains(effective, `"mode": "refresh"`) { - t.Fatalf("invocation=%s effective=%s", invocation, effective) - } -} - -func TestRunChunkPlanDiagnosticsRedactStoreDecisionReason(t *testing.T) { - diagnosticsDir := t.TempDir() - configPath := writeTestConfig(t, cacheTestConfig("auto", t.TempDir(), diagnosticsDir)) - factory := &recordingChunkPlanFactory{store: &recordingChunkPlanStore{ - decision: pipeline.ChunkPlanDecision{Status: pipeline.ChunkPlanInvalid, Reason: "SENTINEL_INVALID_RECORD_CONTENT"}, - }} - code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), cacheTestOptions(t, nil, factory)) - if code != 0 || stderr != "" { - t.Fatalf("code=%d stderr=%q", code, stderr) - } - summary := string(readFile(t, filepath.Join(onlyChildDir(t, diagnosticsDir), diagnostics.ArtifactChunkPlan))) - if strings.Contains(summary, "SENTINEL_INVALID_RECORD_CONTENT") || !strings.Contains(summary, `"lookup_reason": "stored chunk plan is invalid"`) { - t.Fatalf("chunk plan diagnostic = %s", summary) - } -} - -func TestDefaultAutoReusesPlanAcrossIndependentInvocations(t *testing.T) { - cacheBase := filepath.Join(t.TempDir(), "cache") - workspaceDir := filepath.Join(t.TempDir(), "workspace") - configPath := writeTestConfig(t, `version: 2 -workspace: - directory: `+workspaceDir+` - debug: - enabled: true -pipelines: - dnd-session: - input: seriatim - chunk: dnd/scenes - artifacts: - spells: - extract: dnd/spells -`) - inputPath := writeFile(t, "source.json", `{ - "metadata": {"id": "session-alpha"}, - "segments": [ - {"id": 1, "start": 0, "end": 1, "speaker": "Aria", "text": "Aria casts Cure Wounds."}, - {"id": 2, "start": 1, "end": 2, "speaker": "Borin", "text": "Borin recovers."} - ] -}`) - client := newFakeRunLLMClient(false) - opts := Options{ - LLMClientFactory: fakeLLMFactory(client, nil), - LookupEnv: mapLookup(nil), - UserCacheDir: func() (string, error) { return cacheBase, nil }, - } - for i := 0; i < 2; i++ { - code, stderr := runCacheCommand(t, []string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", t.TempDir()}, opts) - if code != 0 { - t.Fatalf("run %d code=%d stderr=%q", i+1, code, stderr) - } - } - if client.calls != 3 { - t.Fatalf("LLM calls = %d, want chunk+extract then extract-only reuse", client.calls) - } - planRoot := filepath.Join(cacheBase, "notarius", "chunk-plans") - entries, err := os.ReadDir(planRoot) - if err != nil || len(entries) != 1 { - t.Fatalf("plan root entries = %v error=%v", entries, err) - } - if _, err := os.Stat(filepath.Join(planRoot, entries[0].Name(), "plan.json")); err != nil { - t.Fatalf("plan file: %v", err) - } - debugRuns := childDirs(t, filepath.Join(workspaceDir, "debug")) - if len(debugRuns) != 2 { - t.Fatalf("debug runs = %#v", debugRuns) - } - attemptCounts := 0 - for _, runDir := range debugRuns { - if _, err := os.Stat(filepath.Join(runDir, "chunk", "attempt-01.json")); err == nil { - attemptCounts++ - } else if !os.IsNotExist(err) { - t.Fatal(err) - } - } - if attemptCounts != 1 { - t.Fatalf("chunk attempt files across runs = %d, want only generating run", attemptCounts) - } -} - -func TestChunkPlanReuseDependsOnlyOnSourceDigest(t *testing.T) { - cacheRoot := filepath.Join(t.TempDir(), "plans") - inputPath := writeSeriatimInput(t) - referencePath := writeFile(t, "players.txt", "Alyx") - profilePath := writeScriptoriumProfileFile(t, "chunk-profile", "http://127.0.0.1:8080/v1", "test-model") - seedConfig := writeTestConfig(t, `version: 2 -workspace: - chunk_cache: - mode: auto - directory: `+cacheRoot+` -pipelines: - seed: - input: seriatim - chunk: - module: generic - options: - max_units: 1 - overlap_units: 0 - artifacts: - spells: - extract: dnd/spells -`) - changedConfig := writeTestConfig(t, `version: 2 -scriptorium: - profile_file: `+profilePath+` -workspace: - chunk_cache: - mode: auto - directory: `+cacheRoot+` -pipelines: - changed: - input: seriatim - chunk: - module: dnd/scenes - llm_profile: chunk-profile - references: - players: `+referencePath+` - validators: - - generic/always_accept - artifacts: - spells: - extract: dnd/spells -`) - client := newFakeRunLLMClient(false) - opts := Options{LLMClientFactory: fakeLLMFactory(client, nil), LookupEnv: mapLookup(nil)} - for _, run := range []struct { - pipeline string - config string - }{{pipeline: "seed", config: seedConfig}, {pipeline: "changed", config: changedConfig}} { - code, stderr := runCacheCommand(t, []string{"run", run.pipeline, "--config", run.config, "--input", inputPath, "--output-dir", t.TempDir()}, opts) - if code != 0 { - t.Fatalf("pipeline %q code=%d stderr=%q", run.pipeline, code, stderr) - } - } - if client.calls != 2 { - t.Fatalf("LLM calls = %d, want one extractor call per run and no changed chunker call", client.calls) - } -} - -func TestResumeAndChunkCacheModesRemainIndependent(t *testing.T) { - for _, mode := range []pipeline.ChunkCacheMode{pipeline.ChunkCacheAuto, pipeline.ChunkCacheBypass, pipeline.ChunkCacheRefresh} { - t.Run(string(mode), func(t *testing.T) { - workspaceDir := filepath.Join(t.TempDir(), "workspace") - cacheRoot := filepath.Join(t.TempDir(), "plans") - configPath := writeTestConfig(t, cacheTestConfigWithWorkspace(string(mode), cacheRoot, workspaceDir)) - factory := &recordingChunkPlanFactory{} - args := append(cacheRunArgs(t, configPath), "--resume") - code, stderr := runCacheCommand(t, args, cacheTestOptions(t, nil, factory)) - if code != 0 || stderr != "" { - t.Fatalf("code=%d stderr=%q", code, stderr) - } - if mode == pipeline.ChunkCacheBypass { - if len(factory.roots) != 0 { - t.Fatalf("bypass roots = %#v", factory.roots) - } - } else if len(factory.roots) != 1 || factory.roots[0] != cacheRoot { - t.Fatalf("roots = %#v, want %q", factory.roots, cacheRoot) - } - if entries := childDirs(t, filepath.Join(workspaceDir, "checkpoints")); len(entries) != 1 { - t.Fatalf("checkpoint roots = %#v", entries) - } - if strings.HasPrefix(cacheRoot, workspaceDir+string(filepath.Separator)) || strings.HasPrefix(workspaceDir, cacheRoot+string(filepath.Separator)) { - t.Fatalf("cache root %q and workspace root %q overlap", cacheRoot, workspaceDir) - } - }) - } -} - -func TestWorkspaceDirectoryDoesNotSelectChunkPlanRoot(t *testing.T) { - cacheBase := filepath.Join(t.TempDir(), "user-cache") - factory := &recordingChunkPlanFactory{} - for _, workspaceDir := range []string{filepath.Join(t.TempDir(), "workspace-one"), filepath.Join(t.TempDir(), "workspace-two")} { - configPath := writeTestConfig(t, cacheTestConfigWithWorkspace("auto", "", workspaceDir)) - opts := cacheTestOptions(t, nil, factory) - opts.UserCacheDir = func() (string, error) { return cacheBase, nil } - code, stderr := runCacheCommand(t, cacheRunArgs(t, configPath), opts) - if code != 0 || stderr != "" { - t.Fatalf("workspace=%q code=%d stderr=%q", workspaceDir, code, stderr) - } - } - want := filepath.Join(cacheBase, "notarius", "chunk-plans") - if len(factory.roots) != 2 || factory.roots[0] != want || factory.roots[1] != want { - t.Fatalf("roots = %#v, want %q twice", factory.roots, want) - } -} - -func cacheTestOptions(t *testing.T, env map[string]string, factory *recordingChunkPlanFactory) Options { - t.Helper() - registries := fakeExecutionRegistries(t) - return Options{ - Catalog: catalogFromRegistries(registries), - Registries: registries, - LLMClientFactory: fakeLLMFactory(nil, nil), - LookupEnv: mapLookup(env), - UserCacheDir: func() (string, error) { return filepath.Join(t.TempDir(), "cache"), nil }, - ChunkPlanStoreFactory: factory.build, - } -} - -func cacheRunArgs(t *testing.T, configPath string) []string { - t.Helper() - return []string{"run", "example", "--config", configPath, "--input", writeFile(t, "input.txt", "input"), "--output-dir", t.TempDir()} -} - -func runCacheCommand(t *testing.T, args []string, opts Options) (int, string) { - t.Helper() - var stdout bytes.Buffer - var stderr bytes.Buffer - code := RunWithOptions(args, &stdout, &stderr, opts) - return code, stderr.String() -} - -func cacheTestConfig(mode, directory, diagnosticsDir string) string { - var b strings.Builder - b.WriteString("version: 2\n") - if mode != "" || directory != "" { - b.WriteString("workspace:\n chunk_cache:\n") - if mode != "" { - b.WriteString(" mode: " + mode + "\n") - } - if directory != "" { - b.WriteString(" directory: " + directory + "\n") - } - } - if diagnosticsDir != "" { - b.WriteString("diagnostics:\n work_dir: " + diagnosticsDir + "\n retention: always\n") - } - b.WriteString("pipelines:\n example:\n input: fake/input\n artifacts:\n spells:\n extract: fake/extract\n") - return b.String() -} - -func cacheTestConfigWithWorkspace(mode, cacheRoot, workspaceDir string) string { - var b strings.Builder - b.WriteString("version: 2\nworkspace:\n directory: " + workspaceDir + "\n resume:\n enabled: true\n chunk_cache:\n mode: " + mode + "\n") - if cacheRoot != "" { - b.WriteString(" directory: " + cacheRoot + "\n") - } - b.WriteString("pipelines:\n example:\n input: fake/input\n artifacts:\n spells:\n extract: fake/extract\n") - return b.String() -} diff --git a/internal/cli/compatibility_test.go b/internal/cli/compatibility_test.go deleted file mode 100644 index fb30944..0000000 --- a/internal/cli/compatibility_test.go +++ /dev/null @@ -1,602 +0,0 @@ -//go:build legacy - -package cli - -import ( - "bytes" - "context" - "encoding/json" - "io/fs" - "path/filepath" - "reflect" - "sort" - "strings" - "sync" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" - "gitea.maximumdirect.net/eric/notarius/internal/core/config" - "gitea.maximumdirect.net/eric/notarius/internal/core/source" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" - spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape" - spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs" - spellrelatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_relatedness" - validjson "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/valid_json" - validjsonschema "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/valid_json_schema" -) - -func TestProductionCompatibilitySnapshot(t *testing.T) { - normalizedOptions, err := normalizeOptions(Options{}) - if err != nil { - t.Fatalf("normalizeOptions() error = %v, want nil", err) - } - if normalizedOptions.Catalog.Inputs != normalizedOptions.Registries.Inputs || - normalizedOptions.Catalog.Chunkers != normalizedOptions.Registries.Chunkers || - normalizedOptions.Catalog.Extractors != normalizedOptions.Registries.Extractors || - normalizedOptions.Catalog.Mergers != normalizedOptions.Registries.Mergers || - normalizedOptions.Catalog.Normalizers != normalizedOptions.Registries.Normalizers || - normalizedOptions.Catalog.Validators != normalizedOptions.Registries.Validators || - normalizedOptions.Catalog.ValidatorChains != normalizedOptions.Registries.ValidatorChains || - normalizedOptions.Catalog.Outputs != normalizedOptions.Registries.Outputs { - t.Fatal("production catalog and execution registries do not share one composition") - } - if normalizedOptions.LLMClientFactory == nil { - t.Fatal("production LLM client factory is nil") - } - - registries, err := productionRegistries() - if err != nil { - t.Fatalf("productionRegistries() error = %v, want nil", err) - } - - keySnapshots := []struct { - name string - got []string - want []string - }{ - {name: "inputs", got: registries.Inputs.RegisteredKeys(), want: []string{"seriatim"}}, - {name: "chunkers", got: registries.Chunkers.RegisteredKeys(), want: []string{"dnd/scenes", "generic"}}, - {name: "extractors", got: registries.Extractors.RegisteredKeys(), want: []string{"dnd/spells"}}, - {name: "mergers", got: registries.Mergers.RegisteredKeys(), want: []string{"appendorder"}}, - {name: "normalizers", got: registries.Normalizers.RegisteredKeys(), want: []string{"noop"}}, - {name: "validators", got: registries.Validators.RegisteredKeys(), want: []string{ - "extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness", - "generic/always_accept", "generic/always_reject", "generic/valid_json", "generic/valid_json_schema", - }}, - {name: "outputs", got: registries.Outputs.RegisteredKeys(), want: []string{"json"}}, - } - for _, snapshot := range keySnapshots { - t.Run(snapshot.name, func(t *testing.T) { - if !reflect.DeepEqual(snapshot.got, snapshot.want) { - t.Fatalf("registered keys = %#v, want compatibility snapshot %#v", snapshot.got, snapshot.want) - } - }) - } - - wantChain := []pipeline.ModuleBinding{ - pipeline.Binding(validjson.Key), - pipeline.Binding(validjsonschema.Key), - pipeline.Binding(spellshape.Key), - pipeline.Binding(spellsourcerefs.Key), - pipeline.Binding(spellrelatedness.Key), - } - if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) { - t.Fatalf("spell validator chain = %#v, want compatibility snapshot %#v", got, wantChain) - } - - assets, err := productionPromptAssets() - if err != nil { - t.Fatalf("productionPromptAssets() error = %v, want nil", err) - } - assertAssetNames(t, assets.PromptFS, []string{ - "dnd.scenes/dnd.scenes.yaml", - "dnd.scenes/instructions.md", - "dnd.scenes/sharedassets/common-dnd-references.md", - "dnd.scenes/sharedassets/common-dnd-system.md", - "dnd.scenes/sharedassets/common-dnd-transcript.md", - "dnd.scenes/task.md", - "dnd.spells/dnd.spells.yaml", - "dnd.spells/instructions.md", - "dnd.spells/sharedassets/common-dnd-references.md", - "dnd.spells/sharedassets/common-dnd-system.md", - "dnd.spells/sharedassets/common-dnd-transcript.md", - "dnd.spells/task.md", - }) - assertAssetNames(t, assets.SchemaFS, []string{ - "dnd_scenes.v1.json", - "dnd_spells_llm.v1.json", - }) - - identitySnapshot := map[string]map[string]any{ - "scenes": sceneManifestMetadata(t), - "spells": spellManifestMetadata(t), - } - for name, metadata := range identitySnapshot { - for _, key := range []string{"prompt_id", "prompt_version", "prompt_sha256", "response_schema_key", "response_schema_id", "response_schema_name", "response_schema_version", "response_schema_sha256"} { - if value, ok := metadata[key].(string); !ok || value == "" { - t.Fatalf("%s metadata[%q] = %#v, want non-empty identity", name, key, metadata[key]) - } - } - } - if got := []any{ - identitySnapshot["scenes"]["prompt_id"], identitySnapshot["scenes"]["prompt_version"], - identitySnapshot["scenes"]["response_schema_key"], identitySnapshot["scenes"]["response_schema_id"], identitySnapshot["scenes"]["response_schema_name"], identitySnapshot["scenes"]["response_schema_version"], - }; !reflect.DeepEqual(got, []any{"dnd.scenes", "v1", "dnd_scenes", "notarius.dnd.scenes", "notarius_dnd_scenes_v1", "v1"}) { - t.Fatalf("scene identities = %#v, want compatibility snapshot", got) - } - if got := []any{ - identitySnapshot["spells"]["prompt_id"], identitySnapshot["spells"]["prompt_version"], - identitySnapshot["spells"]["response_schema_key"], identitySnapshot["spells"]["response_schema_id"], identitySnapshot["spells"]["response_schema_name"], identitySnapshot["spells"]["response_schema_version"], - }; !reflect.DeepEqual(got, []any{"dnd.spells", "v1", "dnd_spells", "notarius.dnd.spells", "notarius_dnd_spells_v1", "v1"}) { - t.Fatalf("spell identities = %#v, want compatibility snapshot", got) - } - - fileConfig, err := config.LoadFileConfig(fixturePath(t, "examples/dnd-spells.config.yml")) - if err != nil { - t.Fatalf("LoadFileConfig() error = %v, want nil", err) - } - cfg := config.Default() - if err := cfg.ApplyFileConfig(fileConfig); err != nil { - t.Fatalf("ApplyFileConfig() error = %v, want nil", err) - } - effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(registries)}) - if err != nil { - t.Fatalf("Resolve() error = %v, want nil", err) - } - resolved := effective.ResolvedPipeline - if resolved.Input.Module != "seriatim" || resolved.Chunk.Module != "generic" || resolved.Output.Module != "json" || len(resolved.ArtifactLanes) != 1 { - t.Fatalf("resolved example = %#v, want maintained production topology", resolved) - } - lane := resolved.ArtifactLanes[0] - if lane.ID != "spells" || lane.Extract.Module != "dnd/spells" || lane.Merge.Module != "appendorder" || lane.Normalize.Module != "noop" { - t.Fatalf("resolved lane = %#v, want maintained spell lane", lane) - } - if got := resolvedValidatorKeys(resolved.ValidatorChains, pipeline.StageExtract, "spells", spells.Key); !reflect.DeepEqual(got, []string{ - "generic/valid_json", "generic/valid_json_schema", "extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness", - }) { - t.Fatalf("resolved validator keys = %#v, want compatibility snapshot", got) - } - - productionFileConfig, err := config.LoadFileConfig(fixturePath(t, "examples/dnd-spells-production.config.yml")) - if err != nil { - t.Fatalf("LoadFileConfig(production example) error = %v, want nil", err) - } - productionConfig := config.Default() - if err := productionConfig.ApplyFileConfig(productionFileConfig); err != nil { - t.Fatalf("ApplyFileConfig(production example) error = %v, want nil", err) - } - productionEffective, err := productionConfig.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(registries)}) - if err != nil { - t.Fatalf("Resolve(production example) error = %v, want nil", err) - } - productionResolved := productionEffective.ResolvedPipeline - if productionConfig.Concurrency.TotalLLM != 1 || productionConfig.Concurrency.StageWorkers["extract"] != 1 || !reflect.DeepEqual(productionResolved.Chunk.Options, map[string]any{"max_units": 50}) { - t.Fatalf("production example concurrency/options = %#v/%#v, want compatibility snapshot", productionConfig.Concurrency, productionResolved.Chunk.Options) - } - bindings := productionResolved.ArtifactLanes[0].ExtractReferences.Bindings - if len(bindings) != 2 || bindings[0].SlotName != "glossary" || bindings[0].Source != "./dnd-spells-glossary.txt" || bindings[1].SlotName != "party" || bindings[1].Source != "./dnd-spells-roster.txt" { - t.Fatalf("production example reference bindings = %#v, want maintained glossary and party bindings", bindings) - } -} - -func TestMaintainedSeriatimToDNDCompatibilityBundle(t *testing.T) { - tests := []struct { - name string - client contracts.StructuredLLMClient - wantStatus string - wantLaneFile bool - wantRejectedCount int - }{ - {name: "approved", client: newFakeRunLLMClient(false), wantStatus: "approved", wantLaneFile: true}, - {name: "validator rejection is nonfatal", client: newFakeRunLLMClient(true), wantStatus: "rejected", wantRejectedCount: 1}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - outputDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - code := RunWithOptions([]string{ - "run", "dnd-session", - "--config", fixturePath(t, "examples/dnd-spells.config.yml"), - "--input", fixturePath(t, "examples/seriatim-minimal-transcript.json"), - "--output-dir", outputDir, - "--diagnostics-dir", t.TempDir(), - }, &stdout, &stderr, Options{LLMClientFactory: fakeLLMFactory(test.client, nil)}) - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - - runDir := onlyChildDir(t, outputDir) - wantFiles := []string{"index.json", "manifest.json", "rejected.json", "warnings.json"} - if test.wantLaneFile { - wantFiles = append(wantFiles, "lanes/spells.json") - } - sort.Strings(wantFiles) - if got := relativeFileNames(t, runDir); !reflect.DeepEqual(got, wantFiles) { - t.Fatalf("durable files = %#v, want compatibility snapshot %#v", got, wantFiles) - } - - var manifest artifacts.RunManifest - readJSONFile(t, filepath.Join(runDir, "manifest.json"), &manifest) - if manifest.PipelineID != "dnd-session" || manifest.InputModule != "seriatim" || manifest.Chunker != "generic" || manifest.OutputEncoder != "json" { - t.Fatalf("manifest module provenance = %#v, want maintained production modules", manifest) - } - if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].ID != "spells" { - t.Fatalf("artifact lanes = %#v, want one spells lane", manifest.ArtifactLanes) - } - laneManifest := manifest.ArtifactLanes[0] - if laneManifest.Extractor != "dnd/spells" || laneManifest.Merger != "appendorder" || laneManifest.Normalizer != "noop" { - t.Fatalf("manifest lane module provenance = %#v, want maintained production modules", laneManifest) - } - if len(manifest.Extractors) != 0 || manifest.Merger != "" || manifest.Normalizer != "" { - t.Fatalf("legacy top-level lane summaries = %#v/%q/%q, want empty compatibility snapshot", manifest.Extractors, manifest.Merger, manifest.Normalizer) - } - if manifest.ValidationStatus != test.wantStatus || len(manifest.RejectedOutputs) != test.wantRejectedCount { - t.Fatalf("manifest outcome = status %q rejected %#v, want %q/%d", manifest.ValidationStatus, manifest.RejectedOutputs, test.wantStatus, test.wantRejectedCount) - } - if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:1c98d94ae632fb10a2b56f684cd4fb1019cedb1a629e57dc0977cf4a54135be0"}) { - t.Fatalf("source digests = %#v, want maintained fixture provenance", manifest.SourceDigests) - } - if got := manifestValidatorKeys(manifestValidatorChain(t, manifest, pipeline.StageExtract, "spells", spells.Key)); !reflect.DeepEqual(got, []string{ - "generic/valid_json", "generic/valid_json_schema", "extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness", - }) { - t.Fatalf("manifest validator chain = %#v, want compatibility snapshot", got) - } - - var index struct { - ManifestFile string `json:"manifest_file"` - OutputFiles []struct { - LaneID string `json:"lane_id"` - MediaType string `json:"media_type"` - File string `json:"file"` - ModuleKey string `json:"module_key"` - SchemaID string `json:"schema_id"` - SchemaName string `json:"schema_name"` - SchemaVersion string `json:"schema_version"` - } `json:"output_files"` - RejectedFile string `json:"rejected_file"` - WarningsFile string `json:"warnings_file"` - } - readJSONFile(t, filepath.Join(runDir, "index.json"), &index) - if index.ManifestFile != "manifest.json" || index.RejectedFile != "rejected.json" || index.WarningsFile != "warnings.json" { - t.Fatalf("output index fixed files = %#v, want compatibility snapshot", index) - } - if test.wantLaneFile { - if len(index.OutputFiles) != 1 { - t.Fatalf("output index entries = %#v, want one", index.OutputFiles) - } - wantOutput := struct { - LaneID, MediaType, File, ModuleKey, SchemaID, SchemaName, SchemaVersion string - }{"spells", "application/json", "lanes/spells.json", "noop", "notarius.dnd.spells", "notarius_dnd_spells_v1", "v1"} - gotOutput := index.OutputFiles[0] - got := struct { - LaneID, MediaType, File, ModuleKey, SchemaID, SchemaName, SchemaVersion string - }{gotOutput.LaneID, gotOutput.MediaType, gotOutput.File, gotOutput.ModuleKey, gotOutput.SchemaID, gotOutput.SchemaName, gotOutput.SchemaVersion} - if got != wantOutput { - t.Fatalf("output index entries = %#v, want compatibility snapshot %#v", index.OutputFiles, wantOutput) - } - assertJSONEqual(t, readFile(t, filepath.Join(runDir, "lanes/spells.json")), []byte(`{ - "spell_casts": [{ - "caster": "Aria", - "spell": "Cure Wounds", - "effect": "Heals a wounded ally.", - "narrative_description": "Aria casts Cure Wounds.", - "source_refs": [{"source_id": "session-alpha", "start_unit_id": 1, "end_unit_id": 1}] - }] - }`)) - } else if len(index.OutputFiles) != 0 { - t.Fatalf("output index entries = %#v, want none for rejected lane", index.OutputFiles) - } - - var warnings struct { - Warnings []contracts.Warning `json:"warnings"` - } - readJSONFile(t, filepath.Join(runDir, "warnings.json"), &warnings) - if len(warnings.Warnings) != 0 { - t.Fatalf("warnings = %#v, want empty compatibility snapshot", warnings.Warnings) - } - var rejected struct { - Rejected []contracts.RejectedOutput `json:"rejected"` - } - readJSONFile(t, filepath.Join(runDir, "rejected.json"), &rejected) - if len(rejected.Rejected) != test.wantRejectedCount { - t.Fatalf("rejected outputs = %#v, want %d", rejected.Rejected, test.wantRejectedCount) - } - if test.wantRejectedCount == 1 { - got := rejected.Rejected[0] - if got.Stage != "extract" || got.LaneID != "spells" || got.ModuleKey != "dnd/spells" || got.ChunkID != "chunk-000001" || got.ChunkIndex != 0 || got.ValidatorName != "extract/dnd/spells/source_refs" || got.ReasonCode != "invalid_source_refs" || got.AttemptCount != 1 { - t.Fatalf("rejection = %#v, want maintained nonfatal validator outcome", got) - } - } - }) - } -} - -func TestProductionLLMCallersShareScheduledClient(t *testing.T) { - underlying := newBlockingProductionLLMClient() - scheduler, err := frameworkllm.NewScheduler(1) - if err != nil { - t.Fatalf("NewScheduler() error = %v, want nil", err) - } - client := frameworkllm.NewScheduledClient(underlying, scheduler) - doc := &source.SourceDocument{ - ID: "session-alpha", Kind: "transcript", Format: "application/json", Digest: "sha256:source", - Units: []source.SourceUnit{ - {ID: 1, Kind: "segment", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}}, - {ID: 2, Kind: "segment", Text: "The spell takes effect.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}}, - }, - } - chunk := source.Chunk{ - ID: "session-alpha:chunk:0", SourceID: doc.ID, Index: 0, Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 2}, - Content: []byte(`{"scene":"Aria casts Cure Wounds."}`), MediaType: "application/json", Units: append([]source.SourceUnit(nil), doc.Units...), - } - - var started sync.WaitGroup - started.Add(2) - errs := make(chan error, 2) - go func() { - started.Done() - chunker, err := scenes.New(client, scenes.Options{}) - if err == nil { - _, err = chunker.Plan(context.Background(), contracts.ChunkRequest{Source: doc}) - } - errs <- err - }() - go func() { - started.Done() - extractor, err := spells.New(client, spells.Options{}) - if err == nil { - _, err = extractor.Extract(context.Background(), contracts.TypedExtractionRequest{Source: doc, Chunk: &chunk}) - } - errs <- err - }() - started.Wait() - - for i := 0; i < 2; i++ { - <-underlying.entered - underlying.release <- struct{}{} - } - for i := 0; i < 2; i++ { - if err := <-errs; err != nil { - t.Fatalf("production LLM caller error = %v, want nil", err) - } - } - if underlying.maxActive != 1 { - t.Fatalf("maximum concurrent provider calls = %d, want total_llm limit 1", underlying.maxActive) - } - sort.Strings(underlying.stageNames) - if !reflect.DeepEqual(underlying.stageNames, []string{"dnd/scenes", "dnd/spells"}) { - t.Fatalf("scheduled stage names = %#v, want both production LLM callers", underlying.stageNames) - } -} - -func TestProductionBundlePreservesLaneAndChunkOrder(t *testing.T) { - configPath := writeTestConfig(t, `version: 2 -pipelines: - dnd-session: - input: seriatim - chunk: - module: generic - options: - max_units: 1 - artifacts: - zeta: - extract: dnd/spells - alpha: - extract: dnd/spells -`) - outputDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - code := RunWithOptions([]string{ - "run", "dnd-session", - "--config", configPath, - "--input", fixturePath(t, "examples/seriatim-minimal-transcript.json"), - "--output-dir", outputDir, - "--diagnostics-dir", t.TempDir(), - }, &stdout, &stderr, Options{LLMClientFactory: fakeLLMFactory(orderingProductionLLMClient{}, nil)}) - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - - runDir := onlyChildDir(t, outputDir) - var index struct { - OutputFiles []struct { - LaneID string `json:"lane_id"` - } `json:"output_files"` - } - readJSONFile(t, filepath.Join(runDir, "index.json"), &index) - if len(index.OutputFiles) != 2 { - t.Fatalf("output index entries = %#v, want two lanes", index.OutputFiles) - } - if got := []string{index.OutputFiles[0].LaneID, index.OutputFiles[1].LaneID}; !reflect.DeepEqual(got, []string{"alpha", "zeta"}) { - t.Fatalf("output lane order = %#v, want resolved lane order", got) - } - for _, laneID := range []string{"alpha", "zeta"} { - var payload struct { - SpellCasts []struct { - Spell string `json:"spell"` - SourceRefs []source.SourceRef `json:"source_refs"` - } `json:"spell_casts"` - } - readJSONFile(t, filepath.Join(runDir, "lanes", laneID+".json"), &payload) - if len(payload.SpellCasts) != 2 { - t.Fatalf("lane %q spell casts = %#v, want one per source chunk", laneID, payload.SpellCasts) - } - got := []any{ - payload.SpellCasts[0].Spell, payload.SpellCasts[0].SourceRefs[0].StartUnitID, - payload.SpellCasts[1].Spell, payload.SpellCasts[1].SourceRefs[0].StartUnitID, - } - if !reflect.DeepEqual(got, []any{"Cure Wounds", 1, "Shield", 2}) { - t.Fatalf("lane %q chunk handoff order = %#v, want source chunk order", laneID, got) - } - } -} - -type blockingProductionLLMClient struct { - mu sync.Mutex - active int - maxActive int - stageNames []string - entered chan struct{} - release chan struct{} -} - -type orderingProductionLLMClient struct{} - -func (orderingProductionLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - material := req.Inputs["transcript"] - unitID := 1 - spellName := "Cure Wounds" - if strings.Contains(string(material.Content), "Shield") { - unitID = 2 - spellName = "Shield" - } - payload, err := json.Marshal(map[string]any{ - "spell_casts": []map[string]any{{ - "caster": "Aria", - "spell": spellName, - "effect": "Fixture effect.", - "narrative_description": "Fixture spell cast.", - "source_refs": []map[string]any{{ - "start_unit_id": unitID, - "end_unit_id": unitID, - }}, - }}, - }) - if err != nil { - return contracts.StructuredCompletionResponse{}, err - } - if err := json.Unmarshal(payload, out); err != nil { - return contracts.StructuredCompletionResponse{}, err - } - return contracts.StructuredCompletionResponse{Content: payload}, nil -} - -func newBlockingProductionLLMClient() *blockingProductionLLMClient { - return &blockingProductionLLMClient{entered: make(chan struct{}, 2), release: make(chan struct{}, 2)} -} - -func (client *blockingProductionLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - client.mu.Lock() - client.active++ - if client.active > client.maxActive { - client.maxActive = client.active - } - client.stageNames = append(client.stageNames, req.StageName) - client.mu.Unlock() - client.entered <- struct{}{} - - select { - case <-ctx.Done(): - return contracts.StructuredCompletionResponse{}, ctx.Err() - case <-client.release: - } - - client.mu.Lock() - client.active-- - client.mu.Unlock() - - var payload []byte - switch req.StageName { - case scenes.Key: - payload = []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":2,"short_title":"Spell","primary_mode":"Narrative","main_participants":["Aria"],"summary":"Aria casts a spell.","boundary_note":"Complete source.","boundary_confidence":"High"}],"boundary_caveats":[]}`) - case spells.Key: - payload = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"Healing","narrative_description":"Aria casts Cure Wounds.","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`) - } - if err := json.Unmarshal(payload, out); err != nil { - return contracts.StructuredCompletionResponse{}, err - } - return contracts.StructuredCompletionResponse{Content: payload}, nil -} - -func sceneManifestMetadata(t *testing.T) map[string]any { - t.Helper() - chunker, err := scenes.New(orderingProductionLLMClient{}, scenes.Options{}) - if err != nil { - t.Fatalf("construct scene chunker: %v", err) - } - return chunker.ManifestMetadata() -} - -func spellManifestMetadata(t *testing.T) map[string]any { - t.Helper() - extractor, err := spells.New(orderingProductionLLMClient{}, spells.Options{}) - if err != nil { - t.Fatalf("construct spell extractor: %v", err) - } - return extractor.ManifestMetadata() -} - -func assertAssetNames(t *testing.T, getFS func() (fs.FS, error), want []string) { - t.Helper() - fSys, err := getFS() - if err != nil { - t.Fatalf("asset filesystem error = %v, want nil", err) - } - var got []string - if err := fs.WalkDir(fSys, ".", func(path string, entry fs.DirEntry, err error) error { - if err == nil && !entry.IsDir() { - got = append(got, path) - } - return err - }); err != nil { - t.Fatalf("walk assets: %v", err) - } - if !reflect.DeepEqual(got, want) { - t.Fatalf("asset names = %#v, want compatibility snapshot %#v", got, want) - } -} - -func resolvedValidatorKeys(chains []pipeline.ResolvedValidatorChain, stage pipeline.ModuleStage, laneID, moduleKey string) []string { - for _, chain := range chains { - if chain.Stage == stage && chain.LaneID == laneID && chain.ModuleKey == moduleKey { - keys := make([]string, 0, len(chain.Validators)) - for _, validator := range chain.Validators { - keys = append(keys, validator.Binding.Module) - } - return keys - } - } - return nil -} - -func relativeFileNames(t *testing.T, root string) []string { - t.Helper() - var names []string - if err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { - if err != nil || entry.IsDir() { - return err - } - rel, err := filepath.Rel(root, path) - if err != nil { - return err - } - names = append(names, filepath.ToSlash(rel)) - return nil - }); err != nil { - t.Fatalf("walk durable output: %v", err) - } - sort.Strings(names) - return names -} - -func assertJSONEqual(t *testing.T, got, want []byte) { - t.Helper() - var gotValue any - var wantValue any - if err := json.Unmarshal(got, &gotValue); err != nil { - t.Fatalf("unmarshal actual JSON: %v", err) - } - if err := json.Unmarshal(want, &wantValue); err != nil { - t.Fatalf("unmarshal expected JSON: %v", err) - } - if !reflect.DeepEqual(gotValue, wantValue) { - t.Fatalf("JSON = %#v, want compatibility snapshot %#v", gotValue, wantValue) - } -} diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go deleted file mode 100644 index 6185602..0000000 --- a/internal/cli/run_test.go +++ /dev/null @@ -1,4258 +0,0 @@ -//go:build legacy - -package cli - -import ( - "bytes" - "context" - "encoding/base64" - "encoding/json" - "errors" - "io/fs" - "os" - "path/filepath" - "reflect" - "regexp" - "sort" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" - "gitea.maximumdirect.net/eric/notarius/internal/core/config" - "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" - "gitea.maximumdirect.net/eric/notarius/internal/core/source" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes" - "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells" - spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape" - spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs" - spellrelatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_relatedness" - "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units" - "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/merge/appendorder" - "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop" - jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json" - alwaysaccept "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_accept" - alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject" - validjson "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/valid_json" - validjsonschema "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/valid_json_schema" - "gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript" - "gitea.maximumdirect.net/eric/scriptorium" -) - -func TestRunNoArgsWritesUsageToStdout(t *testing.T) { - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := Run(nil, &stdout, &stderr) - - if code != 0 { - t.Fatalf("Run() code = %d, want 0", code) - } - if stdout.String() != usage { - t.Fatalf("stdout = %q, want %q", stdout.String(), usage) - } - if stderr.Len() != 0 { - t.Fatalf("stderr = %q, want empty", stderr.String()) - } -} - -func TestRunHelpArgsWriteUsageToStdout(t *testing.T) { - tests := []struct { - name string - args []string - }{ - {name: "help", args: []string{"help"}}, - {name: "long help flag", args: []string{"--help"}}, - {name: "short help flag", args: []string{"-h"}}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := Run(tt.args, &stdout, &stderr) - - if code != 0 { - t.Fatalf("Run() code = %d, want 0", code) - } - if stdout.String() != usage { - t.Fatalf("stdout = %q, want %q", stdout.String(), usage) - } - if !strings.Contains(stdout.String(), "config validate") || !strings.Contains(stdout.String(), "pipelines list") { - t.Fatalf("usage does not mention new commands: %q", stdout.String()) - } - if stderr.Len() != 0 { - t.Fatalf("stderr = %q, want empty", stderr.String()) - } - }) - } -} - -func TestRunUnknownCommandWritesErrorAndUsageToStderr(t *testing.T) { - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := Run([]string{"extract"}, &stdout, &stderr) - - if code != 2 { - t.Fatalf("Run() code = %d, want 2", code) - } - if stdout.Len() != 0 { - t.Fatalf("stdout = %q, want empty", stdout.String()) - } - gotStderr := stderr.String() - if !strings.Contains(gotStderr, "notarius: unknown command \"extract\"") { - t.Fatalf("stderr = %q, want unknown command error", gotStderr) - } - if !strings.Contains(gotStderr, usage) { - t.Fatalf("stderr = %q, want usage", gotStderr) - } -} - -func TestRunConfigValidateSuccessWithFakeCatalog(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events", "notes")) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "example"}, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if !strings.Contains(stdout.String(), "is valid for pipeline") { - t.Fatalf("stdout = %q, want validation success", stdout.String()) - } - if stderr.Len() != 0 { - t.Fatalf("stderr = %q, want empty", stderr.String()) - } -} - -func TestProductionCatalogIncludesProductionModulesValidatorsAndDefaults(t *testing.T) { - catalog, err := productionCatalog() - if err != nil { - t.Fatalf("productionCatalog() error = %v, want nil", err) - } - if catalog.ArtifactCodecs == nil { - t.Fatal("production artifact codec registry = nil, want initialized empty registry") - } - if got := catalog.ArtifactCodecs.RegisteredKinds(); !reflect.DeepEqual(got, []contracts.ArtifactKind{"dnd/spell-list"}) { - t.Fatalf("production artifact codec kinds = %#v, want dnd/spell-list", got) - } - - moduleTests := []struct { - name string - got func() (pipeline.ModuleSpec, bool) - want pipeline.ModuleSpec - }{ - { - name: "seriatim input", - got: func() (pipeline.ModuleSpec, bool) { return catalog.Inputs.Spec(transcript.Key) }, - want: transcript.ModuleSpec(), - }, - { - name: "generic chunker", - got: func() (pipeline.ModuleSpec, bool) { return catalog.Chunkers.Spec(units.Key) }, - want: units.ModuleSpec(), - }, - { - name: "dnd scenes chunker", - got: func() (pipeline.ModuleSpec, bool) { return catalog.Chunkers.Spec(scenes.Key) }, - want: scenes.ModuleSpec(), - }, - { - name: "dnd spells extractor", - got: func() (pipeline.ModuleSpec, bool) { return catalog.Extractors.Spec(spells.Key) }, - want: spells.ModuleSpec(), - }, - { - name: "appendorder merger", - got: func() (pipeline.ModuleSpec, bool) { return catalog.Mergers.Spec(appendorder.Key) }, - want: appendorder.TypedModuleSpec(spells.ModuleSpec().ArtifactKind), - }, - { - name: "noop normalizer", - got: func() (pipeline.ModuleSpec, bool) { return catalog.Normalizers.Spec(noop.Key) }, - want: noop.TypedModuleSpec(spells.ModuleSpec().ArtifactKind), - }, - { - name: "json output", - got: func() (pipeline.ModuleSpec, bool) { return catalog.Outputs.Spec(jsonoutput.Key) }, - want: jsonoutput.ModuleSpec(), - }, - } - - for _, test := range moduleTests { - t.Run(test.name, func(t *testing.T) { - got, ok := test.got() - if !ok { - t.Fatalf("module spec ok = false, want true") - } - if !reflect.DeepEqual(got, test.want) { - t.Fatalf("module spec = %#v, want %#v", got, test.want) - } - }) - } - - validatorTests := []pipeline.ValidatorSpec{ - alwaysaccept.Spec(), - alwaysreject.Spec(), - validjson.Spec(), - validjsonschema.Spec(), - spellshape.Spec(), - spellsourcerefs.Spec(), - spellrelatedness.Spec(), - } - for _, want := range validatorTests { - t.Run("validator "+want.Key, func(t *testing.T) { - got, ok := catalog.Validators.Spec(want.Key) - if !ok { - t.Fatalf("validator spec %q ok = false, want true", want.Key) - } - if !reflect.DeepEqual(got, want) { - t.Fatalf("validator spec = %#v, want %#v", got, want) - } - }) - } - - gotChain := catalog.ValidatorChains.Validators(pipeline.StageExtract, spells.Key) - wantChain := []pipeline.ModuleBinding{ - pipeline.Binding(validjson.Key), - pipeline.Binding(validjsonschema.Key), - pipeline.Binding(spellshape.Key), - pipeline.Binding(spellsourcerefs.Key), - pipeline.Binding(spellrelatedness.Key), - } - if !reflect.DeepEqual(gotChain, wantChain) { - t.Fatalf("dnd spell default validator chain = %#v, want %#v", gotChain, wantChain) - } - if got := catalog.ValidatorChains.Validators(pipeline.StageChunk, units.Key); len(got) != 0 { - t.Fatalf("generic chunker default validator chain = %#v, want empty", got) - } -} - -func TestCatalogConversionsPreserveArtifactCodecRegistry(t *testing.T) { - codecs := pipeline.NewArtifactCodecRegistry() - registries := pipeline.Registries{ArtifactCodecs: codecs} - if isEmptyRegistries(registries) { - t.Fatal("registries with artifact codecs reported empty") - } - - catalog := catalogFromRegistries(registries) - if catalog.ArtifactCodecs != codecs || isEmptyCatalog(catalog) { - t.Fatalf("catalog artifact codecs = %p empty=%t, want %p and non-empty", catalog.ArtifactCodecs, isEmptyCatalog(catalog), codecs) - } - converted := registriesFromCatalog(catalog) - if converted.ArtifactCodecs != codecs { - t.Fatalf("converted artifact codecs = %p, want %p", converted.ArtifactCodecs, codecs) - } -} - -func TestProductionPromptAssetsRegisterAndPrepareDndPrompts(t *testing.T) { - registry, err := productionPromptAssets() - if err != nil { - t.Fatalf("productionPromptAssets() error = %v, want nil", err) - } - promptFS, err := registry.PromptFS() - if err != nil { - t.Fatalf("PromptFS() error = %v, want nil", err) - } - for _, name := range []string{ - "dnd.scenes/dnd.scenes.yaml", - "dnd.scenes/task.md", - "dnd.scenes/instructions.md", - "dnd.scenes/sharedassets/common-dnd-system.md", - "dnd.scenes/sharedassets/common-dnd-transcript.md", - "dnd.scenes/sharedassets/common-dnd-references.md", - "dnd.spells/dnd.spells.yaml", - "dnd.spells/task.md", - "dnd.spells/instructions.md", - "dnd.spells/sharedassets/common-dnd-system.md", - "dnd.spells/sharedassets/common-dnd-transcript.md", - "dnd.spells/sharedassets/common-dnd-references.md", - } { - if _, err := promptFS.Open(name); err != nil { - t.Fatalf("PromptFS().Open(%q) error = %v, want nil", name, err) - } - } - for _, name := range []string{ - "common-dnd-system.md", - "common-dnd-transcript.md", - "common-dnd-references.md", - } { - if _, err := promptFS.Open(name); !errors.Is(err, fs.ErrNotExist) { - t.Fatalf("PromptFS().Open(%q) error = %v, want not exist", name, err) - } - } - - options, err := registry.ScriptoriumOptions() - if err != nil { - t.Fatalf("ScriptoriumOptions() error = %v, want nil", err) - } - options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ - ID: "production-test-profile", - Endpoint: "http://127.0.0.1:1/v1", - Model: "production-test-model", - }))) - engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...) - if err != nil { - t.Fatalf("NewEngine() error = %v, want nil", err) - } - - sceneTranscript := `{"id":"session-1","segments":[{"id":"u1","text":"We enter the crypt."}]}` - scenesPrepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: scenes.PromptID, - PromptVersion: scenes.ResponseSchemaVersion, - ProfileID: "production-test-profile", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.InlineWithURI("file:///session.json", sceneTranscript), - "players": scriptorium.Inline("Alice: Aria"), - "party": scriptorium.Inline("Aria: cleric"), - "glossary": scriptorium.Inline("Brightmantle: temple"), - }, - }) - if err != nil { - t.Fatalf("scene Prepare() error = %v, want nil", err) - } - if got := len(scenesPrepared.Messages); got != 5 { - t.Fatalf("scene message count = %d, want 5", got) - } - if !strings.Contains(scenesPrepared.Messages[1].Content, sceneTranscript) { - t.Fatalf("scene transcript message did not include source input") - } - for _, want := range []string{"Alice: Aria", "Aria: cleric", "Brightmantle: temple"} { - if !strings.Contains(scenesPrepared.Messages[2].Content, want) { - t.Fatalf("scene reference message missing %q", want) - } - } - - spellTranscript := `{"id":"session-1","segments":[{"id":"u1","text":"Mira casts shield."}]}` - spellsPrepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: spells.PromptID, - PromptVersion: spells.SchemaVersion, - ProfileID: "production-test-profile", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.InlineWithURI("file:///session.json", spellTranscript), - "players": scriptorium.Inline("Dana: Mira"), - "party": scriptorium.Inline("Mira: wizard"), - "glossary": scriptorium.Inline("Shield: abjuration"), - }, - }) - if err != nil { - t.Fatalf("spell Prepare() error = %v, want nil", err) - } - if got := len(spellsPrepared.Messages); got != 5 { - t.Fatalf("spell message count = %d, want 5", got) - } - if !strings.Contains(spellsPrepared.Messages[1].Content, spellTranscript) { - t.Fatalf("spell transcript message did not include source input") - } - for _, want := range []string{"Dana: Mira", "Mira: wizard", "Shield: abjuration"} { - if !strings.Contains(spellsPrepared.Messages[2].Content, want) { - t.Fatalf("spell reference message missing %q", want) - } - } -} - -func TestRunConfigValidateUsesProductionCatalogByDefault(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells")) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "dnd-session"}, &stdout, &stderr, Options{}) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if !strings.Contains(stdout.String(), "is valid for pipeline") { - t.Fatalf("stdout = %q, want validation success", stdout.String()) - } -} - -func TestRunConfigValidateAcceptsDNDScenesChunker(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAMLWithChunk("dnd-session", scenes.Key, "dnd/spells")) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "dnd-session"}, &stdout, &stderr, Options{}) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if !strings.Contains(stdout.String(), "is valid for pipeline") { - t.Fatalf("stdout = %q, want validation success", stdout.String()) - } -} - -func TestRunConfigValidateUnknownProductionModuleIncludesContext(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "missing/extract")) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "dnd-session"}, &stdout, &stderr, Options{}) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - got := stderr.String() - for _, want := range []string{"dnd-session", "extract", "missing/extract", "not registered"} { - if !strings.Contains(got, want) { - t.Fatalf("stderr = %q, want substring %q", got, want) - } - } -} - -func TestRunConfigValidateRejectsUnknownProductionValidator(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAMLWithExtractValidators("dnd-session", "\n - missing/validator\n")) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "dnd-session"}, &stdout, &stderr, Options{}) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if got := stderr.String(); !strings.Contains(got, "unknown validator") || !strings.Contains(got, "missing/validator") { - t.Fatalf("stderr = %q, want unknown validator", got) - } -} - -func TestRunConfigValidateRejectsLLMProfileForDeterministicProductionValidator(t *testing.T) { - configPath := writeTestConfig(t, `version: 2 -pipelines: - dnd-session: - input: seriatim - artifacts: - spells: - extract: - module: dnd/spells - validators: - - module: generic/valid_json - llm_profile: review -`) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "dnd-session"}, &stdout, &stderr, Options{}) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - got := stderr.String() - for _, want := range []string{"llm_profile", "deterministic", validjson.Key} { - if !strings.Contains(got, want) { - t.Fatalf("stderr = %q, want substring %q", got, want) - } - } -} - -func TestRunConfigValidateReportsParseErrors(t *testing.T) { - configPath := writeFile(t, "config.yml", "version: 1\n") - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", configPath}, &stdout, &stderr, Options{}) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if stdout.Len() != 0 { - t.Fatalf("stdout = %q, want empty", stdout.String()) - } - if !strings.Contains(stderr.String(), "unsupported config version") { - t.Fatalf("stderr = %q, want parse error", stderr.String()) - } -} - -func TestRunConfigValidatePipelineOnlySuccessAndInvalidLane(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events", "notes")) - - t.Run("success", func(t *testing.T) { - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "example", "--only", "notes"}, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - }) - - t.Run("invalid lane", func(t *testing.T) { - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "example", "--only", "missing"}, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "selected artifact lane") { - t.Fatalf("stderr = %q, want invalid lane error", stderr.String()) - } - }) -} - -func TestRunConfigValidateOnlyWithoutPipelineFails(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events")) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--only", "events"}, &stdout, &stderr, Options{}) - - if code != 2 { - t.Fatalf("RunWithOptions() code = %d, want 2", code) - } - if !strings.Contains(stderr.String(), "--only requires --pipeline") { - t.Fatalf("stderr = %q, want only/pipeline error", stderr.String()) - } -} - -func TestRunConfigValidateRejectsMalformedOnlyValues(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events", "notes")) - tests := []string{",", "notes,", ",notes", "events, ,notes"} - - for _, only := range tests { - t.Run(only, func(t *testing.T) { - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "example", "--only", only}, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t), - }) - - if code != 2 { - t.Fatalf("RunWithOptions() code = %d, want 2", code) - } - if stdout.Len() != 0 { - t.Fatalf("stdout = %q, want empty", stdout.String()) - } - if !strings.Contains(stderr.String(), "--only") { - t.Fatalf("stderr = %q, want --only error", stderr.String()) - } - }) - } -} - -func TestRunPipelinesListSortedTextOutput(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAMLForPipelines(map[string][]string{ - "zeta": {"events"}, - "alpha": {"events"}, - })) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"pipelines", "list", "--config", configPath}, &stdout, &stderr, Options{}) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if got, want := stdout.String(), "alpha\nzeta\n"; got != want { - t.Fatalf("stdout = %q, want %q", got, want) - } -} - -func TestRunPipelinesListStableJSONOutput(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAMLForPipelines(map[string][]string{ - "b": {"events"}, - "a": {"events"}, - })) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"pipelines", "list", "--config", configPath, "--json"}, &stdout, &stderr, Options{}) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if got, want := stdout.String(), "{\"pipelines\":[\"a\",\"b\"]}\n"; got != want { - t.Fatalf("stdout = %q, want %q", got, want) - } -} - -func TestRunUsesNotariusConfigWhenConfigFlagAbsent(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events")) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"pipelines", "list"}, &stdout, &stderr, Options{ - LookupEnv: mapLookup(map[string]string{"NOTARIUS_CONFIG": configPath}), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if got, want := stdout.String(), "example\n"; got != want { - t.Fatalf("stdout = %q, want %q", got, want) - } -} - -func TestRunConfigValidateRejectsStaleLLMProfiles(t *testing.T) { - configPath := writeTestConfig(t, `version: 2 -llm_profiles: - default: {} -pipelines: - example: - input: fake/input - artifacts: - events: - extract: fake/extract -`) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", configPath}, &stdout, &stderr, Options{}) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "llm_profiles") { - t.Fatalf("stderr = %q, want stale llm_profiles error", stderr.String()) - } -} - -func TestRunMissingConfigPathProducesActionableError(t *testing.T) { - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", filepath.Join(t.TempDir(), "missing.yml")}, &stdout, &stderr, Options{}) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "config file") || !strings.Contains(stderr.String(), "not available") { - t.Fatalf("stderr = %q, want actionable missing config error", stderr.String()) - } -} - -func TestRunRejectsAdHocStructuralFlags(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events")) - flags := []string{"--extractor", "--chunker", "--input", "--merge", "--normalize"} - - for _, flagName := range flags { - t.Run(flagName, func(t *testing.T) { - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", configPath, flagName, "value"}, &stdout, &stderr, Options{}) - - if code != 2 { - t.Fatalf("RunWithOptions() code = %d, want 2", code) - } - if !strings.Contains(stderr.String(), "flag provided but not defined") { - t.Fatalf("stderr = %q, want invalid flag error", stderr.String()) - } - }) - } -} - -func TestRunInvalidFlagsExitTwo(t *testing.T) { - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"pipelines", "list", "--bogus"}, &stdout, &stderr, Options{}) - - if code != 2 { - t.Fatalf("RunWithOptions() code = %d, want 2", code) - } - if !strings.Contains(stderr.String(), "flag provided but not defined") { - t.Fatalf("stderr = %q, want invalid flag error", stderr.String()) - } -} - -func TestProductionLLMClientFactoryBuildsScriptoriumRuntime(t *testing.T) { - cfg := config.Default() - - client, profiles, err := productionLLMClientFactory(context.Background(), cfg, "mistral-small-3") - if err != nil { - t.Fatalf("productionLLMClientFactory() error = %v, want nil", err) - } - if client == nil { - t.Fatal("productionLLMClientFactory() client = nil, want client") - } - if profiles != nil { - t.Fatalf("productionLLMClientFactory() profiles = %#v, want runtime-reported profiles", profiles) - } -} - -func TestRunPipelineMissingPipelineID(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells")) - inputPath := writeSeriatimInput(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "--config", configPath, "--input", inputPath}, &stdout, &stderr, Options{}) - - if code != 2 { - t.Fatalf("RunWithOptions() code = %d, want 2", code) - } - if !strings.Contains(stderr.String(), "pipeline ID") { - t.Fatalf("stderr = %q, want missing pipeline ID error", stderr.String()) - } -} - -func TestRunPipelineMissingInputFlag(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells")) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath}, &stdout, &stderr, Options{}) - - if code != 2 { - t.Fatalf("RunWithOptions() code = %d, want 2", code) - } - if !strings.Contains(stderr.String(), "--input") { - t.Fatalf("stderr = %q, want missing input error", stderr.String()) - } -} - -func TestRunPipelineRejectsUnknownFlag(t *testing.T) { - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--extractor", "dnd/spells"}, &stdout, &stderr, Options{}) - - if code != 2 { - t.Fatalf("RunWithOptions() code = %d, want 2", code) - } - if !strings.Contains(stderr.String(), "flag provided but not defined") { - t.Fatalf("stderr = %q, want invalid flag error", stderr.String()) - } -} - -func TestRunPipelineUnknownPipeline(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells")) - inputPath := writeSeriatimInput(t) - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "missing", "--config", configPath, "--input", inputPath, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "pipeline \"missing\" is not configured") { - t.Fatalf("stderr = %q, want unknown pipeline error", stderr.String()) - } -} - -func TestRunPipelineUnknownOnlyLane(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells")) - inputPath := writeSeriatimInput(t) - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--only", "missing", "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "selected artifact lane") { - t.Fatalf("stderr = %q, want selected lane error", stderr.String()) - } -} - -func TestRunPipelineInvalidInputPath(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells")) - inputPath := filepath.Join(t.TempDir(), "missing.json") - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "read input") { - t.Fatalf("stderr = %q, want input read error", stderr.String()) - } -} - -func TestRunPipelineSuccessUsesProductionRegistriesAndFakeLLM(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells")) - inputPath := writeSeriatimInput(t) - outputDir := t.TempDir() - diagnosticsDir := t.TempDir() - client := newFakeRunLLMClient(false) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(client, nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if client.calls != 1 { - t.Fatalf("LLM calls = %d, want 1", client.calls) - } - for _, want := range []string{"dnd-session", "outputs=1", "rejected=0", outputDir} { - if !strings.Contains(stdout.String(), want) { - t.Fatalf("stdout = %q, want substring %q", stdout.String(), want) - } - } - if stderr.Len() != 0 { - t.Fatalf("stderr = %q, want empty", stderr.String()) - } -} - -func TestRunPipelineOnlySelectsRequestedLane(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAMLForLanes("dnd-session", "spells", "rituals")) - inputPath := writeSeriatimInput(t) - outputDir := t.TempDir() - diagnosticsDir := t.TempDir() - client := newFakeRunLLMClient(false) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--only", "spells", "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(client, nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if client.calls != 1 { - t.Fatalf("LLM calls = %d, want only selected lane to run once", client.calls) - } - if !strings.Contains(stdout.String(), "outputs=1") { - t.Fatalf("stdout = %q, want output count", stdout.String()) - } -} - -func TestRunPipelineLLMFactoryFailure(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells")) - inputPath := writeSeriatimInput(t) - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), errors.New("factory unavailable")), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "create LLM client") || !strings.Contains(stderr.String(), "factory unavailable") { - t.Fatalf("stderr = %q, want LLM factory error", stderr.String()) - } -} - -func TestRunPipelineDefaultDNDSpellValidatorsRejectInvalidSourceRefs(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells")) - inputPath := writeSeriatimInput(t) - outputDir := t.TempDir() - diagnosticsDir := t.TempDir() - client := newFakeRunLLMClient(true) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(client, nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if !strings.Contains(stdout.String(), "outputs=0") || !strings.Contains(stdout.String(), "rejected=1") { - t.Fatalf("stdout = %q, want rejected output count", stdout.String()) - } - - var manifest artifacts.RunManifest - readJSONFile(t, filepath.Join(onlyChildDir(t, outputDir), "manifest.json"), &manifest) - if manifest.ValidationStatus != "rejected" { - t.Fatalf("validation status = %q, want rejected", manifest.ValidationStatus) - } - if len(manifest.RejectedOutputs) != 1 { - t.Fatalf("rejected outputs = %#v, want one rejection", manifest.RejectedOutputs) - } - rejection := manifest.RejectedOutputs[0] - if rejection.ValidatorName != spellsourcerefs.Key || rejection.ReasonCode != spellsourcerefs.ReasonCode { - t.Fatalf("rejection = %#v, want source reference validator rejection", rejection) - } - gotChain := manifestValidatorChain(t, manifest, pipeline.StageExtract, "spells", spells.Key) - wantKeys := []string{validjson.Key, validjsonschema.Key, spellshape.Key, spellsourcerefs.Key, spellrelatedness.Key} - if got := manifestValidatorKeys(gotChain); !reflect.DeepEqual(got, wantKeys) { - t.Fatalf("validator chain keys = %#v, want %#v", got, wantKeys) - } -} - -func TestRunPipelineExplicitEmptyValidatorOverrideDisablesDNDSpellDefaults(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAMLWithExtractValidators("dnd-session", " []\n")) - inputPath := writeSeriatimInput(t) - outputDir := t.TempDir() - diagnosticsDir := t.TempDir() - client := newFakeRunLLMClient(true) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(client, nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stdout.String(), "rejected=0") { - t.Fatalf("stdout = %q, want accepted output count", stdout.String()) - } - var manifest artifacts.RunManifest - readJSONFile(t, filepath.Join(onlyChildDir(t, outputDir), "manifest.json"), &manifest) - gotChain := manifestValidatorChain(t, manifest, pipeline.StageExtract, "spells", spells.Key) - if len(gotChain.Validators) != 0 { - t.Fatalf("validator chain = %#v, want explicit empty chain", gotChain) - } -} - -func TestRunPipelineExplicitValidatorOverrideReplacesDNDSpellDefaults(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAMLWithExtractValidators("dnd-session", "\n - "+alwaysaccept.Key+"\n")) - inputPath := writeSeriatimInput(t) - outputDir := t.TempDir() - diagnosticsDir := t.TempDir() - client := newFakeRunLLMClient(true) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(client, nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stdout.String(), "rejected=0") { - t.Fatalf("stdout = %q, want accepted output count", stdout.String()) - } - var manifest artifacts.RunManifest - readJSONFile(t, filepath.Join(onlyChildDir(t, outputDir), "manifest.json"), &manifest) - gotChain := manifestValidatorChain(t, manifest, pipeline.StageExtract, "spells", spells.Key) - if got := manifestValidatorKeys(gotChain); !reflect.DeepEqual(got, []string{alwaysaccept.Key}) { - t.Fatalf("validator chain keys = %#v, want explicit override", got) - } -} - -func TestRunPipelineConfiguredValidatorOrderIsPreserved(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAMLWithExtractValidators("dnd-session", "\n - "+alwaysaccept.Key+"\n - "+validjson.Key+"\n")) - inputPath := writeSeriatimInput(t) - outputDir := t.TempDir() - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - var manifest artifacts.RunManifest - readJSONFile(t, filepath.Join(onlyChildDir(t, outputDir), "manifest.json"), &manifest) - gotChain := manifestValidatorChain(t, manifest, pipeline.StageExtract, "spells", spells.Key) - wantKeys := []string{alwaysaccept.Key, validjson.Key} - if got := manifestValidatorKeys(gotChain); !reflect.DeepEqual(got, wantKeys) { - t.Fatalf("validator chain keys = %#v, want %#v", got, wantKeys) - } -} - -func TestRunPipelineLLMProfileOverrideSelectsFactoryProfile(t *testing.T) { - profilePath := writeScriptoriumProfileFile(t, "runtime", "http://profile.test/v1", "test-model") - configPath := writeTestConfig(t, mvpConfigYAMLWithProfileFile("dnd-session", profilePath)) - inputPath := writeSeriatimInput(t) - outputDir := t.TempDir() - diagnosticsDir := t.TempDir() - client := newFakeRunLLMClient(false) - factory := &recordingLLMFactory{client: client} - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--llm-profile", "runtime", "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{ - LLMClientFactory: factory.build, - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if got, want := strings.Join(factory.profileIDs, ","), "runtime"; got != want { - t.Fatalf("factory profile IDs = %q, want %q", got, want) - } -} - -func TestRunConfigValidateRejectsUnknownExplicitScriptoriumProfile(t *testing.T) { - profilePath := writeScriptoriumProfileFile(t, "known", "http://profile.test/v1", "test-model") - configPath := writeTestConfig(t, `version: 2 -scriptorium: - profile_file: `+profilePath+` -pipelines: - example: - input: fake/input - artifacts: - events: - extract: - module: fake/extract - llm_profile: missing -`) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "example"}, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "Scriptorium profile") || !strings.Contains(stderr.String(), "missing") { - t.Fatalf("stderr = %q, want unknown Scriptorium profile", stderr.String()) - } -} - -func TestRunConfigValidateChecksExplicitLLMValidatorProfileIDs(t *testing.T) { - profilePath := writeScriptoriumProfileFile(t, "known", "http://profile.test/v1", "test-model") - configPath := writeTestConfig(t, `version: 2 -scriptorium: - profile_file: `+profilePath+` -pipelines: - example: - input: fake/input - artifacts: - events: - extract: - module: fake/extract - validators: - - module: fake/llm-validator - llm_profile: missing -`) - catalog := fakeCatalog(t) - mustRegisterValidator(t, catalog.Validators, pipeline.ValidatorSpec{ - Key: "fake/llm-validator", - ExecutionClass: contracts.ExecutionClassLLMBacked, - }) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "example"}, &stdout, &stderr, Options{ - Catalog: catalog, - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "Scriptorium profile") || !strings.Contains(stderr.String(), "missing") { - t.Fatalf("stderr = %q, want unknown validator Scriptorium profile", stderr.String()) - } -} - -func TestRunConfigValidateIncludesMergeAndIgnoresNonLLMStageScriptoriumProfiles(t *testing.T) { - profilePath := writeScriptoriumProfileFile(t, "known", "http://profile.test/v1", "test-model") - configPath := writeTestConfig(t, `version: 2 -scriptorium: - profile_file: `+profilePath+` -pipelines: - example: - input: - module: fake/input - llm_profile: missing-input - output: - module: json - llm_profile: missing-output - artifacts: - events: - extract: fake/extract - merge: - module: appendorder - llm_profile: missing-merge -`) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "example"}, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want failure for missing merge profile", code) - } - if !strings.Contains(stderr.String(), "Scriptorium profile") || !strings.Contains(stderr.String(), "missing-merge") { - t.Fatalf("stderr = %q, want missing merge profile error", stderr.String()) - } -} - -func TestEffectiveLLMProfileIDsUsesLLMCapableStagesOnly(t *testing.T) { - resolved := pipeline.ResolvedPipeline{ - Input: pipeline.ModuleBinding{LLMProfile: "input-profile"}, - Chunk: pipeline.ModuleBinding{LLMProfile: "chunk-profile"}, - Output: pipeline.ModuleBinding{LLMProfile: "output-profile"}, - ArtifactLanes: []pipeline.ResolvedArtifactLane{ - { - Extract: pipeline.ModuleBinding{LLMProfile: "extract-profile"}, - Merge: pipeline.ModuleBinding{LLMProfile: "merge-profile"}, - Normalize: pipeline.ModuleBinding{LLMProfile: "normalize-profile"}, - }, - }, - ValidatorChains: []pipeline.ResolvedValidatorChain{ - { - Stage: pipeline.StageExtract, - LaneID: "events", - ModuleKey: "extract", - Validators: []pipeline.ResolvedValidator{ - { - Binding: pipeline.ModuleBinding{Module: "deterministic-validator", LLMProfile: "ignored-validator-profile"}, - ExecutionClass: contracts.ExecutionClassDeterministic, - }, - { - Binding: pipeline.ModuleBinding{Module: "llm-validator", LLMProfile: "validator-profile"}, - ExecutionClass: contracts.ExecutionClassLLMBacked, - }, - }, - }, - }, - } - - got := effectiveLLMProfileIDs(resolved) - want := []string{"chunk-profile", "extract-profile", "merge-profile", "normalize-profile", "validator-profile"} - if !reflect.DeepEqual(got, want) { - t.Fatalf("effectiveLLMProfileIDs() = %#v, want %#v", got, want) - } -} - -func TestRunPipelineSessionIDFlagRecordsExplicitTrimmedValue(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells")) - inputPath := writeSeriatimInput(t) - outputDir := t.TempDir() - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "dnd-session", - "--config", configPath, - "--input", inputPath, - "--session-id", " external-session ", - "--output-dir", outputDir, - "--diagnostics-dir", diagnosticsDir, - }, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - var manifest artifacts.RunManifest - readJSONFile(t, filepath.Join(onlyChildDir(t, outputDir), "manifest.json"), &manifest) - if got := manifest.Metadata["session_id"]; got != "external-session" { - t.Fatalf("manifest metadata = %#v, want trimmed session ID", manifest.Metadata) - } -} - -func TestRunPipelineSessionIDDefaultsToParsedSourceID(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells")) - inputPath := writeSeriatimInput(t) - outputDir := t.TempDir() - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "dnd-session", - "--config", configPath, - "--input", inputPath, - "--output-dir", outputDir, - "--diagnostics-dir", diagnosticsDir, - }, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - var manifest artifacts.RunManifest - readJSONFile(t, filepath.Join(onlyChildDir(t, outputDir), "manifest.json"), &manifest) - if got := manifest.Metadata["session_id"]; got != "session-alpha" { - t.Fatalf("manifest metadata = %#v, want parsed source ID default", manifest.Metadata) - } -} - -func TestRunPipelineSessionIDFlagRejectsMissingOrBlankValue(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAML("dnd-session", "dnd/spells")) - inputPath := writeSeriatimInput(t) - tests := []struct { - name string - args []string - want string - }{ - { - name: "missing value", - args: []string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--session-id"}, - want: "flag needs an argument", - }, - { - name: "blank value", - args: []string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--session-id", " \t "}, - want: "--session-id must not be empty", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions(test.args, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 2 { - t.Fatalf("RunWithOptions() code = %d, want 2", code) - } - if !strings.Contains(stderr.String(), test.want) { - t.Fatalf("stderr = %q, want substring %q", stderr.String(), test.want) - } - }) - } -} - -func TestRunPipelineReferenceFlagBindsUnambiguousSlot(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events")) - inputPath := filepath.Join(t.TempDir(), "missing.json") - referencePath := writeFile(t, "roster.yml", "Aria\n") - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--diagnostics-dir", diagnosticsDir, - "--reference", "roster=" + referencePath, - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, pipeline.ModuleSpec{ - Key: "fake/extract", - Stage: pipeline.StageExtract, - Requires: []string{"chunks"}, - Provides: []string{"artifact"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "roster"}, - }, - }), - }) - - if code != 1 || !strings.Contains(stderr.String(), "read input") { - t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String()) - } - resolved := readResolvedPipeline(t, diagnosticsDir) - refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings - want := []pipeline.ReferenceBinding{ - {LaneID: "events", SlotName: "roster", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI}, - } - if !reflect.DeepEqual(refs, want) { - t.Fatalf("resolved references = %#v, want %#v", refs, want) - } -} - -func TestCLIReferenceDiscoveryUsesLaneArtifactVariant(t *testing.T) { - catalog := referenceVariantCatalog(t) - cfg := config.Config{Pipelines: map[string]pipeline.PipelineProfile{ - "variants": { - ID: "variants", - Chunk: pipeline.Binding("generic"), - Artifacts: map[string]pipeline.ArtifactLaneProfile{ - "alpha": { - Extract: pipeline.Binding("extract/alpha"), - Merge: pipeline.Binding("shared/merge"), - Normalize: pipeline.Binding("shared/normalize"), - }, - "beta": { - Extract: pipeline.Binding("extract/beta"), - Merge: pipeline.Binding("shared/merge"), - Normalize: pipeline.Binding("shared/normalize"), - }, - }, - }, - }} - targets, err := selectedReferenceTargets(cfg, "variants", nil, catalog) - if err != nil { - t.Fatalf("selectedReferenceTargets() error = %v, want nil", err) - } - - tests := []struct { - name string - selector cliReferenceSelector - laneID string - stage pipeline.ModuleStage - }{ - { - name: "qualified merger", - selector: cliReferenceSelector{LaneID: "alpha", Stage: pipeline.StageMerge, SlotName: "alpha_merge"}, - laneID: "alpha", - stage: pipeline.StageMerge, - }, - { - name: "unqualified merger", - selector: cliReferenceSelector{Stage: pipeline.StageMerge, SlotName: "beta_merge"}, - laneID: "beta", - stage: pipeline.StageMerge, - }, - { - name: "flat normalizer", - selector: cliReferenceSelector{SlotName: "alpha_normalize"}, - laneID: "alpha", - stage: pipeline.StageNormalize, - }, - { - name: "lane normalizer", - selector: cliReferenceSelector{LaneID: "beta", SlotName: "beta_normalize"}, - laneID: "beta", - stage: pipeline.StageNormalize, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - target, err := resolveCLIReferenceTarget(targets, tc.selector) - if err != nil { - t.Fatalf("resolveCLIReferenceTarget() error = %v, want nil", err) - } - if target.laneID != tc.laneID || target.stage != tc.stage { - t.Fatalf("target = %#v, want lane %q stage %q", target, tc.laneID, tc.stage) - } - }) - } - - _, err = resolveCLIReferenceTarget(targets, cliReferenceSelector{LaneID: "beta", Stage: pipeline.StageMerge, SlotName: "alpha_merge"}) - if err == nil || !strings.Contains(err.Error(), "not declared") { - t.Fatalf("beta alpha-variant reference error = %v, want slot rejection", err) - } -} - -func TestCLIReferenceDiscoveryReportsMissingArtifactVariant(t *testing.T) { - catalog := referenceVariantCatalog(t) - if err := pipeline.RegisterExtractor(catalog.Extractors, pipeline.ModuleSpec{Key: "extract/missing", Stage: pipeline.StageExtract, ArtifactKind: "test/missing"}, func() (contracts.Extractor[fakeRunArtifact], error) { - return referenceVariantExtractor{key: "extract/missing"}, nil - }); err != nil { - t.Fatalf("register missing-kind extractor: %v", err) - } - cfg := config.Config{Pipelines: map[string]pipeline.PipelineProfile{ - "variants": { - ID: "variants", - Artifacts: map[string]pipeline.ArtifactLaneProfile{ - "missing": { - Extract: pipeline.Binding("extract/missing"), - Merge: pipeline.Binding("shared/merge"), - Normalize: pipeline.Binding("shared/normalize"), - }, - }, - }, - }} - _, err := selectedReferenceTargets(cfg, "variants", nil, catalog) - want := []string{"pipeline \"variants\"", "lane \"missing\"", "merge module \"shared/merge\"", "artifact kind \"test/missing\"", "registered kinds: test/alpha, test/beta"} - if err == nil { - t.Fatal("selectedReferenceTargets() error = nil, want missing variant error") - } - for _, value := range want { - if !strings.Contains(err.Error(), value) { - t.Fatalf("selectedReferenceTargets() error = %q, want %q", err, value) - } - } -} - -func TestRunPipelineReferenceFlagBindsLaneQualifiedSlot(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events", "notes")) - inputPath := filepath.Join(t.TempDir(), "missing.json") - referencePath := writeFile(t, "notes.yml", "Notes\n") - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--only", "events,notes", - "--diagnostics-dir", diagnosticsDir, - "--reference", "notes.roster=" + referencePath, - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, pipeline.ModuleSpec{ - Key: "fake/extract", - Stage: pipeline.StageExtract, - Requires: []string{"chunks"}, - Provides: []string{"artifact"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "roster"}, - }, - }), - }) - - if code != 1 || !strings.Contains(stderr.String(), "read input") { - t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String()) - } - resolved := readResolvedPipeline(t, diagnosticsDir) - events := resolvedArtifactLane(t, resolved, "events") - if len(events.ExtractReferences.Bindings) != 0 { - t.Fatalf("events references = %#v, want none", events.ExtractReferences.Bindings) - } - notes := resolvedArtifactLane(t, resolved, "notes") - if len(notes.ExtractReferences.Bindings) != 1 || notes.ExtractReferences.Bindings[0].Source != referencePath { - t.Fatalf("notes references = %#v, want lane-qualified binding", notes.ExtractReferences.Bindings) - } -} - -func TestRunPipelineReferenceFlagBindsChunkQualifiedSlot(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events")) - inputPath := filepath.Join(t.TempDir(), "missing.json") - referencePath := writeFile(t, "scenes.md", "Scenes\n") - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--diagnostics-dir", diagnosticsDir, - "--reference", "chunk.scene_guide=" + referencePath, - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, pipeline.ModuleSpec{ - Key: "generic", - Stage: pipeline.StageChunk, - Requires: []string{"source"}, - Provides: []string{"chunks"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "scene_guide"}, - }, - }), - }) - - if code != 1 || !strings.Contains(stderr.String(), "read input") { - t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String()) - } - resolved := readResolvedPipeline(t, diagnosticsDir) - refs := resolved.ChunkReferences.Bindings - want := []pipeline.ReferenceBinding{ - {SlotName: "scene_guide", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI}, - } - if !reflect.DeepEqual(refs, want) { - t.Fatalf("chunk references = %#v, want %#v", refs, want) - } -} - -func TestRunPipelineReferenceFlagBindsExplicitExtractSlot(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events")) - inputPath := filepath.Join(t.TempDir(), "missing.json") - referencePath := writeFile(t, "roster.yml", "Aria\n") - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--diagnostics-dir", diagnosticsDir, - "--reference", "events.extract.roster=" + referencePath, - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, pipeline.ModuleSpec{ - Key: "fake/extract", - Stage: pipeline.StageExtract, - Requires: []string{"chunks"}, - Provides: []string{"artifact"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "roster"}, - }, - }), - }) - - if code != 1 || !strings.Contains(stderr.String(), "read input") { - t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String()) - } - resolved := readResolvedPipeline(t, diagnosticsDir) - refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings - want := []pipeline.ReferenceBinding{ - {LaneID: "events", SlotName: "roster", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI}, - } - if !reflect.DeepEqual(refs, want) { - t.Fatalf("extract references = %#v, want %#v", refs, want) - } -} - -func TestRunPipelineReferenceFlagBindsExplicitNormalizeSlot(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events")) - inputPath := filepath.Join(t.TempDir(), "missing.json") - referencePath := writeFile(t, "normalize.md", "Normalize\n") - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--diagnostics-dir", diagnosticsDir, - "--reference", "events.normalize.notes=" + referencePath, - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, pipeline.ModuleSpec{ - Key: "noop", - Stage: pipeline.StageNormalize, - Requires: []string{"merged"}, - Provides: []string{"normalized"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "notes"}, - }, - }), - }) - - if code != 1 || !strings.Contains(stderr.String(), "read input") { - t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String()) - } - resolved := readResolvedPipeline(t, diagnosticsDir) - refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings - want := []pipeline.ReferenceBinding{ - {LaneID: "events", SlotName: "notes", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI}, - } - if !reflect.DeepEqual(refs, want) { - t.Fatalf("normalize references = %#v, want %#v", refs, want) - } -} - -func TestRunPipelineReferenceFlagBindsExplicitMergeSlot(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events")) - inputPath := filepath.Join(t.TempDir(), "missing.json") - referencePath := writeFile(t, "merge.md", "Merge notes\n") - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--diagnostics-dir", diagnosticsDir, - "--reference", "events.merge.notes=" + referencePath, - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, pipeline.ModuleSpec{ - Key: "appendorder", - Stage: pipeline.StageMerge, - Requires: []string{"artifact"}, - Provides: []string{"merged"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "notes"}, - }, - }), - }) - - if code != 1 || !strings.Contains(stderr.String(), "read input") { - t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String()) - } - resolved := readResolvedPipeline(t, diagnosticsDir) - refs := resolved.ArtifactLanes[0].MergeReferences.Bindings - want := []pipeline.ReferenceBinding{ - {LaneID: "events", SlotName: "notes", Source: referencePath, BindingSource: contracts.ReferenceBindingSourceCLI}, - } - if !reflect.DeepEqual(refs, want) { - t.Fatalf("merge references = %#v, want %#v", refs, want) - } -} - -func TestRunPipelineReferenceFlagBindsUnambiguousMergeSlot(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events")) - inputPath := filepath.Join(t.TempDir(), "missing.json") - referencePath := writeFile(t, "merge.md", "Merge notes\n") - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--diagnostics-dir", diagnosticsDir, - "--reference", "merge.notes=" + referencePath, - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, pipeline.ModuleSpec{ - Key: "appendorder", - Stage: pipeline.StageMerge, - Requires: []string{"artifact"}, - Provides: []string{"merged"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "notes"}, - }, - }), - }) - - if code != 1 || !strings.Contains(stderr.String(), "read input") { - t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String()) - } - resolved := readResolvedPipeline(t, diagnosticsDir) - refs := resolved.ArtifactLanes[0].MergeReferences.Bindings - if len(refs) != 1 || refs[0].Source != referencePath || refs[0].LaneID != "events" { - t.Fatalf("merge references = %#v, want unambiguous merge binding", refs) - } -} - -func TestRunPipelineReferenceFlagBindsFlatSlotAcrossOneTarget(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events")) - inputPath := filepath.Join(t.TempDir(), "missing.json") - referencePath := writeFile(t, "normalize.md", "Normalize\n") - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--diagnostics-dir", diagnosticsDir, - "--reference", "notes=" + referencePath, - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, pipeline.ModuleSpec{ - Key: "noop", - Stage: pipeline.StageNormalize, - Requires: []string{"merged"}, - Provides: []string{"normalized"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "notes"}, - }, - }), - }) - - if code != 1 || !strings.Contains(stderr.String(), "read input") { - t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String()) - } - resolved := readResolvedPipeline(t, diagnosticsDir) - if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 1 || refs[0].Source != referencePath { - t.Fatalf("normalize references = %#v, want flat binding", refs) - } -} - -func TestRunPipelineReferenceFlagBindsLaneSlotAcrossOneTarget(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events")) - inputPath := filepath.Join(t.TempDir(), "missing.json") - referencePath := writeFile(t, "normalize.md", "Normalize\n") - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--diagnostics-dir", diagnosticsDir, - "--reference", "events.notes=" + referencePath, - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, pipeline.ModuleSpec{ - Key: "noop", - Stage: pipeline.StageNormalize, - Requires: []string{"merged"}, - Provides: []string{"normalized"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "notes"}, - }, - }), - }) - - if code != 1 || !strings.Contains(stderr.String(), "read input") { - t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String()) - } - resolved := readResolvedPipeline(t, diagnosticsDir) - if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 1 || refs[0].Source != referencePath { - t.Fatalf("normalize references = %#v, want lane-qualified binding", refs) - } -} - -func TestRunPipelineReferenceFlagRejectsAmbiguousFlatSlot(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events", "notes")) - inputPath := writeSeriatimInput(t) - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--diagnostics-dir", diagnosticsDir, - "--reference", "roster=./roster.yml", - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, pipeline.ModuleSpec{ - Key: "fake/extract", - Stage: pipeline.StageExtract, - Requires: []string{"chunks"}, - Provides: []string{"artifact"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "roster"}, - }, - }), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "multiple selected targets") || !strings.Contains(stderr.String(), "events.extract.roster") || !strings.Contains(stderr.String(), "notes.extract.roster") { - t.Fatalf("stderr = %q, want ambiguous reference error", stderr.String()) - } -} - -func TestRunPipelineReferenceFlagRejectsAmbiguousFlatSlotAcrossChunkAndExtract(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events")) - inputPath := writeSeriatimInput(t) - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--diagnostics-dir", diagnosticsDir, - "--reference", "context=./context.md", - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, - pipeline.ModuleSpec{ - Key: "generic", - Stage: pipeline.StageChunk, - Requires: []string{"source"}, - Provides: []string{"chunks"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "context"}, - }, - }, - pipeline.ModuleSpec{ - Key: "fake/extract", - Stage: pipeline.StageExtract, - Requires: []string{"chunks"}, - Provides: []string{"artifact"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "context"}, - }, - }, - ), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "multiple selected targets") || !strings.Contains(stderr.String(), "chunk.context") || !strings.Contains(stderr.String(), "events.extract.context") { - t.Fatalf("stderr = %q, want ambiguous chunk/extract reference error", stderr.String()) - } -} - -func TestRunPipelineReferenceFlagRejectsAmbiguousLaneSlotAcrossExtractAndNormalize(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events")) - inputPath := writeSeriatimInput(t) - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--diagnostics-dir", diagnosticsDir, - "--reference", "events.context=./context.md", - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, - pipeline.ModuleSpec{ - Key: "fake/extract", - Stage: pipeline.StageExtract, - Requires: []string{"chunks"}, - Provides: []string{"artifact"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "context"}, - }, - }, - pipeline.ModuleSpec{ - Key: "noop", - Stage: pipeline.StageNormalize, - Requires: []string{"merged"}, - Provides: []string{"normalized"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "context"}, - }, - }, - ), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "multiple selected targets in lane") || !strings.Contains(stderr.String(), "events.extract.context") || !strings.Contains(stderr.String(), "events.normalize.context") { - t.Fatalf("stderr = %q, want ambiguous lane reference error", stderr.String()) - } -} - -func TestRunPipelineReferenceFlagsRejectMalformedValues(t *testing.T) { - tests := []struct { - name string - args []string - want string - }{ - {name: "missing equals", args: []string{"--reference", "roster"}, want: "slot=path"}, - {name: "empty path", args: []string{"--reference", "roster="}, want: "path must not be empty"}, - {name: "empty slot", args: []string{"--reference", "=./roster.yml"}, want: "slot must not be empty"}, - {name: "unsupported explicit stage", args: []string{"--reference", "a.b.c=./roster.yml"}, want: "lane.extract.slot, lane.merge.slot, or lane.normalize.slot"}, - {name: "too many selector parts", args: []string{"--reference", "a.b.c.d=./roster.yml"}, want: "slot, chunk.slot, merge.slot, lane.slot, lane.extract.slot, lane.merge.slot, or lane.normalize.slot"}, - {name: "unbind with equals", args: []string{"--without-reference", "roster=./roster.yml"}, want: "without =path"}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events")) - inputPath := writeSeriatimInput(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - args := []string{"run", "example", "--config", configPath, "--input", inputPath} - args = append(args, test.args...) - - code := RunWithOptions(args, &stdout, &stderr, Options{Catalog: fakeCatalog(t)}) - if code != 2 { - t.Fatalf("RunWithOptions() code = %d, want 2", code) - } - if !strings.Contains(stderr.String(), test.want) { - t.Fatalf("stderr = %q, want substring %q", stderr.String(), test.want) - } - }) - } -} - -func TestRunPipelineWithoutReferenceRemovesConfigBindingsForEligibleTargets(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAMLWithPipelineReferences("example", "events", map[string]string{ - "context": "./config-context.md", - "merge_notes": "./config-merge.md", - "notes": "./config-notes.md", - "roster": "./config-roster.yml", - })) - inputPath := filepath.Join(t.TempDir(), "missing.json") - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--diagnostics-dir", diagnosticsDir, - "--without-reference", "chunk.context", - "--without-reference", "events.extract.roster", - "--without-reference", "events.merge.merge_notes", - "--without-reference", "events.normalize.notes", - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, - pipeline.ModuleSpec{ - Key: "generic", - Stage: pipeline.StageChunk, - Requires: []string{"source"}, - Provides: []string{"chunks"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "context"}, - }, - }, - pipeline.ModuleSpec{ - Key: "appendorder", - Stage: pipeline.StageMerge, - Requires: []string{"artifact"}, - Provides: []string{"merged"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "merge_notes"}, - }, - }, - pipeline.ModuleSpec{ - Key: "fake/extract", - Stage: pipeline.StageExtract, - Requires: []string{"chunks"}, - Provides: []string{"artifact"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "roster"}, - }, - }, - pipeline.ModuleSpec{ - Key: "noop", - Stage: pipeline.StageNormalize, - Requires: []string{"merged"}, - Provides: []string{"normalized"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "notes"}, - }, - }, - ), - }) - - if code != 1 || !strings.Contains(stderr.String(), "read input") { - t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String()) - } - resolved := readResolvedPipeline(t, diagnosticsDir) - if refs := resolved.ChunkReferences.Bindings; len(refs) != 0 { - t.Fatalf("chunk references = %#v, want none", refs) - } - if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 { - t.Fatalf("extract references = %#v, want none", refs) - } - if refs := resolved.ArtifactLanes[0].MergeReferences.Bindings; len(refs) != 0 { - t.Fatalf("merge references = %#v, want none", refs) - } - if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 { - t.Fatalf("normalize references = %#v, want none", refs) - } -} - -func TestRunPipelineWithoutReferenceRemovesOptionalConfigBinding(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAMLWithReferences("example", "events", map[string]string{"roster": "./config-roster.yml"})) - inputPath := filepath.Join(t.TempDir(), "missing.json") - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--diagnostics-dir", diagnosticsDir, - "--without-reference", "roster", - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, pipeline.ModuleSpec{ - Key: "fake/extract", - Stage: pipeline.StageExtract, - Requires: []string{"chunks"}, - Provides: []string{"artifact"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "roster"}, - }, - }), - }) - - if code != 1 || !strings.Contains(stderr.String(), "read input") { - t.Fatalf("RunWithOptions() code = %d stderr=%q, want read input failure after resolution", code, stderr.String()) - } - resolved := readResolvedPipeline(t, diagnosticsDir) - if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 { - t.Fatalf("references = %#v, want unbound optional slot", refs) - } -} - -func TestRunPipelineWithoutReferenceFailsWhenRequiredChunkSlotWouldBeMissing(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAMLWithPipelineReferences("example", "events", map[string]string{"context": "./config-context.md"})) - inputPath := writeSeriatimInput(t) - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--diagnostics-dir", diagnosticsDir, - "--without-reference", "chunk.context", - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, pipeline.ModuleSpec{ - Key: "generic", - Stage: pipeline.StageChunk, - Requires: []string{"source"}, - Provides: []string{"chunks"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "context", Required: true}, - }, - }), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "required reference slot") || !strings.Contains(stderr.String(), "context") { - t.Fatalf("stderr = %q, want required chunk reference error", stderr.String()) - } -} - -func TestRunPipelineWithoutReferenceFailsWhenRequiredNormalizeSlotWouldBeMissing(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAMLWithPipelineReferences("example", "events", map[string]string{"notes": "./config-notes.md"})) - inputPath := writeSeriatimInput(t) - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--diagnostics-dir", diagnosticsDir, - "--without-reference", "events.normalize.notes", - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, pipeline.ModuleSpec{ - Key: "noop", - Stage: pipeline.StageNormalize, - Requires: []string{"merged"}, - Provides: []string{"normalized"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "notes", Required: true}, - }, - }), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "required reference slot") || !strings.Contains(stderr.String(), "notes") { - t.Fatalf("stderr = %q, want required normalize reference error", stderr.String()) - } -} - -func TestRunPipelineReferenceFlagRejectsUnselectedLaneSelector(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAML("example", "events", "notes")) - inputPath := writeSeriatimInput(t) - referencePath := writeFile(t, "notes.yml", "Notes\n") - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--only", "events", - "--diagnostics-dir", diagnosticsDir, - "--reference", "notes.extract.roster=" + referencePath, - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, pipeline.ModuleSpec{ - Key: "fake/extract", - Stage: pipeline.StageExtract, - Requires: []string{"chunks"}, - Provides: []string{"artifact"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "roster"}, - }, - }), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), `reference lane "notes" is not selected`) { - t.Fatalf("stderr = %q, want unselected lane error", stderr.String()) - } -} - -func TestRunPipelineWithoutReferenceFailsWhenRequiredSlotWouldBeMissing(t *testing.T) { - configPath := writeTestConfig(t, testConfigYAMLWithReferences("example", "events", map[string]string{"roster": "./config-roster.yml"})) - inputPath := writeSeriatimInput(t) - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{ - "run", "example", - "--config", configPath, - "--input", inputPath, - "--diagnostics-dir", diagnosticsDir, - "--without-reference", "roster", - }, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, pipeline.ModuleSpec{ - Key: "fake/extract", - Stage: pipeline.StageExtract, - Requires: []string{"chunks"}, - Provides: []string{"artifact"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "roster", Required: true}, - }, - }), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "required reference slot") || !strings.Contains(stderr.String(), "roster") { - t.Fatalf("stderr = %q, want required reference error", stderr.String()) - } -} - -func TestRunPipelineWritesDurableOutputFiles(t *testing.T) { - diagnosticsDir := t.TempDir() - outputDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", diagnosticsDir, "always")) - inputPath := writeSeriatimInput(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - runOutputDir := onlyChildDir(t, outputDir) - for _, name := range []string{ - "index.json", - "lanes/spells.json", - "manifest.json", - "rejected.json", - "warnings.json", - } { - if _, err := os.Stat(filepath.Join(runOutputDir, filepath.FromSlash(name))); err != nil { - t.Fatalf("expected output file %q: %v", name, err) - } - } - if !strings.Contains(stdout.String(), runOutputDir) { - t.Fatalf("stdout = %q, want output path %q", stdout.String(), runOutputDir) - } - assertNoTemporaryFiles(t, runOutputDir) -} - -func TestRunPipelineReferenceBytesProduceDistinctManifests(t *testing.T) { - run := func(t *testing.T, referenceText string) artifacts.RunManifest { - t.Helper() - - diagnosticsDir := t.TempDir() - outputDir := t.TempDir() - configDir := t.TempDir() - referencePath := filepath.Join(configDir, "roster.txt") - if err := os.WriteFile(referencePath, []byte(referenceText), 0o644); err != nil { - t.Fatalf("write reference: %v", err) - } - configPath := filepath.Join(configDir, "config.yml") - if err := os.WriteFile(configPath, []byte(testConfigYAMLWithReferencesAndDiagnostics("example", "events", diagnosticsDir, map[string]string{"roster": "roster.txt"})), 0o644); err != nil { - t.Fatalf("write config: %v", err) - } - inputPath := writeFile(t, "source.txt", "source text") - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "example", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{ - Catalog: fakeCatalog(t, pipeline.ModuleSpec{ - Key: "fake/extract", - Stage: pipeline.StageExtract, - Requires: []string{"chunks"}, - Provides: []string{"artifact"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "roster"}, - }, - }), - Registries: fakeExecutionRegistries(t), - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - - var manifest artifacts.RunManifest - readJSONFile(t, filepath.Join(onlyChildDir(t, outputDir), "manifest.json"), &manifest) - if len(manifest.References) != 1 { - t.Fatalf("manifest references = %#v, want one entry", manifest.References) - } - if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:source"}) { - t.Fatalf("source digests = %#v, want source-only digest", manifest.SourceDigests) - } - - var resolvedReferences []artifacts.ReferenceProvenance - readJSONFile(t, filepath.Join(onlyChildDir(t, diagnosticsDir), diagnostics.ArtifactResolvedReferences), &resolvedReferences) - if !reflect.DeepEqual(resolvedReferences, manifest.References) { - t.Fatalf("resolved references = %#v, want manifest references %#v", resolvedReferences, manifest.References) - } - runManifestJSON := string(readFile(t, filepath.Join(onlyChildDir(t, diagnosticsDir), diagnostics.ArtifactRunManifest))) - if strings.Contains(runManifestJSON, "source text") || strings.Contains(runManifestJSON, referenceText) { - t.Fatalf("run manifest diagnostics contains raw prompt material: %s", runManifestJSON) - } - resolvedReferenceJSON := string(readFile(t, filepath.Join(onlyChildDir(t, diagnosticsDir), diagnostics.ArtifactResolvedReferences))) - if strings.Contains(resolvedReferenceJSON, referenceText) || strings.Contains(resolvedReferenceJSON, "content") { - t.Fatalf("resolved references diagnostics contains content: %s", resolvedReferenceJSON) - } - return manifest - } - - first := run(t, "first roster") - second := run(t, "second roster") - - if first.References[0].Digest == second.References[0].Digest { - t.Fatalf("reference digests match for different bytes: %q", first.References[0].Digest) - } - if first.PipelineDigest != second.PipelineDigest { - t.Fatalf("pipeline digests differ = %q vs %q, want reference bytes outside pipeline identity", first.PipelineDigest, second.PipelineDigest) - } -} - -func TestRunPipelineRejectsUnsafeOutputFileName(t *testing.T) { - diagnosticsDir := t.TempDir() - outputDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", diagnosticsDir, "always")) - inputPath := writeSeriatimInput(t) - registries := registriesWithOutput(t, unsafeOutputEncoder{}) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{ - Registries: registries, - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "output file name") { - t.Fatalf("stderr = %q, want unsafe output file error", stderr.String()) - } - runDir := onlyChildDir(t, diagnosticsDir) - if got := string(readFile(t, filepath.Join(runDir, diagnostics.ArtifactErrorLog))); !strings.Contains(got, "output file name") { - t.Fatalf("error log = %q, want unsafe output file error", got) - } - if _, err := os.Stat(filepath.Join(runDir, diagnostics.ArtifactRunManifest)); err != nil { - t.Fatalf("expected diagnostics manifest after unsafe output file failure: %v", err) - } -} - -func TestRunPipelineWritesDiagnosticsArtifactsWhenDurableOutputWriteFails(t *testing.T) { - diagnosticsDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", diagnosticsDir, "always")) - inputPath := writeSeriatimInput(t) - outputRootFile := writeFile(t, "not-a-directory", "occupied") - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputRootFile}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "create output directory") { - t.Fatalf("stderr = %q, want output directory error", stderr.String()) - } - runDir := onlyChildDir(t, diagnosticsDir) - for _, name := range []string{ - diagnostics.ArtifactRunManifest, - diagnostics.ArtifactWarnings, - diagnostics.ArtifactRunReport, - diagnostics.ArtifactErrorLog, - } { - if _, err := os.Stat(filepath.Join(runDir, name)); err != nil { - t.Fatalf("expected diagnostics artifact %q after durable output write failure: %v", name, err) - } - } -} - -func TestRunPipelineWritesDiagnosticsArtifactsOnSuccess(t *testing.T) { - diagnosticsDir := t.TempDir() - outputDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", diagnosticsDir, "always")) - inputPath := writeSeriatimInput(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - runDir := onlyChildDir(t, diagnosticsDir) - for _, name := range []string{ - diagnostics.ArtifactInvocationMetadata, - diagnostics.ArtifactEffectiveConfig, - diagnostics.ArtifactResolvedPipeline, - diagnostics.ArtifactRunManifest, - diagnostics.ArtifactRunReport, - diagnostics.ArtifactWarnings, - } { - if _, err := os.Stat(filepath.Join(runDir, name)); err != nil { - t.Fatalf("expected diagnostics artifact %q: %v", name, err) - } - } - report := string(readFile(t, filepath.Join(runDir, diagnostics.ArtifactRunReport))) - if !strings.Contains(report, `"output_count": 1`) || !strings.Contains(report, `"validation_status": "approved"`) || !strings.Contains(report, outputDir) { - t.Fatalf("unexpected run report: %s", report) - } -} - -func TestRunPipelineWritesWorkspaceDiagnosticsArtifactsOnSuccess(t *testing.T) { - workspaceDir := filepath.Join(t.TempDir(), "workspace") - outputDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDiagnostics("dnd-session", workspaceDir, "always")) - inputPath := writeSeriatimInput(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - runDir := onlyChildDir(t, filepath.Join(workspaceDir, "diagnostics")) - for _, name := range []string{ - diagnostics.ArtifactInvocationMetadata, - diagnostics.ArtifactEffectiveConfig, - diagnostics.ArtifactResolvedPipeline, - diagnostics.ArtifactRunManifest, - diagnostics.ArtifactRunReport, - diagnostics.ArtifactWarnings, - } { - if _, err := os.Stat(filepath.Join(runDir, name)); err != nil { - t.Fatalf("expected workspace diagnostics artifact %q: %v", name, err) - } - } - assertPathNotExist(t, filepath.Join(workspaceDir, "checkpoints")) - assertPathNotExist(t, filepath.Join(workspaceDir, "debug")) -} - -func TestRunPipelineWritesCheckpointsWhenWorkspaceResumeEnabled(t *testing.T) { - workspaceDir := filepath.Join(t.TempDir(), "workspace") - outputDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeEnabled("dnd-session", workspaceDir, "always")) - inputPath := writeSeriatimInput(t) - client := newFakeRunLLMClient(false) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(client, nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if client.calls != 1 { - t.Fatalf("LLM calls = %d, want ordinary run to execute despite checkpoint writing", client.calls) - } - checkpointDir := onlyCheckpointIdentityDir(t, workspaceDir) - for _, name := range []string{ - "source/manifest.json", - "source/source-document.json", - "extract/spells/manifest.json", - "extract/spells/outputs.json", - "merge/spells/manifest.json", - "merge/spells/output.json", - "normalize/spells/manifest.json", - "normalize/spells/output.json", - } { - if _, err := os.Stat(filepath.Join(checkpointDir, name)); err != nil { - t.Fatalf("expected checkpoint artifact %q: %v", name, err) - } - } - assertPathNotExist(t, filepath.Join(checkpointDir, "chunk")) - assertPathNotExist(t, filepath.Join(workspaceDir, "debug")) -} - -func TestRunPipelineWritesDebugWhenWorkspaceDebugEnabled(t *testing.T) { - workspaceDir := filepath.Join(t.TempDir(), "workspace") - outputDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDebugEnabled("dnd-session", workspaceDir, "always")) - inputPath := writeSeriatimInput(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - debugDir := onlyChildDir(t, filepath.Join(workspaceDir, "debug")) - for _, name := range []string{ - "run.json", - "source/input.json", - "source/output.json", - "chunk/input.json", - "chunk/output.json", - "extract/spells/input.json", - "extract/spells/chunk-000001/attempt-01.json", - "extract/spells/chunk-000001/attempt-01/prompt-0001.json", - "extract/spells/chunk-000001/attempt-01/response-0001.json", - "extract/spells/chunk-000001/attempt-01/response-content-0001.json", - "extract/spells/output.json", - "merge/spells/input.json", - "merge/spells/output.json", - "normalize/spells/input.json", - "normalize/spells/output.json", - "output/input.json", - "output/output.json", - } { - if _, err := os.Stat(filepath.Join(debugDir, name)); err != nil { - t.Fatalf("expected debug artifact %q: %v", name, err) - } - } - attemptDebug := string(readFile(t, filepath.Join(debugDir, "extract/spells/chunk-000001/attempt-01.json"))) - if !strings.Contains(attemptDebug, `"llm_calls"`) || !strings.Contains(attemptDebug, `"prompt_path"`) || !strings.Contains(attemptDebug, `"response_path"`) || !strings.Contains(attemptDebug, `"response_content_path"`) { - t.Fatalf("extract attempt debug = %s, want prompt/response llm_calls", attemptDebug) - } - responseDebug := string(readFile(t, filepath.Join(debugDir, "extract/spells/chunk-000001/attempt-01/response-0001.json"))) - if !strings.Contains(responseDebug, `"content_path"`) || strings.Contains(responseDebug, `spell_casts`) { - t.Fatalf("response debug = %s, want metadata with content path and no inline response", responseDebug) - } - responseContent := string(readFile(t, filepath.Join(debugDir, "extract/spells/chunk-000001/attempt-01/response-content-0001.json"))) - if !strings.Contains(responseContent, `"spell_casts"`) || !strings.Contains(responseContent, "\n ") { - t.Fatalf("response content = %s, want pretty JSON response body", responseContent) - } - assertPathNotExist(t, filepath.Join(debugDir, "llm/call-0001.json")) - assertPathNotExist(t, filepath.Join(debugDir, "extract/spells/chunk-000001-attempt-01/llm-call-0001.json")) - if _, err := os.Stat(filepath.Join(debugDir, "extract/spells/chunk-000001-attempt-01.json")); !os.IsNotExist(err) { - t.Fatalf("old extract attempt path still exists: %v", err) - } - assertPathNotExist(t, filepath.Join(workspaceDir, "checkpoints")) -} - -func TestRunPipelineDebugAndResumeCanBeEnabledIndependently(t *testing.T) { - workspaceDir := filepath.Join(t.TempDir(), "workspace") - outputDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDiagnosticsAndStateEnabled("dnd-session", workspaceDir, "always")) - inputPath := writeSeriatimInput(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - if code != 0 { - t.Fatalf("seed RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - code = RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--resume"}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - if code != 0 { - t.Fatalf("resume RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if entries := childDirs(t, filepath.Join(workspaceDir, "checkpoints")); len(entries) != 1 { - t.Fatalf("workspace checkpoint pipeline dirs = %v, want one", entries) - } - if entries := childDirs(t, filepath.Join(workspaceDir, "debug")); len(entries) != 2 { - t.Fatalf("workspace debug run dirs = %v, want two", entries) - } -} - -func TestRunPipelineDebugRedactsObviousSecrets(t *testing.T) { - workspaceDir := filepath.Join(t.TempDir(), "workspace") - outputDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDebugEnabled("dnd-session", workspaceDir, "always")) - inputPath := writeSeriatimInput(t) - client := newFakeRunLLMClient(false) - client.payload = map[string]any{ - "spell_casts": []map[string]any{ - { - "caster": "Aria", - "spell": "sk-secretvalue", - "effect": "Bearer secretvalue", - "narrative_description": "Aria casts a spell.", - "source_refs": []map[string]any{ - {"source_id": "session-alpha", "start_unit_id": 1, "end_unit_id": 1}, - }, - }, - }, - } - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(client, nil), - }) - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - assertDebugTreeDoesNotContain(t, onlyChildDir(t, filepath.Join(workspaceDir, "debug")), "sk-secretvalue", "Bearer secretvalue") -} - -func TestWorkspaceStateRootsDoNotOverlap(t *testing.T) { - workspaceDir := filepath.Join(t.TempDir(), "workspace") - outputDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDiagnosticsAndStateEnabled("dnd-session", workspaceDir, "always")) - inputPath := writeSeriatimInput(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - assertDistinctRoots(t, filepath.Join(workspaceDir, "diagnostics"), filepath.Join(workspaceDir, "checkpoints"), filepath.Join(workspaceDir, "debug")) -} - -func TestRunPipelineResumeRequiresWorkspaceResumeEnabled(t *testing.T) { - workspaceDir := filepath.Join(t.TempDir(), "workspace") - outputDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDiagnostics("dnd-session", workspaceDir, "always")) - inputPath := writeSeriatimInput(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--resume"}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - if !strings.Contains(stderr.String(), "--resume requires workspace.resume.enabled: true") { - t.Fatalf("stderr = %q, want resume configuration error", stderr.String()) - } -} - -func TestRunPipelineResumeReusesWorkspaceCheckpoints(t *testing.T) { - workspaceDir := filepath.Join(t.TempDir(), "workspace") - outputDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeEnabled("dnd-session", workspaceDir, "always")) - inputPath := writeSeriatimInput(t) - firstClient := newFakeRunLLMClient(false) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(firstClient, nil), - }) - if code != 0 { - t.Fatalf("first RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if firstClient.calls != 1 { - t.Fatalf("first LLM calls = %d, want checkpoint seed run to execute", firstClient.calls) - } - - secondClient := newFakeRunLLMClient(false) - stdout.Reset() - stderr.Reset() - code = RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--resume"}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(secondClient, nil), - }) - if code != 0 { - t.Fatalf("resume RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if secondClient.calls != 0 { - t.Fatalf("resume LLM calls = %d, want checkpoint reuse", secondClient.calls) - } - runDirs := childDirs(t, filepath.Join(workspaceDir, "diagnostics")) - if len(runDirs) != 2 { - t.Fatalf("diagnostics run dirs = %v, want fresh diagnostics for each invocation", runDirs) - } - if !anyDiagnosticsFileContains(t, runDirs, diagnostics.ArtifactCheckpointEvents, `"action": "reused"`) { - t.Fatalf("checkpoint event diagnostics under %v did not record reuse", runDirs) - } -} - -func TestRunPipelineResumeInvalidatesWhenInvocationIdentityChanges(t *testing.T) { - t.Run("input", func(t *testing.T) { - workspaceDir := filepath.Join(t.TempDir(), "workspace") - configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeEnabled("dnd-session", workspaceDir, "always")) - seedInput := writeSeriatimInput(t) - changedInput := writeFile(t, "source-changed.json", `{ - "metadata": {"id": "session-alpha"}, - "segments": [{"id": 1, "start": 0, "end": 1, "speaker": "Aria", "text": "Aria casts Shield."}] -}`) - seedWorkspaceCheckpoint(t, configPath, seedInput, nil) - client := runResumeWithClient(t, configPath, changedInput, nil) - if client.calls == 0 { - t.Fatal("resume LLM calls = 0, want execution after input identity change") - } - }) - - t.Run("pipeline digest", func(t *testing.T) { - workspaceDir := filepath.Join(t.TempDir(), "workspace") - seedConfig := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeEnabled("dnd-session", workspaceDir, "always")) - changedConfig := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeAndChunkOptions("dnd-session", workspaceDir, "always")) - inputPath := writeSeriatimInput(t) - seedWorkspaceCheckpoint(t, seedConfig, inputPath, nil) - client := runResumeWithClient(t, changedConfig, inputPath, nil) - if client.calls == 0 { - t.Fatal("resume LLM calls = 0, want execution after pipeline digest change") - } - }) - - t.Run("selected lanes", func(t *testing.T) { - workspaceDir := filepath.Join(t.TempDir(), "workspace") - configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeLanes("dnd-session", workspaceDir, "always", "spells", "items")) - inputPath := writeSeriatimInput(t) - seedWorkspaceCheckpoint(t, configPath, inputPath, nil) - client := runResumeWithClient(t, configPath, inputPath, []string{"--only", "spells"}) - if client.calls == 0 { - t.Fatal("resume LLM calls = 0, want execution after selected lane change") - } - }) - - t.Run("references", func(t *testing.T) { - workspaceDir := filepath.Join(t.TempDir(), "workspace") - configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeEnabled("dnd-session", workspaceDir, "always")) - inputPath := writeSeriatimInput(t) - referencePath := writeFile(t, "players.md", "Aria is a cleric.\n") - seedWorkspaceCheckpoint(t, configPath, inputPath, []string{"--reference", "players=" + referencePath}) - if err := os.WriteFile(referencePath, []byte("Aria is a wizard.\n"), 0o644); err != nil { - t.Fatalf("update reference: %v", err) - } - client := runResumeWithClient(t, configPath, inputPath, []string{"--reference", "players=" + referencePath}) - if client.calls == 0 { - t.Fatal("resume LLM calls = 0, want execution after reference digest change") - } - }) - - t.Run("llm profile override", func(t *testing.T) { - workspaceDir := filepath.Join(t.TempDir(), "workspace") - profilePath := writeScriptoriumProfileFile(t, "runtime", "http://profile.test/v1", "test-model") - configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeAndProfileFile("dnd-session", workspaceDir, "always", profilePath)) - inputPath := writeSeriatimInput(t) - seedWorkspaceCheckpoint(t, configPath, inputPath, nil) - client := runResumeWithClient(t, configPath, inputPath, []string{"--llm-profile", "runtime"}) - if client.calls == 0 { - t.Fatal("resume LLM calls = 0, want execution after LLM profile override change") - } - }) -} - -func TestRunPipelineSkipsDiagnosticsWhenWorkspaceDiagnosticsDisabled(t *testing.T) { - workspaceDir := filepath.Join(t.TempDir(), "workspace") - outputDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDiagnosticsDisabled("dnd-session", workspaceDir)) - inputPath := writeSeriatimInput(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if _, err := os.Stat(filepath.Join(workspaceDir, "diagnostics")); !os.IsNotExist(err) { - t.Fatalf("workspace diagnostics dir stat err = %v, want not exist", err) - } - if entries := childDirs(t, outputDir); len(entries) != 1 { - t.Fatalf("output run dirs = %v, want one run dir", entries) - } -} - -func TestRunPipelineLegacyDiagnosticsConfigStillWritesDiagnostics(t *testing.T) { - diagnosticsDir := t.TempDir() - outputDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", diagnosticsDir, "always")) - inputPath := writeSeriatimInput(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - runDir := onlyChildDir(t, diagnosticsDir) - if _, err := os.Stat(filepath.Join(runDir, diagnostics.ArtifactRunReport)); err != nil { - t.Fatalf("expected legacy diagnostics run report: %v", err) - } -} - -func TestRunPipelineWritesErrorLogAfterDiagnosticsCreation(t *testing.T) { - diagnosticsDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", diagnosticsDir, "always")) - inputPath := writeSeriatimInput(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), errors.New("factory unavailable")), - }) - - if code != 1 { - t.Fatalf("RunWithOptions() code = %d, want 1", code) - } - runDir := onlyChildDir(t, diagnosticsDir) - if got := string(readFile(t, filepath.Join(runDir, diagnostics.ArtifactErrorLog))); !strings.Contains(got, "factory unavailable") { - t.Fatalf("error log = %q, want factory error", got) - } -} - -func TestRunPipelineRetentionNeverRemovesSuccessfulWarningFreeDiagnostics(t *testing.T) { - diagnosticsDir := t.TempDir() - outputDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", diagnosticsDir, "never")) - inputPath := writeSeriatimInput(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if entries := childDirs(t, diagnosticsDir); len(entries) != 0 { - t.Fatalf("diagnostics run dirs = %v, want none", entries) - } -} - -func TestRunPipelineWarningsAreDiagnosedAndReported(t *testing.T) { - diagnosticsDir := t.TempDir() - outputDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", diagnosticsDir, "auto")) - inputPath := writeSeriatimInput(t) - registries := registriesWithOutput(t, warningOutputEncoder{}) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{ - Registries: registries, - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if !strings.Contains(stderr.String(), "1 warning") { - t.Fatalf("stderr = %q, want warning count", stderr.String()) - } - runDir := onlyChildDir(t, diagnosticsDir) - warnings := string(readFile(t, filepath.Join(runDir, diagnostics.ArtifactWarnings))) - if !strings.Contains(warnings, "synthetic_warning") { - t.Fatalf("warnings artifact = %q, want synthetic warning", warnings) - } -} - -func TestRunPipelineDiagnosticsDirFlagOverridesConfig(t *testing.T) { - configDiagnosticsDir := t.TempDir() - overrideDiagnosticsDir := t.TempDir() - outputDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", configDiagnosticsDir, "always")) - inputPath := writeSeriatimInput(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", overrideDiagnosticsDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if entries := childDirs(t, configDiagnosticsDir); len(entries) != 0 { - t.Fatalf("config diagnostics dir entries = %v, want none", entries) - } - if entries := childDirs(t, overrideDiagnosticsDir); len(entries) != 1 { - t.Fatalf("override diagnostics dir entries = %v, want one run dir", entries) - } -} - -func TestRunPipelineDiagnosticsDirFlagOverridesWorkspaceDiagnosticsOnly(t *testing.T) { - workspaceDir := filepath.Join(t.TempDir(), "workspace") - overrideDiagnosticsDir := t.TempDir() - outputDir := t.TempDir() - configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDiagnosticsAndStateEnabled("dnd-session", workspaceDir, "always")) - inputPath := writeSeriatimInput(t) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", overrideDiagnosticsDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if entries := childDirs(t, overrideDiagnosticsDir); len(entries) != 1 { - t.Fatalf("override diagnostics dir entries = %v, want one run dir", entries) - } - assertPathNotExist(t, filepath.Join(workspaceDir, "diagnostics")) - if entries := childDirs(t, filepath.Join(workspaceDir, "checkpoints")); len(entries) != 1 { - t.Fatalf("workspace checkpoint pipeline dirs = %v, want one", entries) - } - if entries := childDirs(t, filepath.Join(workspaceDir, "debug")); len(entries) != 1 { - t.Fatalf("workspace debug run dirs = %v, want one", entries) - } -} - -func TestExampleFixtureConfigValidateAndPipelinesList(t *testing.T) { - for _, path := range []string{ - "examples/dnd-spells.config.yml", - "examples/dnd-spells-production.config.yml", - } { - path := path - t.Run("validate "+filepath.Base(path), func(t *testing.T) { - configPath := fixturePath(t, path) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "dnd-session"}, &stdout, &stderr, Options{}) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if !strings.Contains(stdout.String(), "dnd-session") { - t.Fatalf("stdout = %q, want pipeline ID", stdout.String()) - } - }) - } - - t.Run("list configured pipelines", func(t *testing.T) { - configPath := fixturePath(t, "examples/dnd-spells.config.yml") - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"pipelines", "list", "--config", configPath}, &stdout, &stderr, Options{}) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if got, want := stdout.String(), "dnd-session\n"; got != want { - t.Fatalf("stdout = %q, want %q", got, want) - } - }) -} - -func TestExampleFixtureRunWritesExpectedJSON(t *testing.T) { - configPath := fixturePath(t, "examples/dnd-spells.config.yml") - inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json") - outputDir := t.TempDir() - diagnosticsDir := t.TempDir() - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - runOutputDir := onlyChildDir(t, outputDir) - if !strings.Contains(stdout.String(), runOutputDir) { - t.Fatalf("stdout = %q, want output path %q", stdout.String(), runOutputDir) - } - - var manifest artifacts.RunManifest - readJSONFile(t, filepath.Join(runOutputDir, "manifest.json"), &manifest) - if manifest.PipelineID != "dnd-session" { - t.Fatalf("manifest pipeline ID = %q, want dnd-session", manifest.PipelineID) - } - if manifest.PipelineDigest == "" { - t.Fatal("manifest pipeline digest is empty") - } - if manifest.ValidationStatus != "approved" { - t.Fatalf("validation status = %q, want approved", manifest.ValidationStatus) - } - if len(manifest.ArtifactLanes) != 1 { - t.Fatalf("artifact lanes = %#v, want one lane", manifest.ArtifactLanes) - } - extractorMetadata, ok := manifest.ArtifactLanes[0].Metadata["extractor"].(map[string]any) - if !ok { - t.Fatalf("extractor metadata = %#v, want object", manifest.ArtifactLanes[0].Metadata) - } - if extractorMetadata["prompt_id"] != "dnd.spells" || extractorMetadata["response_schema_key"] != "dnd_spells" { - t.Fatalf("extractor metadata = %#v, want prompt/schema identifiers", extractorMetadata) - } - - var spellOutput struct { - SpellCasts []struct { - Caster string `json:"caster"` - Spell string `json:"spell"` - Effect string `json:"effect"` - SourceRefs []source.SourceRef `json:"source_refs"` - } `json:"spell_casts"` - } - readJSONFile(t, filepath.Join(runOutputDir, "lanes", "spells.json"), &spellOutput) - if len(spellOutput.SpellCasts) != 1 { - t.Fatalf("spell output = %#v, want one spell cast", spellOutput) - } - payload := spellOutput.SpellCasts[0] - if payload.Caster != "Aria" || payload.Spell != "Cure Wounds" || payload.Effect == "" { - t.Fatalf("payload = %#v, want deterministic spell output", payload) - } - if len(payload.SourceRefs) != 1 { - t.Fatalf("source refs = %#v, want one source ref", payload.SourceRefs) - } - ref := payload.SourceRefs[0] - if ref.SourceID != "session-alpha" || ref.StartUnitID != 1 || ref.EndUnitID != 1 { - t.Fatalf("source ref = %#v, want fixture source ref", ref) - } - - warnings := string(readFile(t, filepath.Join(runOutputDir, "warnings.json"))) - if !strings.Contains(warnings, `"warnings": []`) { - t.Fatalf("warnings output = %s, want empty warnings", warnings) - } -} - -func TestExampleFixtureRunWithDNDScenesRecordsChunkerAndWarnings(t *testing.T) { - configPath := writeTestConfig(t, mvpConfigYAMLWithChunk("dnd-session", scenes.Key, "dnd/spells")) - inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json") - outputDir := t.TempDir() - diagnosticsDir := t.TempDir() - client := newSceneRunLLMClient("Scene boundary was ambiguous.") - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(client, nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if client.calls != 2 { - t.Fatalf("LLM calls = %d, want chunking and extraction calls", client.calls) - } - if !strings.Contains(stderr.String(), "1 warning") { - t.Fatalf("stderr = %q, want warning count", stderr.String()) - } - - runOutputDir := onlyChildDir(t, outputDir) - manifestBytes := readFile(t, filepath.Join(runOutputDir, "manifest.json")) - var manifest artifacts.RunManifest - if err := json.Unmarshal(manifestBytes, &manifest); err != nil { - t.Fatalf("unmarshal manifest: %v", err) - } - if manifest.Chunker != scenes.Key { - t.Fatalf("manifest chunker = %q, want %q", manifest.Chunker, scenes.Key) - } - chunkerMetadata := manifest.ModuleMetadata["chunker"] - if chunkerMetadata == nil { - t.Fatalf("module metadata chunker = %#v, want object", manifest.ModuleMetadata["chunker"]) - } - wantMetadataKeys := []string{ - "prompt_id", - "prompt_version", - "prompt_sha256", - "response_schema_key", - "response_schema_id", - "response_schema_name", - "response_schema_version", - "response_schema_sha256", - } - if len(chunkerMetadata) != len(wantMetadataKeys) { - t.Fatalf("chunker metadata keys = %#v, want %d keys", chunkerMetadata, len(wantMetadataKeys)) - } - for _, key := range wantMetadataKeys { - value, ok := chunkerMetadata[key] - if !ok { - t.Fatalf("chunker metadata missing key %q: %#v", key, chunkerMetadata) - } - if _, ok := value.(string); !ok { - t.Fatalf("chunker metadata[%q] = %#v, want string", key, value) - } - } - for _, forbidden := range []string{"prompt", "schema", "source", "text", "payload", "api_key", "secret", "token"} { - if _, ok := chunkerMetadata[forbidden]; ok { - t.Fatalf("chunker metadata leaked forbidden key %q: %#v", forbidden, chunkerMetadata) - } - } - for _, forbidden := range []string{"Source document ID:", "Aria casts Cure Wounds.", "spell_casts", "Scene boundary was ambiguous."} { - if strings.Contains(string(manifestBytes), forbidden) { - t.Fatalf("manifest leaked %q: %s", forbidden, manifestBytes) - } - } - - warnings := string(readFile(t, filepath.Join(runOutputDir, "warnings.json"))) - if !strings.Contains(warnings, "scene_boundary_caveat") || !strings.Contains(warnings, "Scene boundary was ambiguous.") { - t.Fatalf("warnings output = %s, want scene boundary caveat", warnings) - } -} - -func TestExampleFixtureRunOnlySpells(t *testing.T) { - configPath := fixturePath(t, "examples/dnd-spells.config.yml") - inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json") - outputDir := t.TempDir() - diagnosticsDir := t.TempDir() - client := newFakeRunLLMClient(false) - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--only", "spells", "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(client, nil), - }) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if client.calls != 1 { - t.Fatalf("LLM calls = %d, want selected lane once", client.calls) - } - if _, err := os.Stat(filepath.Join(onlyChildDir(t, outputDir), "manifest.json")); err != nil { - t.Fatalf("expected manifest output: %v", err) - } -} - -func TestExampleFixtureFailureCoverage(t *testing.T) { - configPath := fixturePath(t, "examples/dnd-spells.config.yml") - inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json") - - tests := []struct { - name string - args []string - factory LLMClientFactory - wantCode int - wantStderr string - wantOutputStatus string - }{ - { - name: "missing config", - args: []string{"config", "validate", "--config", filepath.Join(filepath.Dir(configPath), "missing.yml"), "--pipeline", "dnd-session"}, - wantCode: 1, - wantStderr: "config file", - }, - { - name: "unknown pipeline", - args: []string{"run", "missing", "--config", configPath, "--input", inputPath}, - factory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - wantCode: 1, - wantStderr: "not configured", - }, - { - name: "invalid Seriatim input", - args: []string{"run", "dnd-session", "--config", configPath, "--input", fixturePath(t, "internal/cli/testdata/invalid-seriatim-empty-segments.json")}, - factory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - wantCode: 1, - wantStderr: "segments must not be empty", - }, - { - name: "invalid only lane", - args: []string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--only", "missing"}, - factory: fakeLLMFactory(newFakeRunLLMClient(false), nil), - wantCode: 1, - wantStderr: "selected artifact lane", - }, - { - name: "fake LLM failure", - args: []string{"run", "dnd-session", "--config", configPath, "--input", inputPath}, - factory: fakeLLMFactory(newErrorRunLLMClient(errors.New("completion unavailable")), nil), - wantCode: 1, - wantStderr: "completion unavailable", - }, - { - name: "malformed LLM response rejected by validation", - args: []string{"run", "dnd-session", "--config", configPath, "--input", inputPath}, - factory: fakeLLMFactory(newMalformedRunLLMClient(), nil), - wantCode: 0, - wantOutputStatus: "rejected", - }, - { - name: "invalid source reference rejected by validation", - args: []string{"run", "dnd-session", "--config", configPath, "--input", inputPath}, - factory: fakeLLMFactory(newFakeRunLLMClient(true), nil), - wantCode: 0, - wantOutputStatus: "rejected", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - outputDir := t.TempDir() - diagnosticsDir := t.TempDir() - args := append([]string(nil), test.args...) - if args[0] == "run" { - args = append(args, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir) - } - factory := test.factory - if factory == nil { - factory = fakeLLMFactory(newFakeRunLLMClient(false), nil) - } - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := RunWithOptions(args, &stdout, &stderr, Options{LLMClientFactory: factory}) - - if code != test.wantCode { - t.Fatalf("RunWithOptions() code = %d, want %d; stderr=%q", code, test.wantCode, stderr.String()) - } - if test.wantStderr != "" && !strings.Contains(stderr.String(), test.wantStderr) { - t.Fatalf("stderr = %q, want substring %q", stderr.String(), test.wantStderr) - } - if test.wantOutputStatus != "" { - runOutputDir := onlyChildDir(t, outputDir) - var manifest artifacts.RunManifest - readJSONFile(t, filepath.Join(runOutputDir, "manifest.json"), &manifest) - if manifest.ValidationStatus != test.wantOutputStatus { - t.Fatalf("validation status = %q, want %q", manifest.ValidationStatus, test.wantOutputStatus) - } - } - }) - } -} - -func writeTestConfig(t *testing.T, content string) string { - t.Helper() - return writeFile(t, "config.yml", content) -} - -func writeFile(t *testing.T, name string, content string) string { - t.Helper() - path := filepath.Join(t.TempDir(), name) - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatalf("write %s: %v", name, err) - } - return path -} - -func fixturePath(t *testing.T, name string) string { - t.Helper() - path := filepath.Join("..", "..", filepath.FromSlash(name)) - if _, err := os.Stat(path); err != nil { - t.Fatalf("fixture %q is not available at %q: %v", name, path, err) - } - return path -} - -func testConfigYAML(pipelineID string, laneIDs ...string) string { - return testConfigYAMLForPipelines(map[string][]string{pipelineID: laneIDs}) -} - -func testConfigYAMLForPipelines(pipelines map[string][]string) string { - var b strings.Builder - b.WriteString("version: 2\n") - b.WriteString("pipelines:\n") - for pipelineID, laneIDs := range pipelines { - b.WriteString(" " + pipelineID + ":\n") - b.WriteString(" input: fake/input\n") - b.WriteString(" artifacts:\n") - for _, laneID := range laneIDs { - b.WriteString(" " + laneID + ":\n") - b.WriteString(" extract: fake/extract\n") - } - } - return b.String() -} - -func testConfigYAMLWithReferences(pipelineID string, laneID string, references map[string]string) string { - var b strings.Builder - b.WriteString("version: 2\n") - b.WriteString("pipelines:\n") - b.WriteString(" " + pipelineID + ":\n") - b.WriteString(" input: fake/input\n") - b.WriteString(" artifacts:\n") - b.WriteString(" " + laneID + ":\n") - b.WriteString(" extract: fake/extract\n") - b.WriteString(" references:\n") - keys := make([]string, 0, len(references)) - for key := range references { - keys = append(keys, key) - } - sort.Strings(keys) - for _, key := range keys { - b.WriteString(" " + key + ": " + references[key] + "\n") - } - return b.String() -} - -func testConfigYAMLWithPipelineReferences(pipelineID string, laneID string, references map[string]string) string { - var b strings.Builder - b.WriteString("version: 2\n") - b.WriteString("pipelines:\n") - b.WriteString(" " + pipelineID + ":\n") - b.WriteString(" input: fake/input\n") - b.WriteString(" references:\n") - keys := make([]string, 0, len(references)) - for key := range references { - keys = append(keys, key) - } - sort.Strings(keys) - for _, key := range keys { - b.WriteString(" " + key + ": " + references[key] + "\n") - } - b.WriteString(" artifacts:\n") - b.WriteString(" " + laneID + ":\n") - b.WriteString(" extract: fake/extract\n") - return b.String() -} - -func testConfigYAMLWithReferencesAndDiagnostics(pipelineID string, laneID string, diagnosticsDir string, references map[string]string) string { - var b strings.Builder - b.WriteString("version: 2\n") - b.WriteString("diagnostics:\n") - b.WriteString(" work_dir: " + diagnosticsDir + "\n") - b.WriteString(" retention: always\n") - b.WriteString("pipelines:\n") - b.WriteString(" " + pipelineID + ":\n") - b.WriteString(" input: fake/input\n") - b.WriteString(" artifacts:\n") - b.WriteString(" " + laneID + ":\n") - b.WriteString(" extract: fake/extract\n") - b.WriteString(" references:\n") - keys := make([]string, 0, len(references)) - for key := range references { - keys = append(keys, key) - } - sort.Strings(keys) - for _, key := range keys { - b.WriteString(" " + key + ": " + references[key] + "\n") - } - return b.String() -} - -func mvpConfigYAML(pipelineID string, extractor string) string { - return `version: 2 -pipelines: - ` + pipelineID + `: - input: seriatim - artifacts: - spells: - extract: ` + extractor + ` -` -} - -func mvpConfigYAMLWithExtractValidators(pipelineID string, validators string) string { - return `version: 2 -pipelines: - ` + pipelineID + `: - input: seriatim - artifacts: - spells: - extract: - module: dnd/spells - validators:` + validators -} - -func mvpConfigYAMLWithChunk(pipelineID string, chunker string, extractor string) string { - return `version: 2 -pipelines: - ` + pipelineID + `: - input: seriatim - chunk: ` + chunker + ` - artifacts: - spells: - extract: ` + extractor + ` -` -} - -func mvpConfigYAMLForLanes(pipelineID string, laneIDs ...string) string { - var b strings.Builder - b.WriteString("version: 2\n") - b.WriteString("pipelines:\n") - b.WriteString(" " + pipelineID + ":\n") - b.WriteString(" input: seriatim\n") - b.WriteString(" artifacts:\n") - for _, laneID := range laneIDs { - b.WriteString(" " + laneID + ":\n") - b.WriteString(" extract: dnd/spells\n") - } - return b.String() -} - -func mvpConfigYAMLWithProfileFile(pipelineID string, profileFile string) string { - return `version: 2 -scriptorium: - profile_file: ` + profileFile + ` -pipelines: - ` + pipelineID + `: - input: seriatim - artifacts: - spells: - extract: dnd/spells -` -} - -func writeScriptoriumProfileFile(t *testing.T, id string, endpoint string, model string) string { - t.Helper() - return writeFile(t, id+".profile.yml", `id: `+id+` -endpoint: `+endpoint+` -model: `+model+` -`) -} - -func mvpConfigYAMLWithDiagnostics(pipelineID, diagnosticsDir, retention string) string { - return `version: 2 -diagnostics: - work_dir: ` + diagnosticsDir + ` - retention: ` + retention + ` -pipelines: - ` + pipelineID + `: - input: seriatim - artifacts: - spells: - extract: dnd/spells -` -} - -func mvpConfigYAMLWithWorkspaceDiagnostics(pipelineID, workspaceDir, retention string) string { - return `version: 2 -workspace: - directory: ` + workspaceDir + ` - diagnostics: - enabled: true - retention: ` + retention + ` -pipelines: - ` + pipelineID + `: - input: seriatim - artifacts: - spells: - extract: dnd/spells -` -} - -func mvpConfigYAMLWithWorkspaceDiagnosticsAndStateEnabled(pipelineID, workspaceDir, retention string) string { - return `version: 2 -workspace: - directory: ` + workspaceDir + ` - diagnostics: - enabled: true - retention: ` + retention + ` - resume: - enabled: true - debug: - enabled: true -pipelines: - ` + pipelineID + `: - input: seriatim - artifacts: - spells: - extract: dnd/spells -` -} - -func mvpConfigYAMLWithWorkspaceResumeEnabled(pipelineID, workspaceDir, retention string) string { - return `version: 2 -workspace: - directory: ` + workspaceDir + ` - diagnostics: - enabled: true - retention: ` + retention + ` - resume: - enabled: true -pipelines: - ` + pipelineID + `: - input: seriatim - artifacts: - spells: - extract: dnd/spells -` -} - -func mvpConfigYAMLWithWorkspaceDebugEnabled(pipelineID, workspaceDir, retention string) string { - return `version: 2 -workspace: - directory: ` + workspaceDir + ` - diagnostics: - enabled: true - retention: ` + retention + ` - debug: - enabled: true -pipelines: - ` + pipelineID + `: - input: seriatim - artifacts: - spells: - extract: dnd/spells -` -} - -func mvpConfigYAMLWithWorkspaceResumeAndChunker(pipelineID, workspaceDir, retention, chunker string) string { - return `version: 2 -workspace: - directory: ` + workspaceDir + ` - diagnostics: - enabled: true - retention: ` + retention + ` - resume: - enabled: true -pipelines: - ` + pipelineID + `: - input: seriatim - chunk: ` + chunker + ` - artifacts: - spells: - extract: dnd/spells -` -} - -func mvpConfigYAMLWithWorkspaceResumeAndChunkOptions(pipelineID, workspaceDir, retention string) string { - return `version: 2 -workspace: - directory: ` + workspaceDir + ` - diagnostics: - enabled: true - retention: ` + retention + ` - resume: - enabled: true -pipelines: - ` + pipelineID + `: - input: seriatim - chunk: - module: generic - options: - max_units: 10 - artifacts: - spells: - extract: dnd/spells -` -} - -func mvpConfigYAMLWithWorkspaceResumeAndProfileFile(pipelineID, workspaceDir, retention, profileFile string) string { - return `version: 2 -scriptorium: - profile_file: ` + profileFile + ` -workspace: - directory: ` + workspaceDir + ` - diagnostics: - enabled: true - retention: ` + retention + ` - resume: - enabled: true -pipelines: - ` + pipelineID + `: - input: seriatim - artifacts: - spells: - extract: dnd/spells -` -} - -func mvpConfigYAMLWithWorkspaceResumeLanes(pipelineID, workspaceDir, retention string, laneIDs ...string) string { - var b strings.Builder - b.WriteString("version: 2\n") - b.WriteString("workspace:\n") - b.WriteString(" directory: " + workspaceDir + "\n") - b.WriteString(" diagnostics:\n") - b.WriteString(" enabled: true\n") - b.WriteString(" retention: " + retention + "\n") - b.WriteString(" resume:\n") - b.WriteString(" enabled: true\n") - b.WriteString("pipelines:\n") - b.WriteString(" " + pipelineID + ":\n") - b.WriteString(" input: seriatim\n") - b.WriteString(" artifacts:\n") - for _, laneID := range laneIDs { - b.WriteString(" " + laneID + ":\n") - b.WriteString(" extract: dnd/spells\n") - } - return b.String() -} - -func mvpConfigYAMLWithWorkspaceDiagnosticsDisabled(pipelineID, workspaceDir string) string { - return `version: 2 -workspace: - directory: ` + workspaceDir + ` - diagnostics: - enabled: false -pipelines: - ` + pipelineID + `: - input: seriatim - artifacts: - spells: - extract: dnd/spells -` -} - -func writeSeriatimInput(t *testing.T) string { - t.Helper() - return writeFile(t, "source.json", `{ - "metadata": { - "id": "session-alpha" - }, - "segments": [ - { - "id": 1, - "start": 0, - "end": 1, - "speaker": "Aria", - "text": "Aria casts Cure Wounds." - } - ] -}`) -} - -type fakeRunLLMClient struct { - invalidSourceRef bool - calls int - err error - payload map[string]any - sceneCaveat string -} - -func newFakeRunLLMClient(invalidSourceRef bool) *fakeRunLLMClient { - return &fakeRunLLMClient{invalidSourceRef: invalidSourceRef} -} - -func newErrorRunLLMClient(err error) *fakeRunLLMClient { - return &fakeRunLLMClient{err: err} -} - -func newMalformedRunLLMClient() *fakeRunLLMClient { - return &fakeRunLLMClient{payload: map[string]any{}} -} - -func newSceneRunLLMClient(caveat string) *fakeRunLLMClient { - return &fakeRunLLMClient{sceneCaveat: caveat} -} - -func (client *fakeRunLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { - client.calls++ - if client.err != nil { - return contracts.StructuredCompletionResponse{}, client.err - } - if req.StageName == scenes.Key && client.payload == nil { - payload := map[string]any{ - "scenes": []map[string]any{ - { - "start_unit_id": 1, - "end_unit_id": 2, - "short_title": "Opening spell", - "primary_mode": "Narrative", - "main_participants": []string{"Aria"}, - "summary": "Aria casts a spell.", - "boundary_note": "The provided source units form one scene.", - "boundary_confidence": "High", - }, - }, - "boundary_caveats": []string{}, - } - if client.sceneCaveat != "" { - payload["boundary_caveats"] = []string{client.sceneCaveat} - } - encoded, err := json.Marshal(payload) - if err != nil { - return contracts.StructuredCompletionResponse{}, err - } - if err := json.Unmarshal(encoded, out); err != nil { - return contracts.StructuredCompletionResponse{}, err - } - return fakeRunStructuredResponse(req, encoded), nil - } - startUnitID := 1 - if client.invalidSourceRef { - startUnitID = 999 - } - payload := client.payload - if payload == nil { - payload = map[string]any{ - "spell_casts": []map[string]any{ - { - "caster": "Aria", - "spell": "Cure Wounds", - "effect": "Heals a wounded ally.", - "narrative_description": "Aria casts Cure Wounds.", - "source_refs": []map[string]any{ - { - "source_id": "session-alpha", - "start_unit_id": startUnitID, - "end_unit_id": 1, - }, - }, - }, - }, - } - } - encoded, err := json.Marshal(payload) - if err != nil { - return contracts.StructuredCompletionResponse{}, err - } - if err := json.Unmarshal(encoded, out); err != nil { - return contracts.StructuredCompletionResponse{}, err - } - return fakeRunStructuredResponse(req, encoded), nil -} - -func fakeRunStructuredResponse(req contracts.StructuredCompletionRequest, content []byte) contracts.StructuredCompletionResponse { - profileID := req.ProfileID - if profileID == "" { - profileID = "fake-profile" - } - return contracts.StructuredCompletionResponse{ - Content: content, - Model: "fake-model", - ProfileID: profileID, - Debug: &contracts.LLMDebugMaterial{ - Prompt: &contracts.LLMDebugPrompt{ - PromptID: req.PromptID, - PromptVersion: req.PromptVersion, - SelectedProfileID: profileID, - SessionID: req.SessionID, - Messages: []contracts.LLMDebugMessage{ - {Role: "user", Content: "fake rendered prompt for " + req.PromptID}, - }, - }, - Response: &contracts.LLMDebugResponse{ - Content: string(content), - PromptID: req.PromptID, - PromptVersion: req.PromptVersion, - SelectedProfileID: profileID, - ModelName: "fake-model", - }, - }, - } -} - -func fakeLLMFactory(client contracts.StructuredLLMClient, err error) LLMClientFactory { - return func(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { - if err != nil { - return nil, nil, err - } - return client, []artifacts.LLMProfileManifest{{ID: strings.TrimSpace(profileID)}}, nil - } -} - -type recordingLLMFactory struct { - client contracts.StructuredLLMClient - profileIDs []string -} - -func (factory *recordingLLMFactory) build(ctx context.Context, cfg config.Config, profileID string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) { - factory.profileIDs = append(factory.profileIDs, strings.TrimSpace(profileID)) - return factory.client, []artifacts.LLMProfileManifest{{ID: strings.TrimSpace(profileID)}}, nil -} - -type unsafeOutputEncoder struct{} - -func (unsafeOutputEncoder) Key() string { - return "json" -} - -func (unsafeOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) { - return contracts.OutputResult{ - Files: []contracts.OutputFile{ - { - Name: "../escape.json", - ContentType: "application/json", - Bytes: []byte("{}\n"), - }, - }, - }, nil -} - -type warningOutputEncoder struct{} - -func (warningOutputEncoder) Key() string { - return "json" -} - -func (warningOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) { - result, err := jsonoutput.New().Encode(ctx, req) - if err != nil { - return contracts.OutputResult{}, err - } - result.Warnings = append(result.Warnings, contracts.Warning{ - Scope: "output", - ReasonCode: "synthetic_warning", - Message: "synthetic output warning", - }) - return result, nil -} - -func registriesWithOutput(t *testing.T, encoder contracts.OutputEncoder) pipeline.Registries { - t.Helper() - registries, err := productionRegistries() - if err != nil { - t.Fatalf("productionRegistries: %v", err) - } - outputs := pipeline.NewOutputEncoderRegistry() - if err := outputs.RegisterWithSpec(jsonoutput.ModuleSpec(), func() (contracts.OutputEncoder, error) { - return encoder, nil - }); err != nil { - t.Fatalf("register test output encoder: %v", err) - } - registries.Outputs = outputs - return registries -} - -func fakeExecutionRegistries(t *testing.T) pipeline.Registries { - t.Helper() - inputs := pipeline.NewInputAdapterRegistry() - chunkers := pipeline.NewChunkerRegistry() - extractors := pipeline.NewExtractorRegistry() - mergers := pipeline.NewMergerRegistry() - normalizers := pipeline.NewNormalizerRegistry() - outputs := pipeline.NewOutputEncoderRegistry() - codecs := pipeline.NewArtifactCodecRegistry() - if err := pipeline.RegisterArtifactCodec(codecs, fakeRunCodec{}); err != nil { - t.Fatal(err) - } - - if err := inputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "fake/input", Stage: pipeline.StageInput, Provides: []string{"source"}}, func() (contracts.InputAdapter, error) { - return fakeRunInputAdapter{}, nil - }); err != nil { - t.Fatalf("register fake input: %v", err) - } - if err := chunkers.RegisterWithSpec(pipeline.ModuleSpec{Key: "generic", Stage: pipeline.StageChunk, Requires: []string{"source"}, Provides: []string{"chunks"}}, func() (contracts.Chunker, error) { - return fakeRunChunker{}, nil - }); err != nil { - t.Fatalf("register fake chunker: %v", err) - } - if err := pipeline.RegisterExtractor(extractors, pipeline.ModuleSpec{ - Key: "fake/extract", - Stage: pipeline.StageExtract, - Requires: []string{"chunks"}, - Provides: []string{"artifact"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "roster"}, - }, ArtifactKind: fakeRunArtifactKind, - }, func() (contracts.Extractor[fakeRunArtifact], error) { - return fakeRunExtractor{}, nil - }); err != nil { - t.Fatalf("register fake extractor: %v", err) - } - if err := pipeline.RegisterMerger(mergers, pipeline.ModuleSpec{Key: "appendorder", Stage: pipeline.StageMerge, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: fakeRunArtifactKind}, func() (contracts.Merger[fakeRunArtifact], error) { - return fakeRunMerger{}, nil - }); err != nil { - t.Fatalf("register fake merger: %v", err) - } - if err := pipeline.RegisterNormalizer(normalizers, pipeline.ModuleSpec{Key: "noop", Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: fakeRunArtifactKind}, func() (contracts.Normalizer[fakeRunArtifact], error) { - return fakeRunNormalizer{}, nil - }); err != nil { - t.Fatalf("register fake normalizer: %v", err) - } - if err := jsonoutput.Register(outputs); err != nil { - t.Fatalf("register json output: %v", err) - } - - return pipeline.Registries{ - Inputs: inputs, - Chunkers: chunkers, - ArtifactCodecs: codecs, - Extractors: extractors, - Mergers: mergers, - Normalizers: normalizers, - Outputs: outputs, - } -} - -type fakeRunInputAdapter struct{} - -func (fakeRunInputAdapter) Key() string { - return "fake/input" -} - -func (fakeRunInputAdapter) Parse(ctx context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) { - return &source.SourceDocument{ - ID: "source", - Kind: "text", - Format: "test", - Digest: "sha256:source", - Units: []source.SourceUnit{ - {ID: 1, Kind: "text", Text: string(req.Raw), Ref: source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}, - }, - }, nil -} - -type fakeRunChunker struct{} - -func (fakeRunChunker) Key() string { - return "generic" -} - -func (fakeRunChunker) ReferenceSlots() []contracts.ReferenceSlot { - return nil -} - -func (fakeRunChunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) { - return contracts.ChunkPlanResult{ - Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}, - }, nil -} - -type fakeRunExtractor struct{} - -const fakeRunArtifactKind contracts.ArtifactKind = "test/fake" - -type fakeRunArtifact struct { - Value bool `json:"value"` -} - -type referenceVariantExtractor struct{ key string } - -func (e referenceVariantExtractor) Key() string { return e.key } -func (referenceVariantExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil } -func (referenceVariantExtractor) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[fakeRunArtifact], error) { - return contracts.TypedExtractionResult[fakeRunArtifact]{}, nil -} - -type referenceVariantMerger struct{ key string } - -func (m referenceVariantMerger) Key() string { return m.key } -func (referenceVariantMerger) Merge(context.Context, contracts.TypedMergeRequest[fakeRunArtifact]) (contracts.TypedMergeResult[fakeRunArtifact], error) { - return contracts.TypedMergeResult[fakeRunArtifact]{}, nil -} - -type referenceVariantNormalizer struct{ key string } - -func (n referenceVariantNormalizer) Key() string { return n.key } -func (referenceVariantNormalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil } -func (referenceVariantNormalizer) Normalize(context.Context, contracts.TypedNormalizeRequest[fakeRunArtifact]) (contracts.TypedNormalizeResult[fakeRunArtifact], error) { - return contracts.TypedNormalizeResult[fakeRunArtifact]{}, nil -} - -type fakeRunCodec struct{} - -func (fakeRunCodec) Kind() contracts.ArtifactKind { return fakeRunArtifactKind } -func (fakeRunCodec) Schema() contracts.ArtifactSchema { - return contracts.ArtifactSchema{ID: "fake.artifact", Name: "fake_artifact", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)} -} -func (fakeRunCodec) MediaType() string { return "application/json" } -func (fakeRunCodec) EncodeCandidate(v fakeRunArtifact) ([]byte, error) { - return json.Marshal(v) -} -func (fakeRunCodec) Encode(v fakeRunArtifact) ([]byte, error) { return json.Marshal(v) } -func (fakeRunCodec) Decode(b []byte) (fakeRunArtifact, error) { - var v fakeRunArtifact - err := json.Unmarshal(b, &v) - return v, err -} - -func (fakeRunExtractor) Key() string { - return "fake/extract" -} - -func (fakeRunExtractor) ReferenceSlots() []contracts.ReferenceSlot { - return []contracts.ReferenceSlot{{Name: "roster"}} -} - -func (fakeRunExtractor) Extract(ctx context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[fakeRunArtifact], error) { - return contracts.TypedExtractionResult[fakeRunArtifact]{Value: fakeRunArtifact{Value: true}}, nil -} - -type fakeRunMerger struct{} - -func (fakeRunMerger) Key() string { - return "appendorder" -} - -func (fakeRunMerger) Merge(ctx context.Context, req contracts.TypedMergeRequest[fakeRunArtifact]) (contracts.TypedMergeResult[fakeRunArtifact], error) { - if len(req.ExtractOutputs) > 0 { - return contracts.TypedMergeResult[fakeRunArtifact]{Value: req.ExtractOutputs[0].Value}, nil - } - return contracts.TypedMergeResult[fakeRunArtifact]{}, nil -} - -type fakeRunNormalizer struct{} - -func (fakeRunNormalizer) Key() string { - return "noop" -} - -func (fakeRunNormalizer) ReferenceSlots() []contracts.ReferenceSlot { - return nil -} - -func (fakeRunNormalizer) Normalize(ctx context.Context, req contracts.TypedNormalizeRequest[fakeRunArtifact]) (contracts.TypedNormalizeResult[fakeRunArtifact], error) { - return contracts.TypedNormalizeResult[fakeRunArtifact]{Value: req.MergeOutput.Value}, nil -} - -func onlyChildDir(t *testing.T, root string) string { - t.Helper() - children := childDirs(t, root) - if len(children) != 1 { - t.Fatalf("child dirs under %q = %v, want one", root, children) - } - return children[0] -} - -func onlyCheckpointIdentityDir(t *testing.T, workspaceDir string) string { - t.Helper() - pipelineDir := onlyChildDir(t, filepath.Join(workspaceDir, "checkpoints")) - inputDir := onlyChildDir(t, pipelineDir) - pipelineDigestDir := onlyChildDir(t, inputDir) - return onlyChildDir(t, pipelineDigestDir) -} - -func anyDiagnosticsFileContains(t *testing.T, runDirs []string, name string, want string) bool { - t.Helper() - for _, runDir := range runDirs { - data, err := os.ReadFile(filepath.Join(runDir, name)) - if err != nil { - if os.IsNotExist(err) { - continue - } - t.Fatalf("read diagnostics artifact %q under %q: %v", name, runDir, err) - } - if strings.Contains(string(data), want) { - return true - } - } - return false -} - -func assertDistinctRoots(t *testing.T, roots ...string) { - t.Helper() - for i, first := range roots { - for _, second := range roots[i+1:] { - firstAbs, err := filepath.Abs(first) - if err != nil { - t.Fatalf("resolve %q: %v", first, err) - } - secondAbs, err := filepath.Abs(second) - if err != nil { - t.Fatalf("resolve %q: %v", second, err) - } - if firstAbs == secondAbs { - t.Fatalf("workspace roots overlap exactly: %q", firstAbs) - } - firstRel, err := filepath.Rel(firstAbs, secondAbs) - if err != nil { - t.Fatalf("rel %q %q: %v", firstAbs, secondAbs, err) - } - secondRel, err := filepath.Rel(secondAbs, firstAbs) - if err != nil { - t.Fatalf("rel %q %q: %v", secondAbs, firstAbs, err) - } - if !strings.HasPrefix(firstRel, ".."+string(filepath.Separator)) && firstRel != ".." { - t.Fatalf("workspace root %q contains %q", firstAbs, secondAbs) - } - if !strings.HasPrefix(secondRel, ".."+string(filepath.Separator)) && secondRel != ".." { - t.Fatalf("workspace root %q contains %q", secondAbs, firstAbs) - } - } - } -} - -var debugBase64FieldPattern = regexp.MustCompile(`"content_base64"\s*:\s*"([^"]*)"`) -var debugRawLLMResponseContentPattern = regexp.MustCompile(`"content"\s*:\s*"(?:\\.|[^"\\])*"`) - -func assertDebugTreeDoesNotContain(t *testing.T, root string, forbidden ...string) { - t.Helper() - if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { - if err != nil { - return err - } - if entry.IsDir() { - return nil - } - if strings.HasPrefix(filepath.Base(path), "response-content-") { - return nil - } - data, err := os.ReadFile(path) - if err != nil { - return err - } - text := debugRawLLMResponseContentPattern.ReplaceAllString(string(data), `"content":"[RAW_LLM_RESPONSE]"`) - for _, value := range forbidden { - if strings.Contains(text, value) { - t.Fatalf("debug artifact %q contains forbidden value %q", path, value) - } - } - for _, match := range debugBase64FieldPattern.FindAllStringSubmatch(text, -1) { - decoded, err := base64.StdEncoding.DecodeString(match[1]) - if err != nil { - continue - } - decodedText := string(decoded) - for _, value := range forbidden { - if strings.Contains(decodedText, value) { - t.Fatalf("debug artifact %q decoded content contains forbidden value %q", path, value) - } - } - } - return nil - }); err != nil { - t.Fatalf("walk debug tree %q: %v", root, err) - } -} - -func seedWorkspaceCheckpoint(t *testing.T, configPath string, inputPath string, extraArgs []string) { - t.Helper() - client := newFakeRunLLMClient(false) - args := []string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", t.TempDir()} - args = append(args, extraArgs...) - var stdout bytes.Buffer - var stderr bytes.Buffer - code := RunWithOptions(args, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(client, nil), - }) - if code != 0 { - t.Fatalf("seed RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if client.calls == 0 { - t.Fatal("seed LLM calls = 0, want checkpoint seed run to execute") - } -} - -func runResumeWithClient(t *testing.T, configPath string, inputPath string, extraArgs []string) *fakeRunLLMClient { - t.Helper() - client := newFakeRunLLMClient(false) - args := []string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", t.TempDir(), "--resume"} - args = append(args, extraArgs...) - var stdout bytes.Buffer - var stderr bytes.Buffer - code := RunWithOptions(args, &stdout, &stderr, Options{ - LLMClientFactory: fakeLLMFactory(client, nil), - }) - if code != 0 { - t.Fatalf("resume RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - return client -} - -func childDirs(t *testing.T, root string) []string { - t.Helper() - entries, err := os.ReadDir(root) - if err != nil { - if os.IsNotExist(err) { - return nil - } - t.Fatalf("read dir %q: %v", root, err) - } - var dirs []string - for _, entry := range entries { - if entry.IsDir() { - dirs = append(dirs, filepath.Join(root, entry.Name())) - } - } - return dirs -} - -func assertPathNotExist(t *testing.T, path string) { - t.Helper() - if _, err := os.Stat(path); !os.IsNotExist(err) { - t.Fatalf("path %q stat err = %v, want not exist", path, err) - } -} - -func readFile(t *testing.T, path string) []byte { - t.Helper() - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read %q: %v", path, err) - } - return data -} - -func readJSONFile(t *testing.T, path string, out any) { - t.Helper() - if err := json.Unmarshal(readFile(t, path), out); err != nil { - t.Fatalf("unmarshal %q: %v", path, err) - } -} - -func readResolvedPipeline(t *testing.T, diagnosticsDir string) pipeline.ResolvedPipeline { - t.Helper() - runDir := onlyChildDir(t, diagnosticsDir) - var resolved pipeline.ResolvedPipeline - readJSONFile(t, filepath.Join(runDir, diagnostics.ArtifactResolvedPipeline), &resolved) - return resolved -} - -func resolvedArtifactLane(t *testing.T, resolved pipeline.ResolvedPipeline, laneID string) pipeline.ResolvedArtifactLane { - t.Helper() - for _, lane := range resolved.ArtifactLanes { - if lane.ID == laneID { - return lane - } - } - t.Fatalf("lane %q not found in resolved pipeline", laneID) - return pipeline.ResolvedArtifactLane{} -} - -func manifestValidatorChain(t *testing.T, manifest artifacts.RunManifest, stage pipeline.ModuleStage, laneID string, module string) artifacts.ValidatorChainManifest { - t.Helper() - for _, chain := range manifest.ValidatorChains { - if chain.Stage == string(stage) && chain.LaneID == laneID && chain.ModuleKey == module { - return chain - } - } - t.Fatalf("validator chain %s/%s/%s not found in %#v", stage, laneID, module, manifest.ValidatorChains) - return artifacts.ValidatorChainManifest{} -} - -func manifestValidatorKeys(chain artifacts.ValidatorChainManifest) []string { - keys := make([]string, 0, len(chain.Validators)) - for _, validator := range chain.Validators { - keys = append(keys, validator.Key) - } - return keys -} - -func assertNoTemporaryFiles(t *testing.T, root string) { - t.Helper() - if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { - if err != nil { - return err - } - if strings.Contains(entry.Name(), ".tmp-") { - t.Fatalf("temporary file remains after success: %s", path) - } - return nil - }); err != nil { - t.Fatalf("walk output dir %q: %v", root, err) - } -} - -func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.ModuleCatalog { - t.Helper() - inputs := pipeline.NewInputAdapterRegistry() - chunkers := pipeline.NewChunkerRegistry() - extractors := pipeline.NewExtractorRegistry() - mergers := pipeline.NewMergerRegistry() - normalizers := pipeline.NewNormalizerRegistry() - validators := pipeline.NewValidatorRegistry() - outputs := pipeline.NewOutputEncoderRegistry() - - specs := map[string]pipeline.ModuleSpec{ - "fake/input": {Key: "fake/input", Stage: pipeline.StageInput, Provides: []string{"source"}}, - "generic": {Key: "generic", Stage: pipeline.StageChunk, Requires: []string{"source"}, Provides: []string{"chunks"}}, - "fake/extract": {Key: "fake/extract", Stage: pipeline.StageExtract, Requires: []string{"chunks"}, Provides: []string{"artifact"}}, - "appendorder": {Key: "appendorder", Stage: pipeline.StageMerge, Requires: []string{"artifact"}, Provides: []string{"merged"}}, - "noop": {Key: "noop", Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}}, - "json": {Key: "json", Stage: pipeline.StageOutput, Requires: []string{"normalized"}}, - } - for _, override := range overrides { - specs[override.Key] = override - } - for _, key := range []string{"fake/extract", "appendorder", "noop"} { - spec := specs[key] - spec.ArtifactKind = fakeRunArtifactKind - specs[key] = spec - } - - mustRegisterInput(t, inputs, specs["fake/input"]) - mustRegisterChunker(t, chunkers, specs["generic"]) - codecs := pipeline.NewArtifactCodecRegistry() - if err := pipeline.RegisterArtifactCodec(codecs, fakeRunCodec{}); err != nil { - t.Fatal(err) - } - mustRegisterExtractor(t, extractors, specs["fake/extract"]) - mustRegisterMerger(t, mergers, specs["appendorder"]) - mustRegisterNormalizer(t, normalizers, specs["noop"]) - mustRegisterOutput(t, outputs, specs["json"]) - - return pipeline.ModuleCatalog{ - Inputs: inputs, - Chunkers: chunkers, - ArtifactCodecs: codecs, - Extractors: extractors, - Mergers: mergers, - Normalizers: normalizers, - Validators: validators, - ValidatorChains: pipeline.NewValidatorChainRegistry(), - Outputs: outputs, - } -} - -func referenceVariantCatalog(t *testing.T) pipeline.ModuleCatalog { - t.Helper() - chunkers := pipeline.NewChunkerRegistry() - extractors := pipeline.NewExtractorRegistry() - mergers := pipeline.NewMergerRegistry() - normalizers := pipeline.NewNormalizerRegistry() - mustRegisterChunker(t, chunkers, pipeline.ModuleSpec{Key: "generic", Stage: pipeline.StageChunk}) - - for _, item := range []struct { - key string - kind contracts.ArtifactKind - }{ - {key: "extract/alpha", kind: "test/alpha"}, - {key: "extract/beta", kind: "test/beta"}, - } { - item := item - if err := pipeline.RegisterExtractor(extractors, pipeline.ModuleSpec{Key: item.key, Stage: pipeline.StageExtract, ArtifactKind: item.kind}, func() (contracts.Extractor[fakeRunArtifact], error) { - return referenceVariantExtractor{key: item.key}, nil - }); err != nil { - t.Fatalf("register extractor %q: %v", item.key, err) - } - } - for _, item := range []struct { - kind contracts.ArtifactKind - mergeSlot string - normalizeSlot string - }{ - {kind: "test/beta", mergeSlot: "beta_merge", normalizeSlot: "beta_normalize"}, - {kind: "test/alpha", mergeSlot: "alpha_merge", normalizeSlot: "alpha_normalize"}, - } { - mergeSpec := pipeline.ModuleSpec{Key: "shared/merge", Stage: pipeline.StageMerge, ArtifactKind: item.kind, ReferenceSlots: []contracts.ReferenceSlot{{Name: item.mergeSlot}}} - if err := pipeline.RegisterMerger(mergers, mergeSpec, func() (contracts.Merger[fakeRunArtifact], error) { - return referenceVariantMerger{key: "shared/merge"}, nil - }); err != nil { - t.Fatalf("register merger %q: %v", item.kind, err) - } - normalizeSpec := pipeline.ModuleSpec{Key: "shared/normalize", Stage: pipeline.StageNormalize, ArtifactKind: item.kind, ReferenceSlots: []contracts.ReferenceSlot{{Name: item.normalizeSlot}}} - if err := pipeline.RegisterNormalizer(normalizers, normalizeSpec, func() (contracts.Normalizer[fakeRunArtifact], error) { - return referenceVariantNormalizer{key: "shared/normalize"}, nil - }); err != nil { - t.Fatalf("register normalizer %q: %v", item.kind, err) - } - } - return pipeline.ModuleCatalog{Chunkers: chunkers, Extractors: extractors, Mergers: mergers, Normalizers: normalizers} -} - -func mustRegisterInput(t *testing.T, registry *pipeline.InputAdapterRegistry, spec pipeline.ModuleSpec) { - t.Helper() - if err := registry.RegisterWithSpec(spec, func() (contracts.InputAdapter, error) { return fakeRunInputAdapter{}, nil }); err != nil { - t.Fatalf("register input: %v", err) - } -} - -func mustRegisterChunker(t *testing.T, registry *pipeline.ChunkerRegistry, spec pipeline.ModuleSpec) { - t.Helper() - if err := registry.RegisterWithSpec(spec, func() (contracts.Chunker, error) { return fakeRunChunker{}, nil }); err != nil { - t.Fatalf("register chunker: %v", err) - } -} - -func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) { - t.Helper() - if err := pipeline.RegisterExtractor(registry, spec, func() (contracts.Extractor[fakeRunArtifact], error) { return fakeRunExtractor{}, nil }); err != nil { - t.Fatalf("register extractor: %v", err) - } -} - -func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) { - t.Helper() - if err := pipeline.RegisterMerger(registry, spec, func() (contracts.Merger[fakeRunArtifact], error) { return fakeRunMerger{}, nil }); err != nil { - t.Fatalf("register merger: %v", err) - } -} - -func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) { - t.Helper() - if err := pipeline.RegisterNormalizer(registry, spec, func() (contracts.Normalizer[fakeRunArtifact], error) { return fakeRunNormalizer{}, nil }); err != nil { - t.Fatalf("register normalizer: %v", err) - } -} - -func mustRegisterOutput(t *testing.T, registry *pipeline.OutputEncoderRegistry, spec pipeline.ModuleSpec) { - t.Helper() - if err := registry.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { return jsonoutput.New(), nil }); err != nil { - t.Fatalf("register output: %v", err) - } -} - -func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ValidatorSpec) { - t.Helper() - if err := pipeline.RegisterChunkValidator(registry, spec, func() (contracts.ChunkValidator, error) { - return fakeConfigValidator{name: spec.Key, executionClass: spec.ExecutionClass}, nil - }); err != nil { - t.Fatalf("register validator: %v", err) - } - if err := pipeline.RegisterTypedValidator[fakeRunArtifact](registry, fakeRunArtifactKind, spec, func() (contracts.TypedValidator[fakeRunArtifact], error) { - return fakeConfigTypedValidator{fakeConfigValidator{name: spec.Key, executionClass: spec.ExecutionClass}}, nil - }); err != nil { - t.Fatalf("register typed validator: %v", err) - } -} - -type fakeConfigValidator struct { - name string - executionClass contracts.ExecutionClass -} - -func (validator fakeConfigValidator) Name() string { - return validator.name -} - -func (validator fakeConfigValidator) ExecutionClass() contracts.ExecutionClass { - return validator.executionClass -} - -func (validator fakeConfigValidator) Validate(ctx context.Context, req contracts.ChunkValidationRequest) (contracts.ValidationResult, error) { - return contracts.ValidationResult{Approved: true}, nil -} - -type fakeConfigTypedValidator struct{ fakeConfigValidator } - -func (validator fakeConfigTypedValidator) Validate(ctx context.Context, req contracts.TypedValidationRequest[fakeRunArtifact]) (contracts.ValidationResult, error) { - return contracts.ValidationResult{Approved: true}, nil -} - -func mapLookup(values map[string]string) func(string) (string, bool) { - return func(key string) (string, bool) { - value, ok := values[key] - return value, ok - } -} diff --git a/internal/cli/test_main_test.go b/internal/cli/test_main_test.go deleted file mode 100644 index 02a4ad9..0000000 --- a/internal/cli/test_main_test.go +++ /dev/null @@ -1,10 +0,0 @@ -package cli - -import ( - "os" - "testing" -) - -func TestMain(m *testing.M) { - os.Exit(m.Run()) -} diff --git a/internal/core/config/chunk_cache_test.go b/internal/core/config/chunk_cache_test.go deleted file mode 100644 index 85f3a3a..0000000 --- a/internal/core/config/chunk_cache_test.go +++ /dev/null @@ -1,128 +0,0 @@ -//go:build legacy - -package config - -import ( - "path/filepath" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" -) - -func TestChunkCacheDefaults(t *testing.T) { - cfg := Default() - if cfg.Workspace.ChunkCache.Mode != pipeline.ChunkCacheAuto || cfg.Workspace.ChunkCache.Directory != "" { - t.Fatalf("chunk cache defaults = %#v", cfg.Workspace.ChunkCache) - } - if err := cfg.Validate(); err != nil { - t.Fatalf("Validate() error = %v", err) - } -} - -func TestChunkCacheFileConfiguration(t *testing.T) { - cfg := parseAndApplyConfig(t, ` -version: 2 -workspace: - chunk_cache: - mode: refresh - directory: " ./state/../plans " -`) - if cfg.Workspace.ChunkCache.Mode != pipeline.ChunkCacheRefresh { - t.Fatalf("mode = %q", cfg.Workspace.ChunkCache.Mode) - } - if got, want := cfg.Workspace.ChunkCache.Directory, filepath.Clean("./state/../plans"); got != want { - t.Fatalf("directory = %q, want %q", got, want) - } - if err := cfg.Validate(); err != nil { - t.Fatalf("Validate() error = %v", err) - } -} - -func TestChunkCacheEnvironmentOverridesFile(t *testing.T) { - fileCfg, err := ParseFileConfigYAML([]byte(` -version: 2 -workspace: - chunk_cache: - mode: bypass - directory: /file/plans -`)) - if err != nil { - t.Fatal(err) - } - cfg := Default() - if err := cfg.ApplyFileConfigWithLookup(fileCfg, emptyLookup); err != nil { - t.Fatal(err) - } - if err := cfg.ApplyEnvOverridesWithLookup(mapLookup(map[string]string{ - "NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE": "refresh", - "NOTARIUS_WORKSPACE_CHUNK_CACHE_DIR": " /environment/../cache/plans ", - })); err != nil { - t.Fatal(err) - } - if cfg.Workspace.ChunkCache.Mode != pipeline.ChunkCacheRefresh || cfg.Workspace.ChunkCache.Directory != filepath.Clean("/environment/../cache/plans") { - t.Fatalf("effective chunk cache = %#v", cfg.Workspace.ChunkCache) - } -} - -func TestChunkCacheEmptyDirectoryEnvironmentSelectsDefault(t *testing.T) { - cfg := Default() - cfg.Workspace.ChunkCache.Directory = "/file/plans" - if err := cfg.ApplyEnvOverridesWithLookup(mapLookup(map[string]string{"NOTARIUS_WORKSPACE_CHUNK_CACHE_DIR": " \t "})); err != nil { - t.Fatal(err) - } - if cfg.Workspace.ChunkCache.Directory != "" { - t.Fatalf("directory = %q, want unset", cfg.Workspace.ChunkCache.Directory) - } -} - -func TestChunkCacheRejectsInvalidSuppliedModes(t *testing.T) { - fileCfg, err := ParseFileConfigYAML([]byte("version: 2\nworkspace:\n chunk_cache:\n mode: sometimes\n")) - if err != nil { - t.Fatal(err) - } - cfg := Default() - if err := cfg.ApplyFileConfigWithLookup(fileCfg, emptyLookup); err == nil || !strings.Contains(err.Error(), "workspace.chunk_cache.mode") { - t.Fatalf("file mode error = %v", err) - } - - cfg = Default() - if err := cfg.ApplyEnvOverridesWithLookup(mapLookup(map[string]string{"NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE": "sometimes"})); err == nil || !strings.Contains(err.Error(), "NOTARIUS_WORKSPACE_CHUNK_CACHE_MODE") { - t.Fatalf("environment mode error = %v", err) - } - - cfg = Default() - cfg.Workspace.ChunkCache.Mode = "sometimes" - if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "chunk cache") { - t.Fatalf("Validate() error = %v", err) - } -} - -func TestChunkCacheConfigurationClonesAndRedacts(t *testing.T) { - cfg := Default() - cfg.Workspace.ChunkCache = WorkspaceChunkCacheConfig{Mode: pipeline.ChunkCacheRefresh, Directory: "/var/cache/notarius/chunk-plans"} - cloned := cloneConfig(cfg) - redacted := cfg.Redacted() - if cloned.Workspace.ChunkCache != cfg.Workspace.ChunkCache || redacted.Workspace.ChunkCache != cfg.Workspace.ChunkCache { - t.Fatalf("cloned=%#v redacted=%#v", cloned.Workspace.ChunkCache, redacted.Workspace.ChunkCache) - } - redacted.Workspace.ChunkCache.Directory = "/changed" - if cfg.Workspace.ChunkCache.Directory != "/var/cache/notarius/chunk-plans" { - t.Fatal("redacted mutation changed original") - } - if err := cfg.Validate(); err != nil { - t.Fatalf("Validate() error = %v", err) - } -} - -func TestChunkCacheDirectoryValidation(t *testing.T) { - cfg := Default() - cfg.Workspace.ChunkCache.Directory = "/var/cache/notarius/chunk-plans" - if err := cfg.Validate(); err != nil { - t.Fatalf("Validate(system root) error = %v", err) - } - cfg.Workspace.ChunkCache.Directory = "bad\x00path" - if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "NUL") { - t.Fatalf("Validate(NUL directory) error = %v", err) - } -} diff --git a/internal/core/config/config_test.go b/internal/core/config/config_test.go deleted file mode 100644 index 31f89bd..0000000 --- a/internal/core/config/config_test.go +++ /dev/null @@ -1,85 +0,0 @@ -//go:build legacy - -package config - -import ( - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" -) - -func TestDefaultValues(t *testing.T) { - cfg := Default() - - if cfg.Scriptorium.ProfileDir != "" || cfg.Scriptorium.ProfileFile != "" { - t.Fatalf("unexpected Scriptorium profile source defaults: %+v", cfg.Scriptorium) - } - if len(cfg.Pipelines) != 0 { - t.Fatalf("expected no built-in pipeline profiles, got %v", cfg.Pipelines) - } - if cfg.Concurrency.TotalLLM != 1 { - t.Fatalf("unexpected total LLM concurrency: %d", cfg.Concurrency.TotalLLM) - } - if got := cfg.Concurrency.StageWorkers["extract"]; got != 1 { - t.Fatalf("unexpected extract workers: %d", got) - } - if cfg.Diagnostics.WorkDir != "/tmp/notarius" { - t.Fatalf("unexpected diagnostics work dir: %q", cfg.Diagnostics.WorkDir) - } - if cfg.Diagnostics.Retention != diagnostics.RetentionAuto { - t.Fatalf("unexpected diagnostics retention: %q", cfg.Diagnostics.Retention) - } - if cfg.Workspace.Directory != "" { - t.Fatalf("unexpected workspace directory: %q", cfg.Workspace.Directory) - } - if !cfg.Workspace.Diagnostics.Enabled || !cfg.DiagnosticsEnabled() { - t.Fatalf("expected workspace diagnostics enabled by default: %+v", cfg.Workspace.Diagnostics) - } - if cfg.Workspace.Diagnostics.Retention != "" { - t.Fatalf("unexpected workspace diagnostics retention: %q", cfg.Workspace.Diagnostics.Retention) - } - if cfg.Workspace.Resume.Enabled { - t.Fatalf("workspace resume should be disabled by default") - } - if cfg.Workspace.Debug.Enabled { - t.Fatalf("workspace debug should be disabled by default") - } -} - -func TestApplyFileConfigMergesWithDefaults(t *testing.T) { - fileCfg, err := ParseFileConfigYAML([]byte(` -version: 2 -scriptorium: - profile_dir: ./profiles -pipelines: - example: - input: fake/input - artifacts: - events: - extract: fake/extract -`)) - if err != nil { - t.Fatalf("ParseFileConfigYAML: %v", err) - } - - cfg := Default() - if err := cfg.applyFileConfigWithLookup(fileCfg, emptyLookup); err != nil { - t.Fatalf("ApplyFileConfig: %v", err) - } - - if cfg.Scriptorium.ProfileDir != "./profiles" { - t.Fatalf("expected Scriptorium profile dir, got %+v", cfg.Scriptorium) - } - if cfg.Concurrency.TotalLLM != 1 { - t.Fatalf("expected default concurrency preserved, got %d", cfg.Concurrency.TotalLLM) - } - if got := cfg.Concurrency.StageWorkers["extract"]; got != 1 { - t.Fatalf("expected default extract workers preserved, got %d", got) - } - if cfg.Diagnostics.Retention != diagnostics.RetentionAuto { - t.Fatalf("expected default diagnostics retention preserved, got %q", cfg.Diagnostics.Retention) - } - if _, ok := cfg.Pipelines["example"]; !ok { - t.Fatalf("expected file pipeline to be applied") - } -} diff --git a/internal/core/config/effective_config_test.go b/internal/core/config/effective_config_test.go deleted file mode 100644 index 0f9e042..0000000 --- a/internal/core/config/effective_config_test.go +++ /dev/null @@ -1,243 +0,0 @@ -//go:build legacy - -package config - -import ( - "strings" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" -) - -func TestResolveRejectsEmptyAndUnknownPipelineID(t *testing.T) { - tests := []struct { - name string - pipelineID string - want string - }{ - {name: "empty", pipelineID: " ", want: "pipeline id"}, - {name: "unknown", pipelineID: "missing", want: "not configured"}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, err := validConfig().Resolve(ResolveInput{PipelineID: tc.pipelineID, Catalog: fakeCatalog(t)}) - if err == nil || !strings.Contains(err.Error(), tc.want) { - t.Fatalf("expected error containing %q, got %v", tc.want, err) - } - }) - } -} - -func TestResolveMaterializesDefaultExtractWorkersFromEffectiveTotal(t *testing.T) { - cfg := validConfig() - cfg.Concurrency.TotalLLM = 4 - - effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)}) - if err != nil { - t.Fatalf("Resolve() error = %v, want nil", err) - } - if got := effective.Config.Concurrency.StageWorkers["extract"]; got != 4 { - t.Fatalf("effective extract workers = %d, want total concurrency 4", got) - } -} - -func TestResolveLaneFilteringSuccessAndFailure(t *testing.T) { - effective, err := validConfig().Resolve(ResolveInput{ - PipelineID: " example ", - Only: []string{" notes "}, - Catalog: fakeCatalog(t), - }) - if err != nil { - t.Fatalf("Resolve: %v", err) - } - - if effective.PipelineID != "example" { - t.Fatalf("unexpected pipeline ID: %q", effective.PipelineID) - } - if len(effective.ResolvedPipeline.ArtifactLanes) != 1 || effective.ResolvedPipeline.ArtifactLanes[0].ID != "notes" { - t.Fatalf("unexpected resolved lanes: %+v", effective.ResolvedPipeline.ArtifactLanes) - } - if effective.ResolvedPipeline.Digest == "" { - t.Fatalf("expected digest") - } - - _, err = validConfig().Resolve(ResolveInput{ - PipelineID: "example", - Only: []string{"missing"}, - Catalog: fakeCatalog(t), - }) - if err == nil || !strings.Contains(err.Error(), "selected artifact lane") { - t.Fatalf("expected invalid lane error, got %v", err) - } -} - -func TestResolveUsesTrimmedPipelineMapKeys(t *testing.T) { - cfg := validConfig() - cfg.Pipelines[" example "] = cfg.Pipelines["example"] - delete(cfg.Pipelines, "example") - - effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)}) - if err != nil { - t.Fatalf("Resolve: %v", err) - } - if effective.PipelineID != "example" { - t.Fatalf("unexpected pipeline ID: %q", effective.PipelineID) - } -} - -func TestResolveSurfacesUnknownModuleKeyThroughCatalog(t *testing.T) { - cfg := validConfig() - lane := cfg.Pipelines["example"].Artifacts["events"] - lane.Extract = pipeline.Binding("missing/extract") - cfg.Pipelines["example"].Artifacts["events"] = lane - - _, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)}) - if err == nil || !strings.Contains(err.Error(), "missing/extract") || !strings.Contains(err.Error(), "events") { - t.Fatalf("expected unknown module error with lane context, got %v", err) - } -} - -func TestResolveSurfacesMissingCapabilityThroughCatalog(t *testing.T) { - _, err := validConfig().Resolve(ResolveInput{ - PipelineID: "example", - Catalog: fakeCatalog(t, pipeline.ModuleSpec{ - Key: "json", - Stage: pipeline.StageOutput, - Requires: []string{"missing-capability"}, - }), - }) - if err == nil || !strings.Contains(err.Error(), "missing capability") || !strings.Contains(err.Error(), "json") { - t.Fatalf("expected missing capability error, got %v", err) - } -} - -func TestResolveCanBindSceneChunkerFromCatalog(t *testing.T) { - cfg := validConfig() - profile := cfg.Pipelines["example"] - profile.Chunk = pipeline.Binding("dnd/scenes") - lane := profile.Artifacts["events"] - profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"events": lane} - cfg.Pipelines["example"] = profile - - catalog := fakeCatalog(t, - pipeline.ModuleSpec{ - Key: "fake/input", - Stage: pipeline.StageInput, - Provides: []string{"source.transcript"}, - }, - pipeline.ModuleSpec{ - Key: "fake/extract", - Stage: pipeline.StageExtract, - Requires: []string{"chunks", "source.transcript"}, - Provides: []string{"artifact"}, - }, - ) - mustRegisterChunker(t, catalog.Chunkers, pipeline.ModuleSpec{ - Key: "dnd/scenes", - Stage: pipeline.StageChunk, - Requires: []string{"source.transcript"}, - Provides: []string{"chunks"}, - }) - - effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: catalog}) - if err != nil { - t.Fatalf("Resolve() error = %v, want nil", err) - } - if got := effective.ResolvedPipeline.Chunk.Module; got != "dnd/scenes" { - t.Fatalf("Chunk.Module = %q, want dnd/scenes", got) - } -} - -func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) { - cfg := validConfig() - first, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)}) - if err != nil { - t.Fatalf("Resolve first: %v", err) - } - - lane := cfg.Pipelines["example"].Artifacts["events"] - lane.Extract.Options = map[string]any{"temperature": 0.2} - cfg.Pipelines["example"].Artifacts["events"] = lane - second, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)}) - if err != nil { - t.Fatalf("Resolve second: %v", err) - } - - if first.ResolvedPipeline.Digest == second.ResolvedPipeline.Digest { - t.Fatalf("expected digest to change, got %q", first.ResolvedPipeline.Digest) - } -} - -func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) { - cfg := validConfig() - profile := cfg.Pipelines["example"] - profile.Input.LLMProfile = "input-profile" - profile.Output.LLMProfile = "output-profile" - lane := profile.Artifacts["events"] - lane.Merge.LLMProfile = "merge-profile" - lane.Extract.Validators = pipeline.ValidatorOverride{ - Set: true, - Validators: []pipeline.ModuleBinding{ - {Module: "fake/llm-validator", LLMProfile: "validator-profile"}, - }, - } - profile.Artifacts["events"] = lane - cfg.Pipelines["example"] = profile - - base, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)}) - if err != nil { - t.Fatalf("Resolve base: %v", err) - } - effective, err := cfg.Resolve(ResolveInput{ - PipelineID: "example", - Catalog: fakeCatalog(t), - LLMProfileOverride: "runtime", - }) - if err != nil { - t.Fatalf("Resolve override: %v", err) - } - - if base.ResolvedPipeline.Digest == effective.ResolvedPipeline.Digest { - t.Fatalf("expected digest to change after LLM profile override") - } - for _, binding := range llmCapableBindings(effective.ResolvedPipeline) { - if binding.LLMProfile != "runtime" { - t.Fatalf("LLM-capable binding profile = %q, want runtime", binding.LLMProfile) - } - } - if effective.ResolvedPipeline.Input.LLMProfile != "input-profile" { - t.Fatalf("input profile = %q, want original input-profile", effective.ResolvedPipeline.Input.LLMProfile) - } - if effective.ResolvedPipeline.Output.LLMProfile != "output-profile" { - t.Fatalf("output profile = %q, want original output-profile", effective.ResolvedPipeline.Output.LLMProfile) - } - eventLane := effective.ResolvedPipeline.ArtifactLanes[0] - if eventLane.Merge.LLMProfile != "runtime" { - t.Fatalf("merge profile = %q, want runtime", eventLane.Merge.LLMProfile) - } - validatorChain := findEffectiveValidatorChain(effective.ResolvedPipeline.ValidatorChains, pipeline.StageExtract, "events", "fake/extract") - if validatorChain == nil || len(validatorChain.Validators) != 1 { - t.Fatalf("validator chain = %#v, want one extract validator", effective.ResolvedPipeline.ValidatorChains) - } - if validatorChain.Validators[0].Binding.LLMProfile != "validator-profile" { - t.Fatalf("validator profile = %q, want original validator-profile", validatorChain.Validators[0].Binding.LLMProfile) - } -} - -func findEffectiveValidatorChain(chains []pipeline.ResolvedValidatorChain, stage pipeline.ModuleStage, laneID string, module string) *pipeline.ResolvedValidatorChain { - for i := range chains { - if chains[i].Stage == stage && chains[i].LaneID == laneID && chains[i].ModuleKey == module { - return &chains[i] - } - } - return nil -} - -func llmCapableBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding { - bindings := []pipeline.ModuleBinding{resolved.Chunk} - for _, lane := range resolved.ArtifactLanes { - bindings = append(bindings, lane.Extract, lane.Merge, lane.Normalize) - } - return bindings -} diff --git a/internal/core/config/env_test.go b/internal/core/config/env_test.go deleted file mode 100644 index 009b3d1..0000000 --- a/internal/core/config/env_test.go +++ /dev/null @@ -1,185 +0,0 @@ -//go:build legacy - -package config - -import ( - "strings" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" - "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" -) - -func TestApplyEnvOverridesOperationalValues(t *testing.T) { - cfg := Default() - cfg.Pipelines["example"] = pipeline.PipelineProfile{ID: "example", Input: pipeline.Binding("before")} - - err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{ - "NOTARIUS_TOTAL_LLM_CONCURRENCY": "3", - "NOTARIUS_STAGE_WORKERS_EXTRACT": "2", - "NOTARIUS_WORK_DIR": "/tmp/notarius-env", - "NOTARIUS_DIAGNOSTICS_RETENTION": "never", - "NOTARIUS_WORKSPACE_DIR": "/var/lib/notarius-env", - "NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED": "false", - "NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION": "always", - "NOTARIUS_WORKSPACE_RESUME_ENABLED": "true", - "NOTARIUS_WORKSPACE_DEBUG_ENABLED": "true", - "NOTARIUS_PIPELINE_INPUT": "after", - })) - if err != nil { - t.Fatalf("ApplyEnvOverrides: %v", err) - } - - if cfg.Scriptorium.ProfileDir != "" || cfg.Scriptorium.ProfileFile != "" { - t.Fatalf("LLM environment overrides must not change Scriptorium config: %+v", cfg.Scriptorium) - } - if cfg.Concurrency.TotalLLM != 3 { - t.Fatalf("unexpected total concurrency: %d", cfg.Concurrency.TotalLLM) - } - if got := cfg.Concurrency.StageWorkers["extract"]; got != 2 { - t.Fatalf("extract workers = %d, want 2", got) - } - if cfg.Workspace.Directory != "/var/lib/notarius-env" { - t.Fatalf("unexpected workspace directory: %q", cfg.Workspace.Directory) - } - if cfg.DiagnosticsEnabled() { - t.Fatalf("expected workspace diagnostics disabled") - } - if cfg.Diagnostics.WorkDir != "/var/lib/notarius-env/diagnostics" || cfg.Diagnostics.Retention != diagnostics.RetentionAlways { - t.Fatalf("unexpected diagnostics config: %+v", cfg.Diagnostics) - } - if !cfg.Workspace.Resume.Enabled { - t.Fatalf("expected workspace resume enabled") - } - if !cfg.Workspace.Debug.Enabled { - t.Fatalf("expected workspace debug enabled") - } - if cfg.Pipelines["example"].Input.Module != "before" { - t.Fatalf("environment overrides must not change pipeline wiring: %+v", cfg.Pipelines["example"]) - } -} - -func TestApplyEnvOverridesRejectsInvalidIntegers(t *testing.T) { - for _, name := range []string{"NOTARIUS_TOTAL_LLM_CONCURRENCY", "NOTARIUS_STAGE_WORKERS_EXTRACT"} { - t.Run(name, func(t *testing.T) { - cfg := Default() - err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{name: "many"})) - if err == nil || !strings.Contains(err.Error(), name) { - t.Fatalf("expected named integer error, got %v", err) - } - }) - } -} - -func TestStageWorkerEnvironmentPrecedenceAndDefaulting(t *testing.T) { - fileCfg, err := ParseFileConfigYAML([]byte(` -version: 2 -concurrency: - total_llm: 4 - stage_workers: - extract: 2 -`)) - if err != nil { - t.Fatalf("ParseFileConfigYAML() error = %v", err) - } - cfg := Default() - if err := cfg.applyFileConfigWithLookup(fileCfg, emptyLookup); err != nil { - t.Fatalf("ApplyFileConfig() error = %v", err) - } - if err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{ - "NOTARIUS_TOTAL_LLM_CONCURRENCY": "5", - "NOTARIUS_STAGE_WORKERS_EXTRACT": "3", - })); err != nil { - t.Fatalf("ApplyEnvOverrides() error = %v", err) - } - if cfg.Concurrency.TotalLLM != 5 || cfg.Concurrency.StageWorkers["extract"] != 3 { - t.Fatalf("effective concurrency = %#v, want total 5 and extract 3", cfg.Concurrency) - } - if err := cfg.Validate(); err != nil { - t.Fatalf("Validate(overridden) error = %v, want nil", err) - } - - defaulted := Default() - if err := defaulted.applyEnvOverridesWithLookup(mapLookup(map[string]string{"NOTARIUS_TOTAL_LLM_CONCURRENCY": "6"})); err != nil { - t.Fatalf("ApplyEnvOverrides(defaulted) error = %v", err) - } - if got := defaulted.Concurrency.StageWorkers["extract"]; got != 6 { - t.Fatalf("defaulted extract workers = %d, want effective total 6", got) - } -} - -func TestStageWorkerRangeValidationUsesFinalEnvironmentTotal(t *testing.T) { - fileCfg, err := ParseFileConfigYAML([]byte(` -version: 2 -concurrency: - total_llm: 4 - stage_workers: - extract: 5 -`)) - if err != nil { - t.Fatalf("ParseFileConfigYAML() error = %v", err) - } - cfg := Default() - if err := cfg.applyFileConfigWithLookup(fileCfg, emptyLookup); err != nil { - t.Fatalf("ApplyFileConfig() error = %v", err) - } - if err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{"NOTARIUS_TOTAL_LLM_CONCURRENCY": "6"})); err != nil { - t.Fatalf("ApplyEnvOverrides() error = %v", err) - } - if err := cfg.Validate(); err != nil { - t.Fatalf("Validate() error = %v, want final total to make extract workers valid", err) - } -} - -func TestApplyEnvOverridesRejectsInvalidBooleans(t *testing.T) { - for _, name := range []string{ - "NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED", - "NOTARIUS_WORKSPACE_RESUME_ENABLED", - "NOTARIUS_WORKSPACE_DEBUG_ENABLED", - } { - t.Run(name, func(t *testing.T) { - cfg := Default() - err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{name: "maybe"})) - if err == nil || !strings.Contains(err.Error(), name) { - t.Fatalf("expected named boolean error, got %v", err) - } - }) - } -} - -func TestApplyEnvOverridesLegacyDiagnosticsRemainCompatibleWithoutWorkspace(t *testing.T) { - cfg := Default() - - err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{ - "NOTARIUS_WORK_DIR": "/tmp/notarius-env", - "NOTARIUS_DIAGNOSTICS_RETENTION": "never", - })) - if err != nil { - t.Fatalf("ApplyEnvOverrides: %v", err) - } - - if cfg.Diagnostics.WorkDir != "/tmp/notarius-env" { - t.Fatalf("diagnostics work dir = %q, want legacy env", cfg.Diagnostics.WorkDir) - } - if cfg.Diagnostics.Retention != diagnostics.RetentionNever { - t.Fatalf("diagnostics retention = %q, want legacy env", cfg.Diagnostics.Retention) - } -} - -func TestLoadFromEnvUsesDefaultConfig(t *testing.T) { - t.Setenv("NOTARIUS_TOTAL_LLM_CONCURRENCY", "2") - - cfg, err := LoadFromEnv() - if err != nil { - t.Fatalf("LoadFromEnv: %v", err) - } - if cfg.Scriptorium.ProfileDir != "" || cfg.Scriptorium.ProfileFile != "" { - t.Fatalf("unexpected Scriptorium config from env: %+v", cfg.Scriptorium) - } - if cfg.Concurrency.TotalLLM != 2 { - t.Fatalf("expected env concurrency override, got %+v", cfg.Concurrency) - } - if got := cfg.Concurrency.StageWorkers["extract"]; got != 2 { - t.Fatalf("expected extract workers to default to total, got %d", got) - } -} diff --git a/internal/core/config/file_config_test.go b/internal/core/config/file_config_test.go deleted file mode 100644 index c6503bb..0000000 --- a/internal/core/config/file_config_test.go +++ /dev/null @@ -1,725 +0,0 @@ -//go:build legacy - -package config - -import ( - "os" - "path/filepath" - "reflect" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" -) - -func TestParseMinimalValidConfig(t *testing.T) { - fileCfg, err := ParseFileConfigYAML([]byte(` -version: 2 -`)) - if err != nil { - t.Fatalf("ParseFileConfigYAML: %v", err) - } - if fileCfg.Version != SupportedFileConfigVersion { - t.Fatalf("unexpected version: %d", fileCfg.Version) - } -} - -func TestLoadFileConfig(t *testing.T) { - path := filepath.Join(t.TempDir(), "config.yml") - if err := os.WriteFile(path, []byte("version: 2\n"), 0o644); err != nil { - t.Fatalf("write config: %v", err) - } - - fileCfg, err := LoadFileConfig(path) - if err != nil { - t.Fatalf("LoadFileConfig: %v", err) - } - if fileCfg.Version != SupportedFileConfigVersion { - t.Fatalf("unexpected version: %d", fileCfg.Version) - } -} - -func TestParseFileConfigRejectsUnknownYAMLFields(t *testing.T) { - _, err := ParseFileConfigYAML([]byte(` -version: 2 -unexpected: true -`)) - if err == nil || !strings.Contains(err.Error(), "field unexpected not found") { - t.Fatalf("expected unknown field error, got %v", err) - } -} - -func TestParseFileConfigRejectsUnknownModuleBindingFields(t *testing.T) { - _, err := ParseFileConfigYAML([]byte(` -version: 2 -pipelines: - example: - input: - module: fake/input - unexpected: true - artifacts: - events: - extract: fake/extract -`)) - if err == nil || !strings.Contains(err.Error(), "field unexpected not found") { - t.Fatalf("expected unknown binding field error, got %v", err) - } -} - -func TestParseFileConfigRejectsMissingAndUnsupportedVersion(t *testing.T) { - tests := []struct { - name string - data string - want string - }{ - {name: "missing", data: `scriptorium: {}`, want: "version is required"}, - {name: "unsupported", data: `version: 1`, want: "unsupported config version"}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, err := ParseFileConfigYAML([]byte(tc.data)) - if err == nil || !strings.Contains(err.Error(), tc.want) { - t.Fatalf("expected error containing %q, got %v", tc.want, err) - } - }) - } -} - -func TestParseFileConfigRejectsStaleLLMProfiles(t *testing.T) { - _, err := ParseFileConfigYAML([]byte(` -version: 2 -llm_profiles: - default: {} -`)) - if err == nil || !strings.Contains(err.Error(), "llm_profiles") { - t.Fatalf("expected stale llm_profiles error, got %v", err) - } -} - -func TestParseFileConfigScriptoriumProfileSources(t *testing.T) { - t.Run("profile dir", func(t *testing.T) { - cfg := parseAndApplyConfig(t, ` -version: 2 -scriptorium: - profile_dir: ./profiles -`) - if cfg.Scriptorium.ProfileDir != "./profiles" || cfg.Scriptorium.ProfileFile != "" { - t.Fatalf("Scriptorium = %+v, want profile_dir", cfg.Scriptorium) - } - }) - - t.Run("profile file", func(t *testing.T) { - cfg := parseAndApplyConfig(t, ` -version: 2 -scriptorium: - profile_file: ./profiles.yml -`) - if cfg.Scriptorium.ProfileFile != "./profiles.yml" || cfg.Scriptorium.ProfileDir != "" { - t.Fatalf("Scriptorium = %+v, want profile_file", cfg.Scriptorium) - } - }) -} - -func TestParseFileConfigModuleBindingForms(t *testing.T) { - cfg := parseAndApplyConfig(t, ` -version: 2 -pipelines: - example: - input: fake/input - chunk: - module: generic - retries: 2 - options: - size: 10 - flags: - - alpha - nested: - enabled: true - artifacts: - events: - extract: - module: fake/extract - llm_profile: fast - retries: 3 - options: - temperature: 0 - merge: - module: appendorder - retries: 1 - normalize: - module: noop - output: json -`) - - profile := cfg.Pipelines["example"] - if profile.Input.Module != "fake/input" { - t.Fatalf("unexpected input binding: %+v", profile.Input) - } - if profile.Chunk.Module != "generic" { - t.Fatalf("unexpected chunk binding: %+v", profile.Chunk) - } - if profile.Chunk.Retries != 2 { - t.Fatalf("chunk retries = %d, want 2", profile.Chunk.Retries) - } - if profile.Chunk.Options["size"] != 10 { - t.Fatalf("expected chunk options to preserve scalar, got %#v", profile.Chunk.Options) - } - if !reflect.DeepEqual(profile.Chunk.Options["flags"], []any{"alpha"}) { - t.Fatalf("expected list option, got %#v", profile.Chunk.Options["flags"]) - } - nested, ok := profile.Chunk.Options["nested"].(map[string]any) - if !ok || nested["enabled"] != true { - t.Fatalf("expected nested map option, got %#v", profile.Chunk.Options["nested"]) - } - - lane := profile.Artifacts["events"] - if lane.Extract.Module != "fake/extract" || lane.Extract.LLMProfile != "fast" { - t.Fatalf("unexpected extract binding: %+v", lane.Extract) - } - if lane.Extract.Retries != 3 || lane.Merge.Retries != 1 { - t.Fatalf("unexpected retries: extract=%d merge=%d", lane.Extract.Retries, lane.Merge.Retries) - } - if lane.Extract.Options["temperature"] != 0 { - t.Fatalf("expected object options, got %#v", lane.Extract.Options) - } - if lane.Merge.Module != "appendorder" || lane.Normalize.Module != "noop" { - t.Fatalf("unexpected lane defaults: %+v", lane) - } - if profile.Output.Module != "json" { - t.Fatalf("unexpected output binding: %+v", profile.Output) - } -} - -func TestParseFileConfigReferenceMaps(t *testing.T) { - cfg := parseAndApplyConfig(t, ` -version: 2 -pipelines: - example: - input: fake/input - references: - " roster ": " ./shared-roster.yml " - artifacts: - events: - extract: fake/extract - references: - " lore ": " ./lore.md " -`) - - profile := cfg.Pipelines["example"] - if !reflect.DeepEqual(profile.References, map[string]string{"roster": "./shared-roster.yml"}) { - t.Fatalf("pipeline references = %#v, want trimmed map", profile.References) - } - gotLaneRefs := profile.Artifacts["events"].References - if !reflect.DeepEqual(gotLaneRefs, map[string]string{"lore": "./lore.md"}) { - t.Fatalf("lane references = %#v, want trimmed map", gotLaneRefs) - } -} - -func TestParseFileConfigStageLocalReferenceMaps(t *testing.T) { - cfg := parseAndApplyConfig(t, ` -version: 2 -pipelines: - example: - input: fake/input - chunk: - module: generic - references: - " scene_guide ": " ./scenes.md " - artifacts: - events: - extract: - module: fake/extract - references: - " glossary ": " ./glossary.md " - " roster ": " ./extract-roster.yml " - references: - roster: ./legacy-roster.yml - lore: ./lore.md - merge: - module: appendorder - references: - " merge_notes ": " ./merge.md " - normalize: - module: noop - references: - " normalization_notes ": " ./normalization.md " -`) - - profile := cfg.Pipelines["example"] - if !reflect.DeepEqual(profile.Chunk.References, map[string]string{"scene_guide": "./scenes.md"}) { - t.Fatalf("chunk references = %#v, want trimmed map", profile.Chunk.References) - } - lane := profile.Artifacts["events"] - if !reflect.DeepEqual(lane.References, map[string]string{"lore": "./lore.md", "roster": "./legacy-roster.yml"}) { - t.Fatalf("lane references = %#v, want trimmed map", lane.References) - } - wantExtract := map[string]string{ - "glossary": "./glossary.md", - "lore": "./lore.md", - "roster": "./extract-roster.yml", - } - if !reflect.DeepEqual(lane.Extract.References, wantExtract) { - t.Fatalf("extract references = %#v, want legacy merged with extract override %#v", lane.Extract.References, wantExtract) - } - if !reflect.DeepEqual(lane.Merge.References, map[string]string{"merge_notes": "./merge.md"}) { - t.Fatalf("merge references = %#v, want trimmed map", lane.Merge.References) - } - if !reflect.DeepEqual(lane.Normalize.References, map[string]string{"normalization_notes": "./normalization.md"}) { - t.Fatalf("normalize references = %#v, want trimmed map", lane.Normalize.References) - } -} - -func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) { - cfg := parseAndApplyConfig(t, ` -version: 2 -pipelines: - example: - input: fake/input - artifacts: - events: - extract: fake/extract - validators: - - fake/validator - - module: fake/llm-validator - llm_profile: careful - options: - threshold: 0.7 -`) - - validators := cfg.Pipelines["example"].Artifacts["events"].Validators - if len(validators) != 2 { - t.Fatalf("expected two validators, got %d", len(validators)) - } - if validators[0].Module != "fake/validator" { - t.Fatalf("unexpected shorthand validator: %+v", validators[0]) - } - if validators[1].Module != "fake/llm-validator" || validators[1].LLMProfile != "careful" { - t.Fatalf("unexpected object validator: %+v", validators[1]) - } - if validators[1].Options["threshold"] != 0.7 { - t.Fatalf("unexpected validator options: %#v", validators[1].Options) - } -} - -func TestParseFileConfigStageLocalValidatorOverrides(t *testing.T) { - cfg := parseAndApplyConfig(t, ` -version: 2 -pipelines: - example: - input: fake/input - chunk: - module: generic - validators: [] - artifacts: - events: - extract: - module: fake/extract - validators: - - fake/validator - - module: fake/llm-validator - llm_profile: careful - options: - threshold: 0.7 - merge: - module: appendorder - validators: [] - normalize: - module: noop -`) - - profile := cfg.Pipelines["example"] - if !profile.Chunk.Validators.Set || len(profile.Chunk.Validators.Validators) != 0 { - t.Fatalf("chunk validator override = %#v, want explicit empty", profile.Chunk.Validators) - } - lane := profile.Artifacts["events"] - if !lane.Extract.Validators.Set { - t.Fatalf("extract validator override Set = false, want true") - } - validators := lane.Extract.Validators.Validators - if len(validators) != 2 { - t.Fatalf("extract validators = %#v, want two validators", validators) - } - if validators[0].Module != "fake/validator" { - t.Fatalf("first validator = %#v, want fake/validator", validators[0]) - } - if validators[1].Module != "fake/llm-validator" || validators[1].LLMProfile != "careful" { - t.Fatalf("second validator = %#v, want LLM validator with profile", validators[1]) - } - if validators[1].Options["threshold"] != 0.7 { - t.Fatalf("second validator options = %#v, want threshold", validators[1].Options) - } - if !lane.Merge.Validators.Set || len(lane.Merge.Validators.Validators) != 0 { - t.Fatalf("merge validator override = %#v, want explicit empty", lane.Merge.Validators) - } - if lane.Normalize.Validators.Set { - t.Fatalf("normalize validator override Set = true, want omitted") - } -} - -func TestApplyFileConfigRejectsDuplicateTrimmedPipelineIDs(t *testing.T) { - fileCfg, err := ParseFileConfigYAML([]byte(` -version: 2 -pipelines: - example: - input: fake/input - " example ": - input: fake/other-input -`)) - if err != nil { - t.Fatalf("ParseFileConfigYAML: %v", err) - } - - cfg := Default() - err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup) - if err == nil || !strings.Contains(err.Error(), "pipeline id") || !strings.Contains(err.Error(), "duplicated") { - t.Fatalf("expected duplicate pipeline ID error, got %v", err) - } -} - -func TestApplyFileConfigRejectsDuplicateTrimmedArtifactLaneIDs(t *testing.T) { - fileCfg, err := ParseFileConfigYAML([]byte(` -version: 2 -pipelines: - example: - input: fake/input - artifacts: - events: - extract: fake/extract - " events ": - extract: fake/other-extract -`)) - if err != nil { - t.Fatalf("ParseFileConfigYAML: %v", err) - } - - cfg := Default() - err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup) - if err == nil || !strings.Contains(err.Error(), `pipeline "example" artifact lane id`) || !strings.Contains(err.Error(), "duplicated") { - t.Fatalf("expected duplicate artifact lane ID error, got %v", err) - } -} - -func TestApplyFileConfigRejectsDuplicateTrimmedReferenceSlots(t *testing.T) { - tests := []struct { - name string - raw string - want string - }{ - { - name: "pipeline", - raw: ` -version: 2 -pipelines: - example: - input: fake/input - references: - roster: ./first.yml - " roster ": ./second.yml -`, - want: `pipeline "example" reference slot`, - }, - { - name: "lane", - raw: ` -version: 2 -pipelines: - example: - input: fake/input - artifacts: - events: - extract: fake/extract - references: - roster: ./first.yml - " roster ": ./second.yml -`, - want: `pipeline "example" lane "events" reference slot`, - }, - { - name: "chunk", - raw: ` -version: 2 -pipelines: - example: - input: fake/input - chunk: - module: generic - references: - roster: ./first.yml - " roster ": ./second.yml -`, - want: `pipeline "example" chunk reference slot`, - }, - { - name: "extract", - raw: ` -version: 2 -pipelines: - example: - input: fake/input - artifacts: - events: - extract: - module: fake/extract - references: - roster: ./first.yml - " roster ": ./second.yml -`, - want: `pipeline "example" lane "events" extract reference slot`, - }, - { - name: "normalize", - raw: ` -version: 2 -pipelines: - example: - input: fake/input - artifacts: - events: - extract: fake/extract - normalize: - module: noop - references: - roster: ./first.yml - " roster ": ./second.yml -`, - want: `pipeline "example" lane "events" normalize reference slot`, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - fileCfg, err := ParseFileConfigYAML([]byte(tc.raw)) - if err != nil { - t.Fatalf("ParseFileConfigYAML: %v", err) - } - cfg := Default() - err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup) - if err == nil || !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), "duplicated") { - t.Fatalf("expected duplicate reference slot error, got %v", err) - } - }) - } -} - -func TestApplyFileConfigRejectsInvalidScriptoriumSources(t *testing.T) { - tests := []struct { - name string - raw string - want string - }{ - {name: "empty profile dir", raw: "profile_dir: ' '", want: "profile_dir"}, - {name: "empty profile file", raw: "profile_file: ' '", want: "profile_file"}, - {name: "both sources", raw: "profile_dir: ./profiles\n profile_file: ./profiles.yml", want: "mutually exclusive"}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - cfg := Default() - fileCfg, err := ParseFileConfigYAML([]byte(` -version: 2 -scriptorium: - ` + tc.raw + ` -`)) - if err != nil { - t.Fatalf("ParseFileConfigYAML: %v", err) - } - err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup) - if err == nil { - err = cfg.Validate() - } - if err == nil || !strings.Contains(err.Error(), tc.want) { - t.Fatalf("expected error containing %q, got %v", tc.want, err) - } - }) - } -} - -func TestApplyFileConfigOperationalSections(t *testing.T) { - cfg := parseAndApplyConfig(t, ` -version: 2 -concurrency: - total_llm: 4 -diagnostics: - work_dir: /tmp/notarius-test - retention: always -`) - - if cfg.Concurrency.TotalLLM != 4 { - t.Fatalf("unexpected total concurrency: %d", cfg.Concurrency.TotalLLM) - } - if got := cfg.Concurrency.StageWorkers["extract"]; got != 4 { - t.Fatalf("default extract workers = %d, want total concurrency", got) - } - if cfg.Diagnostics.WorkDir != "/tmp/notarius-test" { - t.Fatalf("unexpected work dir: %q", cfg.Diagnostics.WorkDir) - } - if cfg.Diagnostics.Retention != diagnostics.RetentionAlways { - t.Fatalf("unexpected retention: %q", cfg.Diagnostics.Retention) - } -} - -func TestApplyFileConfigStageWorkers(t *testing.T) { - cfg := parseAndApplyConfig(t, ` -version: 2 -concurrency: - total_llm: 4 - stage_workers: - extract: 3 -`) - - if got := cfg.Concurrency.StageWorkers["extract"]; got != 3 { - t.Fatalf("extract workers = %d, want 3", got) - } - if err := cfg.Validate(); err != nil { - t.Fatalf("Validate() error = %v, want nil", err) - } -} - -func TestApplyFileConfigEmptyStageWorkersDefaultsExtractToTotal(t *testing.T) { - cfg := parseAndApplyConfig(t, ` -version: 2 -concurrency: - total_llm: 4 - stage_workers: {} -`) - if got := cfg.Concurrency.StageWorkers["extract"]; got != 4 { - t.Fatalf("extract workers = %d, want total concurrency 4", got) - } -} - -func TestApplyFileConfigRejectsUnsupportedStageWorkerKeys(t *testing.T) { - for _, test := range []struct { - name string - key string - want string - }{ - {name: "empty", key: "' '", want: "must not be empty"}, - {name: "unknown", key: "merge", want: "not supported"}, - } { - t.Run(test.name, func(t *testing.T) { - fileCfg, err := ParseFileConfigYAML([]byte("version: 2\nconcurrency:\n stage_workers:\n " + test.key + ": 1\n")) - if err != nil { - t.Fatalf("ParseFileConfigYAML() error = %v", err) - } - cfg := Default() - err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup) - if err == nil || !strings.Contains(err.Error(), test.want) { - t.Fatalf("ApplyFileConfig() error = %v, want %q", err, test.want) - } - }) - } -} - -func TestApplyFileConfigWorkspaceSection(t *testing.T) { - cfg := parseAndApplyConfig(t, ` -version: 2 -workspace: - directory: /var/lib/notarius - diagnostics: - enabled: false - retention: never - resume: - enabled: true - debug: - enabled: true -diagnostics: - work_dir: /tmp/legacy - retention: always -`) - - if cfg.Workspace.Directory != "/var/lib/notarius" { - t.Fatalf("workspace directory = %q, want /var/lib/notarius", cfg.Workspace.Directory) - } - if cfg.DiagnosticsEnabled() { - t.Fatalf("expected diagnostics disabled") - } - if cfg.Diagnostics.WorkDir != "/var/lib/notarius/diagnostics" { - t.Fatalf("effective diagnostics work dir = %q, want workspace diagnostics root", cfg.Diagnostics.WorkDir) - } - if cfg.Diagnostics.Retention != diagnostics.RetentionNever { - t.Fatalf("effective diagnostics retention = %q, want workspace override", cfg.Diagnostics.Retention) - } - if !cfg.Workspace.Resume.Enabled { - t.Fatalf("expected resume enabled") - } - if !cfg.Workspace.Debug.Enabled { - t.Fatalf("expected debug enabled") - } -} - -func TestApplyFileConfigLegacyDiagnosticsRemainCompatible(t *testing.T) { - cfg := parseAndApplyConfig(t, ` -version: 2 -diagnostics: - work_dir: /tmp/legacy - retention: always -`) - - if cfg.Workspace.Directory != "" { - t.Fatalf("workspace directory = %q, want unset", cfg.Workspace.Directory) - } - if !cfg.DiagnosticsEnabled() { - t.Fatalf("expected diagnostics enabled") - } - if cfg.Diagnostics.WorkDir != "/tmp/legacy" { - t.Fatalf("effective diagnostics work dir = %q, want legacy", cfg.Diagnostics.WorkDir) - } - if cfg.Diagnostics.Retention != diagnostics.RetentionAlways { - t.Fatalf("effective diagnostics retention = %q, want legacy", cfg.Diagnostics.Retention) - } -} - -func TestApplyFileConfigWorkspaceRetentionOverridesLegacyRetentionOnlyWhenSet(t *testing.T) { - t.Run("legacy retained", func(t *testing.T) { - cfg := parseAndApplyConfig(t, ` -version: 2 -workspace: - directory: /var/lib/notarius -diagnostics: - retention: never -`) - if cfg.Diagnostics.Retention != diagnostics.RetentionNever { - t.Fatalf("effective diagnostics retention = %q, want legacy", cfg.Diagnostics.Retention) - } - }) - - t.Run("workspace overrides", func(t *testing.T) { - cfg := parseAndApplyConfig(t, ` -version: 2 -workspace: - directory: /var/lib/notarius - diagnostics: - retention: always -diagnostics: - retention: never -`) - if cfg.Diagnostics.Retention != diagnostics.RetentionAlways { - t.Fatalf("effective diagnostics retention = %q, want workspace", cfg.Diagnostics.Retention) - } - }) -} - -func parseAndApplyConfig(t *testing.T, raw string) Config { - t.Helper() - fileCfg, err := ParseFileConfigYAML([]byte(raw)) - if err != nil { - t.Fatalf("ParseFileConfigYAML: %v", err) - } - cfg := Default() - if err := cfg.applyFileConfigWithLookup(fileCfg, emptyLookup); err != nil { - t.Fatalf("ApplyFileConfig: %v", err) - } - return cfg -} - -func emptyLookup(string) (string, bool) { - return "", false -} - -func mapLookup(values map[string]string) func(string) (string, bool) { - return func(key string) (string, bool) { - value, ok := values[key] - return value, ok - } -} diff --git a/internal/core/config/redaction_test.go b/internal/core/config/redaction_test.go deleted file mode 100644 index 0060d6c..0000000 --- a/internal/core/config/redaction_test.go +++ /dev/null @@ -1,174 +0,0 @@ -//go:build legacy - -package config - -import ( - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" -) - -func TestRedactedConfigCopiesScriptoriumConfig(t *testing.T) { - cfg := Default() - cfg.Scriptorium.ProfileDir = "./profiles" - cfg.Workspace.Directory = "/var/lib/notarius" - cfg.Workspace.Resume.Enabled = true - cfg.Concurrency.StageWorkers["extract"] = 1 - - redacted := cfg.Redacted() - - if redacted.Scriptorium.ProfileDir != "./profiles" { - t.Fatalf("expected Scriptorium profile source preserved, got %+v", redacted.Scriptorium) - } - redacted.Scriptorium.ProfileDir = "./changed" - if cfg.Scriptorium.ProfileDir != "./profiles" { - t.Fatalf("redaction mutated original config") - } - if redacted.Workspace.Directory != "/var/lib/notarius" || !redacted.Workspace.Resume.Enabled { - t.Fatalf("expected workspace config preserved, got %+v", redacted.Workspace) - } - redacted.Workspace.Directory = "/changed" - if cfg.Workspace.Directory != "/var/lib/notarius" { - t.Fatalf("redaction mutated original workspace config") - } - redacted.Concurrency.StageWorkers["extract"] = 9 - if cfg.Concurrency.StageWorkers["extract"] != 1 { - t.Fatalf("redaction aliased stage worker map") - } -} - -func TestConfigRedactedSummaryPayloadCopiesConfig(t *testing.T) { - cfg := Default() - cfg.Scriptorium.ProfileFile = "./profiles.yml" - - payload, ok := cfg.RedactedSummaryPayload().(Config) - if !ok { - t.Fatalf("expected Config payload, got %T", cfg.RedactedSummaryPayload()) - } - if payload.Scriptorium.ProfileFile != "./profiles.yml" { - t.Fatalf("expected Scriptorium profile file preserved, got %+v", payload.Scriptorium) - } -} - -func TestEffectiveConfigRedactedSummaryPayloadCopies(t *testing.T) { - cfg := validConfig() - cfg.Concurrency.TotalLLM = 4 - cfg.Concurrency.StageWorkers["extract"] = 2 - lane := cfg.Pipelines["example"].Artifacts["events"] - lane.Extract.Options = map[string]any{"temperature": 0.2} - lane.References = map[string]string{"roster": "./roster.yml"} - lane.Extract.References = map[string]string{"glossary": "./glossary.md"} - lane.Normalize.References = map[string]string{"notes": "./normalize.md"} - cfg.Pipelines["example"].Artifacts["events"] = lane - pipelineProfile := cfg.Pipelines["example"] - pipelineProfile.Chunk.References = map[string]string{"scene_guide": "./scene.md"} - cfg.Pipelines["example"] = pipelineProfile - - effective, err := cfg.Resolve(ResolveInput{ - PipelineID: "example", - Only: []string{"events"}, - Catalog: fakeCatalog(t, - pipeline.ModuleSpec{ - Key: "generic", - Stage: pipeline.StageChunk, - Requires: []string{"source"}, - Provides: []string{"chunks"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "scene_guide"}, - }, - }, - pipeline.ModuleSpec{ - Key: "fake/extract", - Stage: pipeline.StageExtract, - Requires: []string{"chunks"}, - Provides: []string{"artifact"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "glossary"}, - {Name: "roster"}, - }, - }, - pipeline.ModuleSpec{ - Key: "noop", - Stage: pipeline.StageNormalize, - Requires: []string{"merged"}, - Provides: []string{"normalized"}, - ReferenceSlots: []contracts.ReferenceSlot{ - {Name: "notes"}, - }, - }, - ), - }) - if err != nil { - t.Fatalf("Resolve: %v", err) - } - - payload, ok := effective.RedactedSummaryPayload().(EffectiveConfig) - if !ok { - t.Fatalf("expected EffectiveConfig payload, got %T", effective.RedactedSummaryPayload()) - } - if payload.PipelineID != effective.PipelineID || payload.ResolvedPipeline.Digest != effective.ResolvedPipeline.Digest { - t.Fatalf("expected pipeline metadata preserved, got %+v", payload) - } - payload.Config.Concurrency.StageWorkers["extract"] = 4 - if effective.Config.Concurrency.StageWorkers["extract"] != 2 { - t.Fatalf("expected effective stage worker map to be copied") - } - - payload.Only[0] = "changed" - if effective.Only[0] != "events" { - t.Fatalf("expected only lanes to be copied") - } - payload.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] = 1.0 - if effective.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] != 0.2 { - t.Fatalf("expected resolved pipeline options to be copied") - } - payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source = "./changed.yml" - if referenceBindingSource(effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings, "roster") != "./roster.yml" { - t.Fatalf("expected resolved pipeline references to be copied") - } - payload.ResolvedPipeline.Chunk.References["scene_guide"] = "./changed-scene.md" - if effective.ResolvedPipeline.Chunk.References["scene_guide"] != "./scene.md" { - t.Fatalf("expected chunk references to be copied") - } - payload.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] = "./changed-glossary.md" - if effective.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] != "./glossary.md" { - t.Fatalf("expected extract references to be copied") - } - payload.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] = "./changed-normalize.md" - if effective.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] != "./normalize.md" { - t.Fatalf("expected normalize references to be copied") - } - - effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{ - Slots: map[string]contracts.ResolvedReferenceSlot{ - "roster": { - Slot: contracts.ReferenceSlot{Name: "roster"}, - Items: []contracts.ReferenceItem{ - { - SlotName: "roster", - Content: []byte("reference content"), - }, - }, - }, - }, - } - payload, ok = effective.RedactedSummaryPayload().(EffectiveConfig) - if !ok { - t.Fatalf("expected EffectiveConfig payload, got %T", effective.RedactedSummaryPayload()) - } - payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content[0] = 'X' - got := effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content - if string(got) != "reference content" { - t.Fatalf("expected materialized reference content to be copied, got %q", got) - } -} - -func referenceBindingSource(bindings []pipeline.ReferenceBinding, slotName string) string { - for _, binding := range bindings { - if binding.SlotName == slotName { - return binding.Source - } - } - return "" -} diff --git a/internal/core/config/validation_test.go b/internal/core/config/validation_test.go deleted file mode 100644 index 0180f5d..0000000 --- a/internal/core/config/validation_test.go +++ /dev/null @@ -1,666 +0,0 @@ -//go:build legacy - -package config - -import ( - "fmt" - "strings" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" - "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" -) - -func TestValidateSuccessForValidConfig(t *testing.T) { - cfg := validConfig() - - if err := cfg.Validate(); err != nil { - t.Fatalf("Validate: %v", err) - } -} - -func TestValidateAllowsExplicitScriptoriumProfileIDOnBinding(t *testing.T) { - cfg := validConfig() - lane := cfg.Pipelines["example"].Artifacts["events"] - lane.Extract.LLMProfile = "scriptorium-profile" - cfg.Pipelines["example"].Artifacts["events"] = lane - - if err := cfg.Validate(); err != nil { - t.Fatalf("Validate() error = %v, want nil", err) - } -} - -func TestValidateRejectsWhitespaceOnlyExplicitLLMProfile(t *testing.T) { - cfg := validConfig() - lane := cfg.Pipelines["example"].Artifacts["events"] - lane.Extract.LLMProfile = " " - cfg.Pipelines["example"].Artifacts["events"] = lane - - err := cfg.Validate() - if err == nil || !strings.Contains(err.Error(), "llm_profile") || !strings.Contains(err.Error(), "events") { - t.Fatalf("expected llm_profile error with lane context, got %v", err) - } -} - -func TestValidateRejectsInvalidNumericFields(t *testing.T) { - tests := []struct { - name string - mutate func(Config) Config - want string - }{ - { - name: "total concurrency", - mutate: func(cfg Config) Config { - cfg.Concurrency.TotalLLM = 0 - return cfg - }, - want: "total LLM concurrency", - }, - { - name: "negative retries", - mutate: func(cfg Config) Config { - profile := cfg.Pipelines["example"] - lane := profile.Artifacts["events"] - lane.Merge.Retries = -1 - profile.Artifacts["events"] = lane - cfg.Pipelines["example"] = profile - return cfg - }, - want: "retries", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - err := tc.mutate(validConfig()).Validate() - if err == nil || !strings.Contains(err.Error(), tc.want) { - t.Fatalf("expected error containing %q, got %v", tc.want, err) - } - }) - } -} - -func TestValidateStageWorkerBoundaries(t *testing.T) { - for _, test := range []struct { - name string - workers int - wantErr bool - }{ - {name: "below minimum", workers: 0, wantErr: true}, - {name: "minimum", workers: 1}, - {name: "maximum", workers: 4}, - {name: "above maximum", workers: 5, wantErr: true}, - } { - t.Run(test.name, func(t *testing.T) { - cfg := validConfig() - cfg.Concurrency.TotalLLM = 4 - cfg.Concurrency.StageWorkers["extract"] = test.workers - err := cfg.Validate() - if test.wantErr && (err == nil || !strings.Contains(err.Error(), "stage_workers.extract")) { - t.Fatalf("Validate() error = %v, want extract worker range error", err) - } - if !test.wantErr && err != nil { - t.Fatalf("Validate() error = %v, want nil", err) - } - }) - } -} - -func TestValidateRejectsUnknownEffectiveStageWorkerKey(t *testing.T) { - cfg := validConfig() - cfg.Concurrency.StageWorkers["merge"] = 1 - err := cfg.Validate() - if err == nil || !strings.Contains(err.Error(), "stage_workers key") || !strings.Contains(err.Error(), "merge") { - t.Fatalf("Validate() error = %v, want unknown stage worker key", err) - } -} - -func TestValidateRejectsMutuallyExclusiveScriptoriumProfileSources(t *testing.T) { - cfg := validConfig() - cfg.Scriptorium.ProfileDir = "./profiles" - cfg.Scriptorium.ProfileFile = "./profiles.yml" - - err := cfg.Validate() - if err == nil || !strings.Contains(err.Error(), "mutually exclusive") { - t.Fatalf("expected Scriptorium source conflict, got %v", err) - } -} - -func TestValidateRejectsInvalidDiagnosticsRetention(t *testing.T) { - cfg := validConfig() - cfg.Diagnostics.Retention = diagnostics.RetentionMode("sometimes") - - err := cfg.Validate() - if err == nil || !strings.Contains(err.Error(), "retention") { - t.Fatalf("expected retention error, got %v", err) - } -} - -func TestValidateRejectsInvalidWorkspaceDiagnosticsRetention(t *testing.T) { - cfg := validConfig() - cfg.Workspace.Diagnostics.Retention = diagnostics.RetentionMode("sometimes") - cfg.Workspace.Diagnostics.retentionSet = true - - err := cfg.Validate() - if err == nil || !strings.Contains(err.Error(), "workspace diagnostics retention") { - t.Fatalf("expected workspace retention error, got %v", err) - } -} - -func TestValidateRejectsInvalidReferenceMaps(t *testing.T) { - tests := []struct { - name string - mutate func(Config) Config - want []string - }{ - { - name: "empty chunk slot", - mutate: func(cfg Config) Config { - profile := cfg.Pipelines["example"] - profile.Chunk.References = map[string]string{" ": "./roster.yml"} - cfg.Pipelines["example"] = profile - return cfg - }, - want: []string{"example", "chunk", "reference slot", "empty"}, - }, - { - name: "empty chunk source", - mutate: func(cfg Config) Config { - profile := cfg.Pipelines["example"] - profile.Chunk.References = map[string]string{"roster": " "} - cfg.Pipelines["example"] = profile - return cfg - }, - want: []string{"example", "chunk", "roster", "source", "empty"}, - }, - { - name: "empty extract slot", - mutate: func(cfg Config) Config { - profile := cfg.Pipelines["example"] - lane := profile.Artifacts["events"] - lane.Extract.References = map[string]string{" ": "./roster.yml"} - profile.Artifacts["events"] = lane - cfg.Pipelines["example"] = profile - return cfg - }, - want: []string{"example", "events", "extract", "reference slot", "empty"}, - }, - { - name: "empty extract source", - mutate: func(cfg Config) Config { - profile := cfg.Pipelines["example"] - lane := profile.Artifacts["events"] - lane.Extract.References = map[string]string{"roster": " "} - profile.Artifacts["events"] = lane - cfg.Pipelines["example"] = profile - return cfg - }, - want: []string{"example", "events", "extract", "roster", "source", "empty"}, - }, - { - name: "empty normalize slot", - mutate: func(cfg Config) Config { - profile := cfg.Pipelines["example"] - lane := profile.Artifacts["events"] - lane.Normalize.References = map[string]string{" ": "./roster.yml"} - profile.Artifacts["events"] = lane - cfg.Pipelines["example"] = profile - return cfg - }, - want: []string{"example", "events", "normalize", "reference slot", "empty"}, - }, - { - name: "empty normalize source", - mutate: func(cfg Config) Config { - profile := cfg.Pipelines["example"] - lane := profile.Artifacts["events"] - lane.Normalize.References = map[string]string{"roster": " "} - profile.Artifacts["events"] = lane - cfg.Pipelines["example"] = profile - return cfg - }, - want: []string{"example", "events", "normalize", "roster", "source", "empty"}, - }, - { - name: "empty pipeline slot", - mutate: func(cfg Config) Config { - profile := cfg.Pipelines["example"] - profile.References = map[string]string{" ": "./roster.yml"} - cfg.Pipelines["example"] = profile - return cfg - }, - want: []string{"example", "reference slot", "empty"}, - }, - { - name: "empty pipeline source", - mutate: func(cfg Config) Config { - profile := cfg.Pipelines["example"] - profile.References = map[string]string{"roster": " "} - cfg.Pipelines["example"] = profile - return cfg - }, - want: []string{"example", "roster", "source", "empty"}, - }, - { - name: "empty lane slot", - mutate: func(cfg Config) Config { - profile := cfg.Pipelines["example"] - lane := profile.Artifacts["events"] - lane.References = map[string]string{" ": "./roster.yml"} - profile.Artifacts["events"] = lane - cfg.Pipelines["example"] = profile - return cfg - }, - want: []string{"example", "events", "reference slot", "empty"}, - }, - { - name: "empty lane source", - mutate: func(cfg Config) Config { - profile := cfg.Pipelines["example"] - lane := profile.Artifacts["events"] - lane.References = map[string]string{"roster": " "} - profile.Artifacts["events"] = lane - cfg.Pipelines["example"] = profile - return cfg - }, - want: []string{"example", "events", "roster", "source", "empty"}, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - err := tc.mutate(validConfig()).Validate() - if err == nil { - t.Fatal("Validate() error = nil, want error") - } - for _, want := range tc.want { - if !strings.Contains(err.Error(), want) { - t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want) - } - } - }) - } -} - -func TestValidateRejectsReferencesOnUnsupportedBindings(t *testing.T) { - tests := []struct { - name string - mutate func(Config) Config - want []string - }{ - { - name: "input", - mutate: func(cfg Config) Config { - profile := cfg.Pipelines["example"] - profile.Input.References = map[string]string{"roster": "./roster.yml"} - cfg.Pipelines["example"] = profile - return cfg - }, - want: []string{"example", "input", "references", "not supported"}, - }, - { - name: "output", - mutate: func(cfg Config) Config { - profile := cfg.Pipelines["example"] - profile.Output.References = map[string]string{"roster": "./roster.yml"} - cfg.Pipelines["example"] = profile - return cfg - }, - want: []string{"example", "output", "references", "not supported"}, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - err := tc.mutate(validConfig()).Validate() - if err == nil { - t.Fatal("Validate() error = nil, want error") - } - for _, want := range tc.want { - if !strings.Contains(err.Error(), want) { - t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want) - } - } - }) - } -} - -func TestValidateRejectsEmptyIDs(t *testing.T) { - tests := []struct { - name string - mutate func(Config) Config - want string - }{ - { - name: "pipeline", - mutate: func(cfg Config) Config { - cfg.Pipelines[" "] = pipeline.PipelineProfile{} - return cfg - }, - want: "pipeline id", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - err := tc.mutate(validConfig()).Validate() - if err == nil || !strings.Contains(err.Error(), tc.want) { - t.Fatalf("expected error containing %q, got %v", tc.want, err) - } - }) - } -} - -func TestValidateRejectsIDsDuplicatedAfterTrimming(t *testing.T) { - tests := []struct { - name string - mutate func(Config) Config - want string - }{ - { - name: "pipeline", - mutate: func(cfg Config) Config { - cfg.Pipelines[" example "] = cfg.Pipelines["example"] - return cfg - }, - want: "duplicated", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - err := tc.mutate(validConfig()).Validate() - if err == nil || !strings.Contains(err.Error(), tc.want) { - t.Fatalf("expected error containing %q, got %v", tc.want, err) - } - }) - } -} - -func TestValidateRejectsConfiguredValidators(t *testing.T) { - cfg := validConfig() - profile := cfg.Pipelines["example"] - lane := profile.Artifacts["events"] - lane.Validators = []pipeline.ModuleBinding{pipeline.Binding("fake/validator")} - profile.Artifacts["events"] = lane - cfg.Pipelines["example"] = profile - - err := cfg.Validate() - if err == nil { - t.Fatal("Validate() error = nil, want configured validators error") - } - for _, want := range []string{"example", "events", "validators", "extract.validators", "merge.validators", "normalize.validators"} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want) - } - } -} - -func TestValidateAcceptsStageLocalValidatorOverrides(t *testing.T) { - cfg := validConfig() - profile := cfg.Pipelines["example"] - profile.Chunk.Validators = pipeline.ValidatorOverride{Set: true} - lane := profile.Artifacts["events"] - lane.Extract.Validators = pipeline.ValidatorOverride{ - Set: true, - Validators: []pipeline.ModuleBinding{ - pipeline.Binding("fake/validator"), - {Module: "fake/llm-validator", LLMProfile: "careful", Options: map[string]any{"threshold": 0.7}}, - }, - } - lane.Merge.Validators = pipeline.ValidatorOverride{Set: true} - profile.Artifacts["events"] = lane - cfg.Pipelines["example"] = profile - - if err := cfg.Validate(); err != nil { - t.Fatalf("Validate() error = %v, want nil", err) - } -} - -func TestValidateRejectsInvalidValidatorBindings(t *testing.T) { - tests := []struct { - name string - binding pipeline.ModuleBinding - want string - }{ - { - name: "empty module", - binding: pipeline.ModuleBinding{}, - want: "module must not be empty", - }, - { - name: "references", - binding: pipeline.ModuleBinding{Module: "fake/validator", References: map[string]string{"roster": "./roster.txt"}}, - want: "references are not supported", - }, - { - name: "nested validators", - binding: pipeline.ModuleBinding{Module: "fake/validator", Validators: pipeline.ValidatorOverride{Set: true}}, - want: "nested validators are not supported", - }, - { - name: "retries", - binding: pipeline.ModuleBinding{Module: "fake/validator", Retries: 1}, - want: "retries are not supported", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cfg := validConfig() - profile := cfg.Pipelines["example"] - lane := profile.Artifacts["events"] - lane.Extract.Validators = pipeline.ValidatorOverride{ - Set: true, - Validators: []pipeline.ModuleBinding{test.binding}, - } - profile.Artifacts["events"] = lane - cfg.Pipelines["example"] = profile - - err := cfg.Validate() - if err == nil || !strings.Contains(err.Error(), test.want) { - t.Fatalf("Validate() error = %v, want %q", err, test.want) - } - }) - } -} - -func validConfig() Config { - cfg := Default() - cfg.Pipelines["example"] = pipeline.PipelineProfile{ - Input: pipeline.Binding("fake/input"), - Artifacts: map[string]pipeline.ArtifactLaneProfile{ - "events": { - Extract: pipeline.Binding("fake/extract"), - }, - "notes": { - Extract: pipeline.Binding("fake/extract"), - }, - }, - } - return cfg -} - -func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.ModuleCatalog { - t.Helper() - specs := map[string]pipeline.ModuleSpec{ - "fake/input": { - Key: "fake/input", - Stage: pipeline.StageInput, - Provides: []string{"source"}, - }, - "generic": { - Key: "generic", - Stage: pipeline.StageChunk, - Requires: []string{"source"}, - Provides: []string{"chunks"}, - }, - "fake/extract": { - Key: "fake/extract", - Stage: pipeline.StageExtract, - Requires: []string{"chunks"}, - Provides: []string{"artifact"}, - }, - "appendorder": { - Key: "appendorder", - Stage: pipeline.StageMerge, - Requires: []string{"artifact"}, - Provides: []string{"merged"}, - }, - "noop": { - Key: "noop", - Stage: pipeline.StageNormalize, - Requires: []string{"merged"}, - Provides: []string{"normalized"}, - }, - "fake/validator": { - Key: "fake/validator", - Stage: pipeline.StageValidate, - Requires: []string{"normalized"}, - Provides: []string{"validated"}, - }, - "fake/llm-validator": { - Key: "fake/llm-validator", - Stage: pipeline.StageValidate, - Requires: []string{"normalized"}, - Provides: []string{"validated"}, - }, - "json": { - Key: "json", - Stage: pipeline.StageOutput, - Requires: []string{"normalized"}, - }, - } - for _, override := range overrides { - specs[override.Key] = override - } - for _, key := range []string{"fake/extract", "appendorder", "noop"} { - spec := specs[key] - spec.ArtifactKind = fakeArtifactKind - specs[key] = spec - } - - inputs := pipeline.NewInputAdapterRegistry() - chunkers := pipeline.NewChunkerRegistry() - extractors := pipeline.NewExtractorRegistry() - mergers := pipeline.NewMergerRegistry() - normalizers := pipeline.NewNormalizerRegistry() - validators := pipeline.NewValidatorRegistry() - outputs := pipeline.NewOutputEncoderRegistry() - - mustRegisterInput(t, inputs, specs["fake/input"]) - mustRegisterChunker(t, chunkers, specs["generic"]) - mustRegisterExtractor(t, extractors, specs["fake/extract"]) - mustRegisterMerger(t, mergers, specs["appendorder"]) - mustRegisterNormalizer(t, normalizers, specs["noop"]) - mustRegisterValidator(t, validators, specs["fake/validator"]) - mustRegisterValidator(t, validators, specs["fake/llm-validator"]) - mustRegisterOutput(t, outputs, specs["json"]) - - codecs := pipeline.NewArtifactCodecRegistry() - if err := pipeline.RegisterArtifactCodec(codecs, fakeArtifactCodec{}); err != nil { - t.Fatalf("register artifact codec: %v", err) - } - - return pipeline.ModuleCatalog{ - Inputs: inputs, - Chunkers: chunkers, - ArtifactCodecs: codecs, - Extractors: extractors, - Mergers: mergers, - Normalizers: normalizers, - Validators: validators, - ValidatorChains: pipeline.NewValidatorChainRegistry(), - Outputs: outputs, - } -} - -func mustRegisterInput(t *testing.T, registry *pipeline.InputAdapterRegistry, spec pipeline.ModuleSpec) { - t.Helper() - if err := registry.RegisterWithSpec(spec, func() (contracts.InputAdapter, error) { return nil, nil }); err != nil { - t.Fatalf("register input: %v", err) - } -} - -func mustRegisterChunker(t *testing.T, registry *pipeline.ChunkerRegistry, spec pipeline.ModuleSpec) { - t.Helper() - if err := registry.RegisterWithSpec(spec, func() (contracts.Chunker, error) { return nil, nil }); err != nil { - t.Fatalf("register chunker: %v", err) - } -} - -func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) { - t.Helper() - validateOptions := func(options map[string]any) error { - if err := pipeline.RejectUnknownOptions(options, "temperature"); err != nil { - return err - } - if value, ok := options["temperature"]; ok { - if _, ok := value.(float64); !ok { - return fmt.Errorf("temperature must be a number") - } - } - return nil - } - if err := pipeline.RegisterExtractorBuilder[fakeArtifact](registry, spec, validateOptions, func(pipeline.BuildRequest) (contracts.Extractor[fakeArtifact], error) { return nil, nil }); err != nil { - t.Fatalf("register extractor: %v", err) - } -} - -func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) { - t.Helper() - if err := pipeline.RegisterMerger[fakeArtifact](registry, spec, func() (contracts.Merger[fakeArtifact], error) { return nil, nil }); err != nil { - t.Fatalf("register merger: %v", err) - } -} - -func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) { - t.Helper() - if err := pipeline.RegisterNormalizer[fakeArtifact](registry, spec, func() (contracts.Normalizer[fakeArtifact], error) { return nil, nil }); err != nil { - t.Fatalf("register normalizer: %v", err) - } -} - -func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ModuleSpec) { - t.Helper() - executionClass := contracts.ExecutionClassDeterministic - if spec.Key == "fake/llm-validator" { - executionClass = contracts.ExecutionClassLLMBacked - } - validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: executionClass} - if err := pipeline.RegisterTypedValidator[fakeArtifact](registry, fakeArtifactKind, validatorSpec, func() (contracts.TypedValidator[fakeArtifact], error) { return nil, nil }); err != nil { - t.Fatalf("register validator: %v", err) - } -} - -const fakeArtifactKind contracts.ArtifactKind = "test/artifact" - -type fakeArtifact string - -type fakeArtifactCodec struct{} - -func (fakeArtifactCodec) Kind() contracts.ArtifactKind { return fakeArtifactKind } -func (fakeArtifactCodec) Schema() contracts.ArtifactSchema { - return contracts.ArtifactSchema{ID: "urn:notarius:test:artifact", Name: "Test artifact", Version: "1", JSONSchema: []byte(`{"type":"string"}`)} -} -func (fakeArtifactCodec) MediaType() string { return "application/json" } -func (fakeArtifactCodec) EncodeCandidate(value fakeArtifact) ([]byte, error) { - return []byte(fmt.Sprintf("%q", value)), nil -} -func (fakeArtifactCodec) Encode(value fakeArtifact) ([]byte, error) { - return []byte(fmt.Sprintf("%q", value)), nil -} -func (fakeArtifactCodec) Decode(content []byte) (fakeArtifact, error) { - if len(content) < 2 { - return "", fmt.Errorf("invalid test artifact") - } - return fakeArtifact(content[1 : len(content)-1]), nil -} - -func mustRegisterOutput(t *testing.T, registry *pipeline.OutputEncoderRegistry, spec pipeline.ModuleSpec) { - t.Helper() - if err := registry.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { return nil, nil }); err != nil { - t.Fatalf("register output: %v", err) - } -} diff --git a/internal/core/diagnostics/artifacts.go b/internal/core/diagnostics/artifacts.go deleted file mode 100644 index 74e8689..0000000 --- a/internal/core/diagnostics/artifacts.go +++ /dev/null @@ -1,15 +0,0 @@ -package diagnostics - -const ( - ArtifactInvocationMetadata = "invocation.json" - ArtifactEffectiveConfig = "effective-config.json" - ArtifactResolvedPipeline = "resolved-pipeline.json" - ArtifactResolvedReferences = "resolved-references.json" - ArtifactCheckpointEvents = "checkpoint-events.json" - ArtifactSourceDocument = "source-document.json" - ArtifactRunManifest = "run-manifest.json" - ArtifactChunkPlan = "chunk-plan.json" - ArtifactRunReport = "run-report.json" - ArtifactWarnings = "warnings.json" - ArtifactErrorLog = "error.log" -) diff --git a/internal/core/diagnostics/artifacts_test.go b/internal/core/diagnostics/artifacts_test.go deleted file mode 100644 index 8f9e4e9..0000000 --- a/internal/core/diagnostics/artifacts_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package diagnostics - -import "testing" - -func TestArtifactNamesUseExtractionOrientedNames(t *testing.T) { - names := []string{ - ArtifactInvocationMetadata, - ArtifactEffectiveConfig, - ArtifactResolvedPipeline, - ArtifactResolvedReferences, - ArtifactSourceDocument, - ArtifactRunManifest, - ArtifactChunkPlan, - ArtifactRunReport, - ArtifactWarnings, - ArtifactErrorLog, - } - - for _, name := range names { - if name == "" { - t.Fatalf("artifact name must not be empty") - } - } -} diff --git a/internal/core/diagnostics/run_dir.go b/internal/core/diagnostics/run_dir.go deleted file mode 100644 index a7dd680..0000000 --- a/internal/core/diagnostics/run_dir.go +++ /dev/null @@ -1,293 +0,0 @@ -package diagnostics - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" -) - -const ( - defaultWorkDir = "/tmp/notarius" - maxRunDirectoryCreateAttempts = 16 -) - -var utcNow = func() time.Time { - return time.Now().UTC() -} - -// RunDirectory represents a per-run diagnostics directory. -type RunDirectory struct { - path string - retention RetentionMode - createdAt time.Time -} - -type RetentionMode string - -const ( - RetentionAuto RetentionMode = "auto" - RetentionAlways RetentionMode = "always" - RetentionNever RetentionMode = "never" -) - -type RetentionDecisionInput struct { - RetentionMode RetentionMode - RunSucceeded bool - HasWarnings bool -} - -type RedactedEffectiveConfigPayload interface { - RedactedSummaryPayload() any -} - -// InvocationMetadata captures non-secret invocation details for diagnostics. -type InvocationMetadata struct { - Operation string `json:"operation"` - PipelineID string `json:"pipeline_id,omitempty"` - PipelineDigest string `json:"pipeline_digest,omitempty"` - Resume bool `json:"resume,omitempty"` - InputPath string `json:"input_path,omitempty"` - ConfigPath string `json:"config_path,omitempty"` - ConfigSource string `json:"config_source,omitempty"` - OnlyLanes []string `json:"only_lanes,omitempty"` - ChunkCacheOverride string `json:"chunk_cache_override,omitempty"` - RunID string `json:"run_id"` - StartedAt time.Time `json:"started_at"` -} - -func ShouldRetainRunDirectory(input RetentionDecisionInput) bool { - if !input.RunSucceeded { - return true - } - - switch input.RetentionMode { - case RetentionAlways: - return true - case RetentionNever: - return false - case RetentionAuto, "": - return input.HasWarnings - default: - return true - } -} - -func NewRunDirectory(workDir string, retention RetentionMode) (*RunDirectory, error) { - if strings.TrimSpace(workDir) == "" { - workDir = defaultWorkDir - } - if retention == "" { - retention = RetentionAuto - } - - if err := os.MkdirAll(workDir, 0o755); err != nil { - return nil, fmt.Errorf("create diagnostics work directory %q: %w", workDir, err) - } - - var lastRunPath string - for attempt := 0; attempt < maxRunDirectoryCreateAttempts; attempt++ { - createdAt := utcNow() - runID := fmt.Sprintf("run-%d", createdAt.UnixNano()) - runPath := filepath.Join(workDir, runID) - lastRunPath = runPath - if err := os.Mkdir(runPath, 0o755); err != nil { - if os.IsExist(err) { - continue - } - return nil, fmt.Errorf("create diagnostics run directory %q: %w", runPath, err) - } - - return &RunDirectory{ - path: runPath, - retention: retention, - createdAt: createdAt, - }, nil - } - - return nil, fmt.Errorf("create diagnostics run directory %q: exhausted unique run ID attempts", lastRunPath) -} - -func (r *RunDirectory) Path() string { - if r == nil { - return "" - } - return r.path -} - -func (r *RunDirectory) RunID() string { - if r == nil { - return "" - } - return filepath.Base(r.path) -} - -func (r *RunDirectory) WriteInvocationMetadata(metadata InvocationMetadata) error { - if r == nil { - return fmt.Errorf("run directory must not be nil") - } - if metadata.RunID == "" { - metadata.RunID = r.RunID() - } - if metadata.StartedAt.IsZero() { - metadata.StartedAt = r.createdAt - } - return r.WriteJSONArtifact(ArtifactInvocationMetadata, metadata) -} - -func (r *RunDirectory) WriteRedactedEffectiveConfig(payload RedactedEffectiveConfigPayload) error { - if payload == nil { - return fmt.Errorf("redacted effective config payload must not be nil") - } - return r.WriteJSONArtifact(ArtifactEffectiveConfig, payload.RedactedSummaryPayload()) -} - -func (r *RunDirectory) WriteResolvedPipeline(payload any) error { - return r.WriteJSONArtifact(ArtifactResolvedPipeline, payload) -} - -func (r *RunDirectory) WriteResolvedReferences(payload any) error { - return r.WriteJSONArtifact(ArtifactResolvedReferences, payload) -} - -func (r *RunDirectory) WriteCheckpointEvents(payload any) error { - return r.WriteJSONArtifact(ArtifactCheckpointEvents, payload) -} - -func (r *RunDirectory) WriteSourceDocument(payload any) error { - return r.WriteJSONArtifact(ArtifactSourceDocument, payload) -} - -func (r *RunDirectory) WriteRunManifest(manifest artifacts.RunManifest) error { - return r.WriteJSONArtifact(ArtifactRunManifest, manifest) -} - -func (r *RunDirectory) WriteChunkPlan(summary artifacts.ChunkPlanSummary) error { - return r.WriteJSONArtifact(ArtifactChunkPlan, summary) -} - -func (r *RunDirectory) WriteRunReport(payload any) error { - return r.WriteJSONArtifact(ArtifactRunReport, payload) -} - -func (r *RunDirectory) WriteWarnings(warnings []contracts.Warning) error { - return r.WriteJSONArtifact(ArtifactWarnings, warnings) -} - -func (r *RunDirectory) WriteErrorLog(errorMessage string) error { - if r == nil { - return fmt.Errorf("run directory must not be nil") - } - path, err := r.artifactPath(ArtifactErrorLog) - if err != nil { - return err - } - if err := writeFileAtomic(path, []byte(errorMessage+"\n"), 0o644); err != nil { - return fmt.Errorf("write diagnostics artifact %q: %w", ArtifactErrorLog, err) - } - return nil -} - -func (r *RunDirectory) WriteJSONArtifact(name string, payload any) error { - if r == nil { - return fmt.Errorf("run directory must not be nil") - } - path, err := r.artifactPath(name) - if err != nil { - return err - } - - data, err := json.MarshalIndent(payload, "", " ") - if err != nil { - return fmt.Errorf("marshal diagnostics artifact %q: %w", name, err) - } - data = append(data, '\n') - if err := writeFileAtomic(path, data, 0o644); err != nil { - return fmt.Errorf("write diagnostics artifact %q: %w", name, err) - } - return nil -} - -func (r *RunDirectory) ApplyRetention(input RetentionDecisionInput) error { - if r == nil { - return fmt.Errorf("run directory must not be nil") - } - decision := input - if decision.RetentionMode == "" { - decision.RetentionMode = r.retention - } - if ShouldRetainRunDirectory(decision) { - return nil - } - if err := os.RemoveAll(r.path); err != nil { - return fmt.Errorf("remove diagnostics run directory %q: %w", r.path, err) - } - return nil -} - -func (r *RunDirectory) artifactPath(name string) (string, error) { - name = strings.TrimSpace(name) - if name == "" { - return "", fmt.Errorf("diagnostics artifact name must not be empty") - } - if filepath.IsAbs(name) { - return "", fmt.Errorf("diagnostics artifact name %q must not be absolute", name) - } - if name != filepath.Base(name) || strings.Contains(name, "/") || strings.Contains(name, `\`) { - return "", fmt.Errorf("diagnostics artifact name %q must not contain path separators", name) - } - - runPath, err := filepath.Abs(r.path) - if err != nil { - return "", fmt.Errorf("resolve diagnostics run directory %q: %w", r.path, err) - } - artifactPath, err := filepath.Abs(filepath.Join(runPath, name)) - if err != nil { - return "", fmt.Errorf("resolve diagnostics artifact %q: %w", name, err) - } - if filepath.Dir(artifactPath) != runPath { - return "", fmt.Errorf("diagnostics artifact name %q resolves outside run directory", name) - } - return artifactPath, nil -} - -func writeFileAtomic(path string, data []byte, perm os.FileMode) error { - dir := filepath.Dir(path) - if err := os.MkdirAll(dir, 0o755); err != nil { - return err - } - - temp, err := os.CreateTemp(dir, "."+filepath.Base(path)+".tmp-*") - if err != nil { - return err - } - tempPath := temp.Name() - removeTemp := true - defer func() { - if removeTemp { - _ = os.Remove(tempPath) - } - }() - - if _, err := temp.Write(data); err != nil { - _ = temp.Close() - return err - } - if err := temp.Chmod(perm); err != nil { - _ = temp.Close() - return err - } - if err := temp.Close(); err != nil { - return err - } - if err := os.Rename(tempPath, path); err != nil { - return err - } - removeTemp = false - return nil -} diff --git a/internal/core/diagnostics/run_dir_test.go b/internal/core/diagnostics/run_dir_test.go deleted file mode 100644 index ad16418..0000000 --- a/internal/core/diagnostics/run_dir_test.go +++ /dev/null @@ -1,383 +0,0 @@ -package diagnostics - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "regexp" - "strings" - "testing" - "time" - - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" -) - -func TestNewRunDirectoryCreatesRunDirectoryAndRunID(t *testing.T) { - workDir := t.TempDir() - runDir, err := NewRunDirectory(workDir, RetentionAuto) - if err != nil { - t.Fatalf("NewRunDirectory: %v", err) - } - - if filepath.Dir(runDir.Path()) != workDir { - t.Fatalf("unexpected run directory parent: %q", runDir.Path()) - } - if ok := regexp.MustCompile(`^run-\d+$`).MatchString(runDir.RunID()); !ok { - t.Fatalf("unexpected run ID: %q", runDir.RunID()) - } - info, err := os.Stat(runDir.Path()) - if err != nil { - t.Fatalf("stat run directory: %v", err) - } - if !info.IsDir() { - t.Fatalf("expected run path to be a directory") - } -} - -func TestNewRunDirectoryRetriesOnRunIDCollision(t *testing.T) { - workDir := t.TempDir() - first := time.Unix(0, 100).UTC() - second := first.Add(time.Nanosecond) - if err := os.Mkdir(filepath.Join(workDir, fmt.Sprintf("run-%d", first.UnixNano())), 0o755); err != nil { - t.Fatalf("create existing run directory: %v", err) - } - restoreUTCNow := replaceUTCNow(func() func() time.Time { - calls := 0 - return func() time.Time { - calls++ - if calls == 1 { - return first - } - return second - } - }()) - t.Cleanup(restoreUTCNow) - - runDir, err := NewRunDirectory(workDir, RetentionAuto) - if err != nil { - t.Fatalf("NewRunDirectory: %v", err) - } - - wantRunID := fmt.Sprintf("run-%d", second.UnixNano()) - if runDir.RunID() != wantRunID { - t.Fatalf("RunID = %q, want %q", runDir.RunID(), wantRunID) - } - if _, err := os.Stat(runDir.Path()); err != nil { - t.Fatalf("stat run directory: %v", err) - } -} - -func TestNewRunDirectoryReturnsErrorAfterRunIDCollisionsExhausted(t *testing.T) { - workDir := t.TempDir() - collisionTime := time.Unix(0, 200).UTC() - collisionPath := filepath.Join(workDir, fmt.Sprintf("run-%d", collisionTime.UnixNano())) - if err := os.Mkdir(collisionPath, 0o755); err != nil { - t.Fatalf("create existing run directory: %v", err) - } - restoreUTCNow := replaceUTCNow(func() time.Time { - return collisionTime - }) - t.Cleanup(restoreUTCNow) - - _, err := NewRunDirectory(workDir, RetentionAuto) - if err == nil || !strings.Contains(err.Error(), "exhausted unique run ID attempts") { - t.Fatalf("expected exhausted collision error, got %v", err) - } -} - -func TestNewRunDirectoryUsesDefaultWorkDirectory(t *testing.T) { - runDir, err := NewRunDirectory("", RetentionAuto) - if err != nil { - t.Fatalf("NewRunDirectory: %v", err) - } - t.Cleanup(func() { - _ = os.RemoveAll(runDir.Path()) - _ = os.Remove(defaultWorkDir) - }) - - if filepath.Dir(runDir.Path()) != defaultWorkDir { - t.Fatalf("expected default work directory %q, got %q", defaultWorkDir, filepath.Dir(runDir.Path())) - } -} - -func TestWriteJSONArtifactWritesIndentedNewlineTerminatedJSON(t *testing.T) { - runDir := newTestRunDirectory(t) - - if err := runDir.WriteJSONArtifact("artifact.json", map[string]any{"value": "ok"}); err != nil { - t.Fatalf("WriteJSONArtifact: %v", err) - } - - data := readArtifact(t, runDir, "artifact.json") - if !strings.HasSuffix(string(data), "\n") { - t.Fatalf("expected trailing newline, got %q", data) - } - if !strings.Contains(string(data), "\n \"value\": \"ok\"\n") { - t.Fatalf("expected indented JSON, got %s", data) - } -} - -func TestWriteJSONArtifactLeavesNoTemporaryFiles(t *testing.T) { - runDir := newTestRunDirectory(t) - - if err := runDir.WriteJSONArtifact("artifact.json", map[string]any{"value": "ok"}); err != nil { - t.Fatalf("WriteJSONArtifact: %v", err) - } - - entries, err := os.ReadDir(runDir.Path()) - if err != nil { - t.Fatalf("read run directory: %v", err) - } - for _, entry := range entries { - if strings.Contains(entry.Name(), ".tmp-") { - t.Fatalf("temporary diagnostics file remains after success: %s", entry.Name()) - } - } -} - -func TestWriteInvocationMetadataFillsMissingRunIDAndStartTime(t *testing.T) { - runDir := newTestRunDirectory(t) - - if err := runDir.WriteInvocationMetadata(InvocationMetadata{Operation: "validate"}); err != nil { - t.Fatalf("WriteInvocationMetadata: %v", err) - } - - var got InvocationMetadata - if err := json.Unmarshal(readArtifact(t, runDir, ArtifactInvocationMetadata), &got); err != nil { - t.Fatalf("unmarshal invocation metadata: %v", err) - } - if got.RunID != runDir.RunID() { - t.Fatalf("unexpected run ID: got %q want %q", got.RunID, runDir.RunID()) - } - if got.StartedAt.IsZero() { - t.Fatalf("expected started_at to be filled") - } - if got.Operation != "validate" { - t.Fatalf("unexpected operation: %q", got.Operation) - } -} - -func TestWriteInvocationMetadataPreservesProvidedRunIDAndStartTime(t *testing.T) { - runDir := newTestRunDirectory(t) - startedAt := time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC) - - if err := runDir.WriteInvocationMetadata(InvocationMetadata{ - Operation: "validate", - RunID: "provided", - StartedAt: startedAt, - }); err != nil { - t.Fatalf("WriteInvocationMetadata: %v", err) - } - - var got InvocationMetadata - if err := json.Unmarshal(readArtifact(t, runDir, ArtifactInvocationMetadata), &got); err != nil { - t.Fatalf("unmarshal invocation metadata: %v", err) - } - if got.RunID != "provided" { - t.Fatalf("unexpected run ID: %q", got.RunID) - } - if !got.StartedAt.Equal(startedAt) { - t.Fatalf("unexpected started_at: %s", got.StartedAt) - } -} - -func TestWriteTypedArtifacts(t *testing.T) { - runDir := newTestRunDirectory(t) - - if err := runDir.WriteRedactedEffectiveConfig(fakeRedactedEffectiveConfig{payload: map[string]any{"redacted": true}}); err != nil { - t.Fatalf("WriteRedactedEffectiveConfig: %v", err) - } - if err := runDir.WriteResolvedPipeline(map[string]any{"pipeline": "test"}); err != nil { - t.Fatalf("WriteResolvedPipeline: %v", err) - } - if err := runDir.WriteResolvedReferences([]artifacts.ReferenceProvenance{{LaneID: "events", SlotName: "roster"}}); err != nil { - t.Fatalf("WriteResolvedReferences: %v", err) - } - if err := runDir.WriteSourceDocument(map[string]any{"source_id": "source-1"}); err != nil { - t.Fatalf("WriteSourceDocument: %v", err) - } - if err := runDir.WriteRunManifest(artifacts.RunManifest{RunID: "run-1"}); err != nil { - t.Fatalf("WriteRunManifest: %v", err) - } - if err := runDir.WriteChunkPlan(artifacts.ChunkPlanSummary{Mode: "auto", RequestedModule: "chunk/test", LookupStatus: "invalid", LookupReason: "stored chunk plan failed validation", ValidationStatus: "not_run", PublicationStatus: "not_published"}); err != nil { - t.Fatalf("WriteChunkPlan: %v", err) - } - if err := runDir.WriteRunReport(map[string]any{"ok": true}); err != nil { - t.Fatalf("WriteRunReport: %v", err) - } - if err := runDir.WriteWarnings([]contracts.Warning{{ReasonCode: "test", Message: "warning"}}); err != nil { - t.Fatalf("WriteWarnings: %v", err) - } - - for _, name := range []string{ - ArtifactEffectiveConfig, - ArtifactResolvedPipeline, - ArtifactResolvedReferences, - ArtifactSourceDocument, - ArtifactRunManifest, - ArtifactChunkPlan, - ArtifactRunReport, - ArtifactWarnings, - } { - if _, err := os.Stat(filepath.Join(runDir.Path(), name)); err != nil { - t.Fatalf("expected artifact %q: %v", name, err) - } - } - chunkSummary, err := os.ReadFile(filepath.Join(runDir.Path(), ArtifactChunkPlan)) - if err != nil { - t.Fatal(err) - } - for _, forbidden := range []string{"units", "annotations", "source content", "prompt", "response", "raw invalid"} { - if strings.Contains(string(chunkSummary), forbidden) { - t.Fatalf("chunk plan summary leaked %q: %s", forbidden, chunkSummary) - } - } -} - -func TestWriteRedactedEffectiveConfigWritesPayloadReturnedByProvider(t *testing.T) { - runDir := newTestRunDirectory(t) - - if err := runDir.WriteRedactedEffectiveConfig(fakeRedactedEffectiveConfig{ - payload: map[string]any{ - "api_key": "[REDACTED]", - "model": "test-model", - }, - }); err != nil { - t.Fatalf("WriteRedactedEffectiveConfig: %v", err) - } - - data := string(readArtifact(t, runDir, ArtifactEffectiveConfig)) - if !strings.Contains(data, `"api_key": "[REDACTED]"`) || !strings.Contains(data, `"model": "test-model"`) { - t.Fatalf("unexpected effective config artifact: %s", data) - } -} - -func TestWriteErrorLogWritesPlainTextWithTrailingNewline(t *testing.T) { - runDir := newTestRunDirectory(t) - - if err := runDir.WriteErrorLog("something failed"); err != nil { - t.Fatalf("WriteErrorLog: %v", err) - } - - if got := string(readArtifact(t, runDir, ArtifactErrorLog)); got != "something failed\n" { - t.Fatalf("unexpected error log: %q", got) - } -} - -func TestArtifactPathRejectsUnsafeNames(t *testing.T) { - runDir := newTestRunDirectory(t) - - tests := []string{ - "", - " ", - "/absolute.json", - "nested/artifact.json", - `nested\artifact.json`, - "../escape.json", - } - - for _, name := range tests { - t.Run(name, func(t *testing.T) { - if err := runDir.WriteJSONArtifact(name, map[string]any{}); err == nil { - t.Fatalf("expected unsafe artifact name %q to be rejected", name) - } - }) - } -} - -func TestShouldRetainRunDirectoryDecisions(t *testing.T) { - tests := []struct { - name string - input RetentionDecisionInput - want bool - }{ - {name: "failed auto retained", input: RetentionDecisionInput{RetentionMode: RetentionAuto, RunSucceeded: false}, want: true}, - {name: "failed always retained", input: RetentionDecisionInput{RetentionMode: RetentionAlways, RunSucceeded: false}, want: true}, - {name: "failed never retained", input: RetentionDecisionInput{RetentionMode: RetentionNever, RunSucceeded: false}, want: true}, - {name: "successful always retained", input: RetentionDecisionInput{RetentionMode: RetentionAlways, RunSucceeded: true}, want: true}, - {name: "successful never removed", input: RetentionDecisionInput{RetentionMode: RetentionNever, RunSucceeded: true}, want: false}, - {name: "successful auto without warnings removed", input: RetentionDecisionInput{RetentionMode: RetentionAuto, RunSucceeded: true}, want: false}, - {name: "successful auto with warnings retained", input: RetentionDecisionInput{RetentionMode: RetentionAuto, RunSucceeded: true, HasWarnings: true}, want: true}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - if got := ShouldRetainRunDirectory(tc.input); got != tc.want { - t.Fatalf("ShouldRetainRunDirectory() = %v, want %v", got, tc.want) - } - }) - } -} - -func TestApplyRetentionRemovesOnlyRunDirectory(t *testing.T) { - workDir := t.TempDir() - runDir, err := NewRunDirectory(workDir, RetentionNever) - if err != nil { - t.Fatalf("NewRunDirectory: %v", err) - } - siblingPath := filepath.Join(workDir, "sibling") - if err := os.WriteFile(siblingPath, []byte("keep"), 0o644); err != nil { - t.Fatalf("write sibling: %v", err) - } - - if err := runDir.ApplyRetention(RetentionDecisionInput{RunSucceeded: true}); err != nil { - t.Fatalf("ApplyRetention: %v", err) - } - - if _, err := os.Stat(runDir.Path()); !os.IsNotExist(err) { - t.Fatalf("expected run directory removed, stat err=%v", err) - } - if _, err := os.Stat(workDir); err != nil { - t.Fatalf("expected work directory retained: %v", err) - } - if _, err := os.Stat(siblingPath); err != nil { - t.Fatalf("expected sibling retained: %v", err) - } -} - -func TestApplyRetentionKeepsRetainedRunDirectory(t *testing.T) { - runDir := newTestRunDirectory(t) - - if err := runDir.ApplyRetention(RetentionDecisionInput{RunSucceeded: true, HasWarnings: true}); err != nil { - t.Fatalf("ApplyRetention: %v", err) - } - - if _, err := os.Stat(runDir.Path()); err != nil { - t.Fatalf("expected run directory retained: %v", err) - } -} - -func newTestRunDirectory(t *testing.T) *RunDirectory { - t.Helper() - runDir, err := NewRunDirectory(t.TempDir(), RetentionAuto) - if err != nil { - t.Fatalf("NewRunDirectory: %v", err) - } - return runDir -} - -func replaceUTCNow(replacement func() time.Time) func() { - original := utcNow - utcNow = replacement - return func() { - utcNow = original - } -} - -func readArtifact(t *testing.T, runDir *RunDirectory, name string) []byte { - t.Helper() - data, err := os.ReadFile(filepath.Join(runDir.Path(), name)) - if err != nil { - t.Fatalf("read artifact %q: %v", name, err) - } - return data -} - -type fakeRedactedEffectiveConfig struct { - payload any -} - -func (f fakeRedactedEffectiveConfig) RedactedSummaryPayload() any { - return f.payload -} diff --git a/internal/core/workspace/files.go b/internal/core/workspace/files.go deleted file mode 100644 index ac4cb53..0000000 --- a/internal/core/workspace/files.go +++ /dev/null @@ -1,114 +0,0 @@ -package workspace - -import ( - "encoding/json" - "fmt" - "os" - "path" - "path/filepath" - "strings" -) - -func SafePath(root string, name string) (string, error) { - root = strings.TrimSpace(root) - if root == "" { - return "", fmt.Errorf("workspace root must not be empty") - } - name = strings.TrimSpace(name) - if name == "" { - return "", fmt.Errorf("workspace artifact name must not be empty") - } - if strings.Contains(name, `\`) { - return "", fmt.Errorf("workspace artifact name %q must use slash-separated relative paths", name) - } - if path.IsAbs(name) || filepath.IsAbs(name) { - return "", fmt.Errorf("workspace artifact name %q must be relative", name) - } - if name == "." || strings.Contains(name, "..") { - return "", fmt.Errorf("workspace artifact name %q must not contain ..", name) - } - cleaned := path.Clean(name) - if cleaned != name { - return "", fmt.Errorf("workspace artifact name %q must be clean", name) - } - - absRoot, err := filepath.Abs(root) - if err != nil { - return "", fmt.Errorf("resolve workspace root %q: %w", root, err) - } - target, err := filepath.Abs(filepath.Join(absRoot, filepath.FromSlash(cleaned))) - if err != nil { - return "", fmt.Errorf("resolve workspace artifact %q: %w", name, err) - } - rel, err := filepath.Rel(absRoot, target) - if err != nil { - return "", fmt.Errorf("resolve workspace artifact %q: %w", name, err) - } - if rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { - return "", fmt.Errorf("workspace artifact name %q resolves outside workspace root", name) - } - return target, nil -} - -func WriteJSON(root string, name string, payload any) error { - target, err := SafePath(root, name) - if err != nil { - return err - } - data, err := json.MarshalIndent(payload, "", " ") - if err != nil { - return fmt.Errorf("marshal workspace artifact %q: %w", name, err) - } - data = append(data, '\n') - if err := writeFileAtomic(target, data, 0o644); err != nil { - return fmt.Errorf("write workspace artifact %q: %w", name, err) - } - return nil -} - -func WriteBytes(root string, name string, data []byte) error { - target, err := SafePath(root, name) - if err != nil { - return err - } - if err := writeFileAtomic(target, data, 0o644); err != nil { - return fmt.Errorf("write workspace artifact %q: %w", name, err) - } - return nil -} - -func writeFileAtomic(target string, data []byte, perm os.FileMode) error { - dir := filepath.Dir(target) - if err := os.MkdirAll(dir, 0o755); err != nil { - return err - } - - temp, err := os.CreateTemp(dir, "."+filepath.Base(target)+".tmp-*") - if err != nil { - return err - } - tempPath := temp.Name() - removeTemp := true - defer func() { - if removeTemp { - _ = os.Remove(tempPath) - } - }() - - if _, err := temp.Write(data); err != nil { - _ = temp.Close() - return err - } - if err := temp.Chmod(perm); err != nil { - _ = temp.Close() - return err - } - if err := temp.Close(); err != nil { - return err - } - if err := os.Rename(tempPath, target); err != nil { - return err - } - removeTemp = false - return nil -} diff --git a/internal/core/workspace/files_test.go b/internal/core/workspace/files_test.go deleted file mode 100644 index d09c91e..0000000 --- a/internal/core/workspace/files_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package workspace - -import ( - "os" - "path/filepath" - "strings" - "testing" -) - -func TestSafePathAcceptsCleanRelativePaths(t *testing.T) { - root := t.TempDir() - - got, err := SafePath(root, "source/manifest.json") - if err != nil { - t.Fatalf("SafePath: %v", err) - } - - want := filepath.Join(root, "source", "manifest.json") - if got != want { - t.Fatalf("SafePath = %q, want %q", got, want) - } -} - -func TestSafePathRejectsUnsafeNames(t *testing.T) { - root := t.TempDir() - tests := []struct { - name string - path string - want string - }{ - {name: "empty", path: " ", want: "empty"}, - {name: "absolute", path: filepath.Join(root, "artifact.json"), want: "relative"}, - {name: "parent segment", path: "../artifact.json", want: ".."}, - {name: "embedded parent", path: "source/../artifact.json", want: ".."}, - {name: "backslash", path: `source\artifact.json`, want: "slash-separated"}, - {name: "unclean", path: "source//artifact.json", want: "clean"}, - {name: "dot", path: ".", want: ".."}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got, err := SafePath(root, tc.path) - if err == nil { - t.Fatalf("SafePath returned %q, want error", got) - } - if !strings.Contains(err.Error(), tc.want) { - t.Fatalf("SafePath error = %v, want containing %q", err, tc.want) - } - }) - } -} - -func TestSafePathRejectsEmptyRoot(t *testing.T) { - got, err := SafePath(" ", "artifact.json") - if err == nil { - t.Fatalf("SafePath returned %q, want error", got) - } - if !strings.Contains(err.Error(), "root") { - t.Fatalf("SafePath error = %v, want root error", err) - } -} - -func TestSafePathDoesNotPermitEscapingRoot(t *testing.T) { - root := t.TempDir() - for _, name := range []string{ - "..", - "../outside.json", - "nested/../../outside.json", - } { - t.Run(name, func(t *testing.T) { - got, err := SafePath(root, name) - if err == nil { - t.Fatalf("SafePath returned %q, want error", got) - } - }) - } -} - -func TestWriteJSONWritesIndentedAtomicArtifact(t *testing.T) { - root := t.TempDir() - - err := WriteJSON(root, "source/manifest.json", map[string]any{ - "status": "succeeded", - "count": 2, - }) - if err != nil { - t.Fatalf("WriteJSON: %v", err) - } - - got := string(readFile(t, filepath.Join(root, "source", "manifest.json"))) - if !strings.HasSuffix(got, "\n") { - t.Fatalf("expected trailing newline, got %q", got) - } - if !strings.Contains(got, `"status": "succeeded"`) || !strings.Contains(got, `"count": 2`) { - t.Fatalf("unexpected JSON: %s", got) - } - assertNoTempFiles(t, filepath.Join(root, "source")) -} - -func TestWriteBytesWritesNestedArtifact(t *testing.T) { - root := t.TempDir() - - if err := WriteBytes(root, "chunk/chunks.json", []byte("payload")); err != nil { - t.Fatalf("WriteBytes: %v", err) - } - - got := string(readFile(t, filepath.Join(root, "chunk", "chunks.json"))) - if got != "payload" { - t.Fatalf("bytes = %q, want payload", got) - } - assertNoTempFiles(t, filepath.Join(root, "chunk")) -} - -func TestWritersRejectUnsafePaths(t *testing.T) { - root := t.TempDir() - - if err := WriteBytes(root, "../outside.json", []byte("payload")); err == nil { - t.Fatalf("WriteBytes accepted unsafe path") - } - if err := WriteJSON(root, `debug\trace.json`, map[string]string{"x": "y"}); err == nil { - t.Fatalf("WriteJSON accepted unsafe path") - } - if _, err := os.Stat(filepath.Join(root, "..", "outside.json")); !os.IsNotExist(err) { - t.Fatalf("outside path stat err = %v, want not exist", err) - } -} - -func readFile(t *testing.T, path string) []byte { - t.Helper() - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read %q: %v", path, err) - } - return data -} - -func assertNoTempFiles(t *testing.T, dir string) { - t.Helper() - entries, err := os.ReadDir(dir) - if err != nil { - t.Fatalf("read dir %q: %v", dir, err) - } - for _, entry := range entries { - if strings.Contains(entry.Name(), ".tmp-") { - t.Fatalf("temporary file was not cleaned up: %s", entry.Name()) - } - } -} diff --git a/internal/core/workspace/settings.go b/internal/core/workspace/settings.go deleted file mode 100644 index d311487..0000000 --- a/internal/core/workspace/settings.go +++ /dev/null @@ -1,54 +0,0 @@ -package workspace - -import ( - "fmt" - "path/filepath" - "strings" - - "gitea.maximumdirect.net/eric/notarius/internal/core/config" -) - -type Settings struct { - RootDir string - DiagnosticsRoot string - CheckpointsRoot string - DebugRoot string - DiagnosticsEnabled bool - ResumeEnabled bool - DebugEnabled bool -} - -func FromConfig(cfg config.Config) Settings { - _ = cfg - return Settings{} -} - -func (s Settings) DiagnosticsRunDirectory(runID string) (string, error) { - if !s.DiagnosticsEnabled || strings.TrimSpace(s.DiagnosticsRoot) == "" { - return "", nil - } - return safeSingleDirectory(s.DiagnosticsRoot, runID, "diagnostics run ID") -} - -func (s Settings) DebugRunDirectory(runID string) (string, error) { - if !s.DebugEnabled || strings.TrimSpace(s.DebugRoot) == "" { - return "", nil - } - return safeSingleDirectory(s.DebugRoot, runID, "debug run ID") -} - -func cleanPath(path string) string { - path = strings.TrimSpace(path) - if path == "" { - return "" - } - return filepath.Clean(path) -} - -func safeSingleDirectory(root string, name string, label string) (string, error) { - name = strings.TrimSpace(name) - if strings.Contains(name, "/") || strings.Contains(name, `\`) { - return "", fmt.Errorf("%s %q must be a single directory name", label, name) - } - return SafePath(root, name) -} diff --git a/internal/core/workspace/settings_test.go b/internal/core/workspace/settings_test.go deleted file mode 100644 index 3141106..0000000 --- a/internal/core/workspace/settings_test.go +++ /dev/null @@ -1,120 +0,0 @@ -//go:build legacy - -package workspace - -import ( - "path/filepath" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/core/config" -) - -func TestFromConfigBuildsWorkspaceRoots(t *testing.T) { - cfg := config.Default() - cfg.Workspace.Directory = "/var/lib/notarius" - cfg.Workspace.Resume.Enabled = true - cfg.Workspace.Debug.Enabled = true - cfg.RecomputeEffectiveDiagnostics() - - settings := FromConfig(cfg) - - if settings.RootDir != "/var/lib/notarius" { - t.Fatalf("RootDir = %q, want /var/lib/notarius", settings.RootDir) - } - if settings.DiagnosticsRoot != "/var/lib/notarius/diagnostics" || !settings.DiagnosticsEnabled { - t.Fatalf("diagnostics settings = %+v, want workspace diagnostics root enabled", settings) - } - if settings.CheckpointsRoot != "/var/lib/notarius/checkpoints" || !settings.ResumeEnabled { - t.Fatalf("checkpoint settings = %+v, want workspace checkpoints root enabled", settings) - } - if settings.DebugRoot != "/var/lib/notarius/debug" || !settings.DebugEnabled { - t.Fatalf("debug settings = %+v, want workspace debug root enabled", settings) - } -} - -func TestFromConfigKeepsLegacyDiagnosticsRootWithoutWorkspaceRoot(t *testing.T) { - cfg := config.Default() - cfg.Diagnostics.WorkDir = "/tmp/notarius-legacy" - cfg.Workspace.Resume.Enabled = true - cfg.Workspace.Debug.Enabled = true - - settings := FromConfig(cfg) - - if settings.RootDir != "" { - t.Fatalf("RootDir = %q, want empty", settings.RootDir) - } - if settings.DiagnosticsRoot != "/tmp/notarius-legacy" || !settings.DiagnosticsEnabled { - t.Fatalf("diagnostics settings = %+v, want legacy diagnostics root enabled", settings) - } - if settings.CheckpointsRoot != "" || settings.ResumeEnabled { - t.Fatalf("checkpoint settings = %+v, want disabled empty root", settings) - } - if settings.DebugRoot != "" || settings.DebugEnabled { - t.Fatalf("debug settings = %+v, want disabled empty root", settings) - } -} - -func TestPathConstructors(t *testing.T) { - root := t.TempDir() - settings := Settings{ - RootDir: root, - DiagnosticsRoot: filepath.Join(root, "diagnostics"), - CheckpointsRoot: filepath.Join(root, "checkpoints"), - DebugRoot: filepath.Join(root, "debug"), - DiagnosticsEnabled: true, - ResumeEnabled: true, - DebugEnabled: true, - } - - diagnosticsDir, err := settings.DiagnosticsRunDirectory("run-123") - if err != nil { - t.Fatalf("DiagnosticsRunDirectory: %v", err) - } - if diagnosticsDir != filepath.Join(root, "diagnostics", "run-123") { - t.Fatalf("diagnostics dir = %q", diagnosticsDir) - } - - debugDir, err := settings.DebugRunDirectory("run-456") - if err != nil { - t.Fatalf("DebugRunDirectory: %v", err) - } - if debugDir != filepath.Join(root, "debug", "run-456") { - t.Fatalf("debug dir = %q", debugDir) - } -} - -func TestDisabledPathConstructorsReturnEmptyPaths(t *testing.T) { - settings := Settings{} - - for name, call := range map[string]func() (string, error){ - "diagnostics": func() (string, error) { return settings.DiagnosticsRunDirectory("run-1") }, - "debug": func() (string, error) { return settings.DebugRunDirectory("run-1") }, - } { - t.Run(name, func(t *testing.T) { - got, err := call() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if got != "" { - t.Fatalf("path = %q, want empty", got) - } - }) - } -} - -func TestRunDirectoryConstructorsRejectNestedNames(t *testing.T) { - root := t.TempDir() - settings := Settings{ - DiagnosticsRoot: filepath.Join(root, "diagnostics"), - DebugRoot: filepath.Join(root, "debug"), - DiagnosticsEnabled: true, - DebugEnabled: true, - } - - if got, err := settings.DiagnosticsRunDirectory("run-1/nested"); err == nil { - t.Fatalf("DiagnosticsRunDirectory returned %q, want error", got) - } - if got, err := settings.DebugRunDirectory("run-1/nested"); err == nil { - t.Fatalf("DebugRunDirectory returned %q, want error", got) - } -} diff --git a/internal/framework/checkpoint/manifest.go b/internal/framework/checkpoint/manifest.go index 20b0325..654f392 100644 --- a/internal/framework/checkpoint/manifest.go +++ b/internal/framework/checkpoint/manifest.go @@ -3,6 +3,8 @@ package checkpoint import "time" const ( + // These names and values are frozen checkpoint wire-compatibility + // identifiers. They intentionally retain the former terminology. WorkspaceSchemaVersion = "notarius.workspace.v2" WorkspaceSchemaVersionV1 = "notarius.workspace.v1" )