From 0d947549fb2bd85506bf1e3fbb250baa164be08d Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 26 Jul 2026 17:17:35 +0000 Subject: [PATCH] Test JSON run result behavior --- internal/cli/run_result_command_test.go | 199 ++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 internal/cli/run_result_command_test.go diff --git a/internal/cli/run_result_command_test.go b/internal/cli/run_result_command_test.go new file mode 100644 index 0000000..5f2e814 --- /dev/null +++ b/internal/cli/run_result_command_test.go @@ -0,0 +1,199 @@ +package cli + +import ( + "bytes" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject" +) + +func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) { + outputRoot := filepath.Join(t.TempDir(), "output") + var stdout, stderr strings.Builder + code := RunWithOptions([]string{ + "run", "dnd-session", + "--config", repositoryPath("examples", "dnd-minimal.config.yml"), + "--input", repositoryPath("examples", "seriatim-minimal-transcript.json"), + "--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, "--json", + }, &stdout, &stderr, productionRunOptions(t, &productionFakeLLMClient{})) + if code != 0 || stderr.Len() != 0 { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + + receipt := decodeRunResultDocument(t, stdout.String()) + if got := receipt["schema_version"]; got != "notarius.run-result.v1" { + t.Fatalf("schema_version = %q", got) + } + if got := receipt["run_id"]; got != productionRunID { + t.Fatalf("run_id = %q", got) + } + if got := receipt["pipeline_id"]; got != "dnd-session" { + t.Fatalf("pipeline_id = %q", got) + } + if got := receipt["index_file"]; got != "index.json" { + t.Fatalf("index_file = %q", got) + } + if got := receipt["normalized_output_count"]; got != float64(1) { + t.Fatalf("normalized_output_count = %v", got) + } + if got := receipt["rejected_output_count"]; got != float64(0) { + t.Fatalf("rejected_output_count = %v", got) + } + if got := receipt["warning_count"]; got != float64(0) { + t.Fatalf("warning_count = %v", got) + } + if got := receipt["validation_status"]; got != "approved" { + t.Fatalf("validation_status = %q", got) + } + + outputDirectory, ok := receipt["output_directory"].(string) + if !ok || !filepath.IsAbs(outputDirectory) || outputDirectory != filepath.Join(outputRoot, productionRunID) { + t.Fatalf("output_directory = %q", receipt["output_directory"]) + } + indexFile := receipt["index_file"].(string) + assertFile(t, filepath.Join(outputDirectory, indexFile)) +} + +func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) { + roots := newStateTestRoots(t) + harness := newStateTestHarness() + harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}} + var stdout, stderr bytes.Buffer + code := RunWithOptions([]string{ + "run", "sample", "--config", roots.config, "--input", roots.input, + "--chunk_cache", "bypass", "--debug", "--json", + }, &stdout, &stderr, harness.options()) + if code != 0 || !strings.Contains(stderr.String(), "1 warning(s)") { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + + receipt := decodeRunResultDocument(t, stdout.String()) + if got := receipt["warning_count"]; got != float64(1) { + t.Fatalf("warning_count = %v", got) + } + debugDirectory, ok := receipt["debug_directory"].(string) + if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) { + t.Fatalf("debug_directory = %q", receipt["debug_directory"]) + } + if strings.Contains(stdout.String(), "complete:") || strings.Contains(stdout.String(), "debug=") { + t.Fatalf("machine stdout contains human reporting: %q", stdout.String()) + } +} + +func TestRunResultReportsSuccessfulRejection(t *testing.T) { + roots := newStateTestRoots(t) + configBytes, err := os.ReadFile(roots.config) + if err != nil { + t.Fatal(err) + } + configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - generic/always_reject\n")) + if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil { + t.Fatal(err) + } + + harness := newStateTestHarness() + opts := harness.options() + if err := alwaysreject.RegisterTyped[stateTestArtifact](opts.Registries.Validators, stateTestArtifactKind); err != nil { + t.Fatal(err) + } + opts.Catalog = catalogFromRegistries(opts.Registries) + + var stdout, stderr bytes.Buffer + code := RunWithOptions([]string{ + "run", "sample", "--config", roots.config, "--input", roots.input, + "--chunk_cache", "bypass", "--json", + }, &stdout, &stderr, opts) + if code != 0 || stderr.Len() != 0 { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + receipt := decodeRunResultDocument(t, stdout.String()) + if got := receipt["normalized_output_count"]; got != float64(0) { + t.Fatalf("normalized_output_count = %v", got) + } + if got := receipt["rejected_output_count"]; got != float64(1) { + t.Fatalf("rejected_output_count = %v", got) + } + if got := receipt["validation_status"]; got != "rejected" { + t.Fatalf("validation_status = %q", got) + } +} + +func TestRunResultIsAbsentForSyntaxAndRuntimeFailures(t *testing.T) { + t.Run("syntax", func(t *testing.T) { + var stdout, stderr bytes.Buffer + code := RunWithOptions([]string{"run", "sample", "--json"}, &stdout, &stderr, newStateTestHarness().options()) + if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + }) + + t.Run("runtime", func(t *testing.T) { + roots := newStateTestRoots(t) + harness := newStateTestHarness() + harness.extractErr = errors.New("injected extraction failure") + var stdout, stderr bytes.Buffer + code := RunWithOptions([]string{ + "run", "sample", "--config", roots.config, "--input", roots.input, + "--chunk_cache", "bypass", "--json", + }, &stdout, &stderr, harness.options()) + if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + }) +} + +func TestRunResultDeliveryFailureRetainsPublishedBundles(t *testing.T) { + roots := newStateTestRoots(t) + writerErr := errors.New("result writer sentinel") + stdout := &resultDeliveryWriter{err: writerErr} + var stderr bytes.Buffer + code := RunWithOptions([]string{ + "run", "sample", "--config", roots.config, "--input", roots.input, + "--chunk_cache", "bypass", "--debug", "--json", + }, stdout, &stderr, newStateTestHarness().options()) + if code != 1 || !strings.Contains(stderr.String(), "write run result") || strings.Contains(stderr.String(), writerErr.Error()) { + t.Fatalf("code=%d stderr=%q", code, stderr.String()) + } + if stdout.accepted.Len() != 0 { + t.Fatalf("accepted stdout = %q", stdout.accepted.String()) + } + assertStateTestOutput(t, roots.output) + debugBundle := onlyChildDir(t, roots.debug) + report := readStateTestRunReport(t, debugBundle) + if !report.Succeeded { + t.Fatalf("debug report = %#v, want successful persisted run", report) + } + if strings.Contains(readAllFiles(t, debugBundle), writerErr.Error()) { + t.Fatalf("debug bundle contains result writer error") + } +} + +func decodeRunResultDocument(t *testing.T, stdout string) map[string]any { + t.Helper() + if strings.Count(stdout, "\n") != 1 { + t.Fatalf("stdout = %q, want one JSON document", stdout) + } + var receipt map[string]any + if err := json.Unmarshal([]byte(stdout), &receipt); err != nil { + t.Fatalf("decode run result: %v; stdout=%q", err, stdout) + } + return receipt +} + +type resultDeliveryWriter struct { + err error + accepted bytes.Buffer +} + +func (w *resultDeliveryWriter) Write(content []byte) (int, error) { + if w.err != nil { + return 0, w.err + } + return w.accepted.Write(content) +}