package cli import ( "bytes" "encoding/json" "errors" "os" "path/filepath" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject" ) func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) { outputRoot := filepath.Join(t.TempDir(), "output") var stdout, stderr strings.Builder code := RunWithOptions([]string{ "run", "dnd-session", "--config", repositoryPath("examples", "dnd-minimal.config.yml"), "--input", repositoryPath("examples", "seriatim-minimal-transcript.json"), "--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, "--json", }, &stdout, &stderr, productionRunOptions(t, &productionFakeLLMClient{})) if code != 0 || stderr.Len() != 0 { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } receipt := decodeRunResultDocument(t, stdout.String()) if got := receipt["schema_version"]; got != "notarius.run-result.v1" { t.Fatalf("schema_version = %q", got) } if got := receipt["run_id"]; got != productionRunID { t.Fatalf("run_id = %q", got) } if got := receipt["pipeline_id"]; got != "dnd-session" { t.Fatalf("pipeline_id = %q", got) } if got := receipt["index_file"]; got != "index.json" { t.Fatalf("index_file = %q", got) } if got := receipt["normalized_output_count"]; got != float64(1) { t.Fatalf("normalized_output_count = %v", got) } if got := receipt["rejected_output_count"]; got != float64(0) { t.Fatalf("rejected_output_count = %v", got) } if got := receipt["warning_count"]; got != float64(0) { t.Fatalf("warning_count = %v", got) } if got := receipt["validation_status"]; got != "approved" { t.Fatalf("validation_status = %q", got) } outputDirectory, ok := receipt["output_directory"].(string) if !ok || !filepath.IsAbs(outputDirectory) || outputDirectory != filepath.Join(outputRoot, productionRunID) { t.Fatalf("output_directory = %q", receipt["output_directory"]) } indexFile := receipt["index_file"].(string) assertFile(t, filepath.Join(outputDirectory, indexFile)) } func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) { roots := newStateTestRoots(t) harness := newStateTestHarness() harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}} var stdout, stderr bytes.Buffer code := RunWithOptions([]string{ "run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug", "--json", }, &stdout, &stderr, harness.options()) if code != 0 || !strings.Contains(stderr.String(), "1 warning(s)") { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } receipt := decodeRunResultDocument(t, stdout.String()) if got := receipt["warning_count"]; got != float64(1) { t.Fatalf("warning_count = %v", got) } debugDirectory, ok := receipt["debug_directory"].(string) if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) { t.Fatalf("debug_directory = %q", receipt["debug_directory"]) } if strings.Contains(stdout.String(), "complete:") || strings.Contains(stdout.String(), "debug=") { t.Fatalf("machine stdout contains human reporting: %q", stdout.String()) } } func TestRunResultReportsSuccessfulRejection(t *testing.T) { roots := newStateTestRoots(t) configBytes, err := os.ReadFile(roots.config) if err != nil { t.Fatal(err) } configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - generic/always_reject\n")) if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil { t.Fatal(err) } harness := newStateTestHarness() opts := harness.options() if err := alwaysreject.RegisterTyped[stateTestArtifact](opts.Registries.Validators, stateTestArtifactKind); err != nil { t.Fatal(err) } opts.Catalog = catalogFromRegistries(opts.Registries) var stdout, stderr bytes.Buffer code := RunWithOptions([]string{ "run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--json", }, &stdout, &stderr, opts) if code != 0 || stderr.Len() != 0 { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } receipt := decodeRunResultDocument(t, stdout.String()) if got := receipt["normalized_output_count"]; got != float64(0) { t.Fatalf("normalized_output_count = %v", got) } if got := receipt["rejected_output_count"]; got != float64(1) { t.Fatalf("rejected_output_count = %v", got) } if got := receipt["validation_status"]; got != "rejected" { t.Fatalf("validation_status = %q", got) } } func TestRunResultIsAbsentForSyntaxAndRuntimeFailures(t *testing.T) { t.Run("syntax", func(t *testing.T) { var stdout, stderr bytes.Buffer code := RunWithOptions([]string{"run", "sample", "--json"}, &stdout, &stderr, newStateTestHarness().options()) if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } }) t.Run("runtime", func(t *testing.T) { roots := newStateTestRoots(t) harness := newStateTestHarness() harness.extractErr = errors.New("injected extraction failure") var stdout, stderr bytes.Buffer code := RunWithOptions([]string{ "run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--json", }, &stdout, &stderr, harness.options()) if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } }) } func TestRunResultDeliveryFailureRetainsPublishedBundles(t *testing.T) { roots := newStateTestRoots(t) writerErr := errors.New("result writer sentinel") stdout := &resultDeliveryWriter{err: writerErr} var stderr bytes.Buffer code := RunWithOptions([]string{ "run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug", "--json", }, stdout, &stderr, newStateTestHarness().options()) if code != 1 || !strings.Contains(stderr.String(), "write run result") || strings.Contains(stderr.String(), writerErr.Error()) { t.Fatalf("code=%d stderr=%q", code, stderr.String()) } if stdout.accepted.Len() != 0 { t.Fatalf("accepted stdout = %q", stdout.accepted.String()) } assertStateTestOutput(t, roots.output) debugBundle := onlyChildDir(t, roots.debug) report := readStateTestRunReport(t, debugBundle) if !report.Succeeded { t.Fatalf("debug report = %#v, want successful persisted run", report) } if strings.Contains(readAllFiles(t, debugBundle), writerErr.Error()) { t.Fatalf("debug bundle contains result writer error") } } func decodeRunResultDocument(t *testing.T, stdout string) map[string]any { t.Helper() if strings.Count(stdout, "\n") != 1 { t.Fatalf("stdout = %q, want one JSON document", stdout) } var receipt map[string]any if err := json.Unmarshal([]byte(stdout), &receipt); err != nil { t.Fatalf("decode run result: %v; stdout=%q", err, stdout) } return receipt } type resultDeliveryWriter struct { err error accepted bytes.Buffer } func (w *resultDeliveryWriter) Write(content []byte) (int, error) { if w.err != nil { return 0, w.err } return w.accepted.Write(content) }