Test JSON run result behavior
This commit is contained in:
199
internal/cli/run_result_command_test.go
Normal file
199
internal/cli/run_result_command_test.go
Normal file
@@ -0,0 +1,199 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject"
|
||||
)
|
||||
|
||||
func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
|
||||
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||
var stdout, stderr strings.Builder
|
||||
code := RunWithOptions([]string{
|
||||
"run", "dnd-session",
|
||||
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
|
||||
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
||||
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, "--json",
|
||||
}, &stdout, &stderr, productionRunOptions(t, &productionFakeLLMClient{}))
|
||||
if code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
receipt := decodeRunResultDocument(t, stdout.String())
|
||||
if got := receipt["schema_version"]; got != "notarius.run-result.v1" {
|
||||
t.Fatalf("schema_version = %q", got)
|
||||
}
|
||||
if got := receipt["run_id"]; got != productionRunID {
|
||||
t.Fatalf("run_id = %q", got)
|
||||
}
|
||||
if got := receipt["pipeline_id"]; got != "dnd-session" {
|
||||
t.Fatalf("pipeline_id = %q", got)
|
||||
}
|
||||
if got := receipt["index_file"]; got != "index.json" {
|
||||
t.Fatalf("index_file = %q", got)
|
||||
}
|
||||
if got := receipt["normalized_output_count"]; got != float64(1) {
|
||||
t.Fatalf("normalized_output_count = %v", got)
|
||||
}
|
||||
if got := receipt["rejected_output_count"]; got != float64(0) {
|
||||
t.Fatalf("rejected_output_count = %v", got)
|
||||
}
|
||||
if got := receipt["warning_count"]; got != float64(0) {
|
||||
t.Fatalf("warning_count = %v", got)
|
||||
}
|
||||
if got := receipt["validation_status"]; got != "approved" {
|
||||
t.Fatalf("validation_status = %q", got)
|
||||
}
|
||||
|
||||
outputDirectory, ok := receipt["output_directory"].(string)
|
||||
if !ok || !filepath.IsAbs(outputDirectory) || outputDirectory != filepath.Join(outputRoot, productionRunID) {
|
||||
t.Fatalf("output_directory = %q", receipt["output_directory"])
|
||||
}
|
||||
indexFile := receipt["index_file"].(string)
|
||||
assertFile(t, filepath.Join(outputDirectory, indexFile))
|
||||
}
|
||||
|
||||
func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--chunk_cache", "bypass", "--debug", "--json",
|
||||
}, &stdout, &stderr, harness.options())
|
||||
if code != 0 || !strings.Contains(stderr.String(), "1 warning(s)") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
|
||||
receipt := decodeRunResultDocument(t, stdout.String())
|
||||
if got := receipt["warning_count"]; got != float64(1) {
|
||||
t.Fatalf("warning_count = %v", got)
|
||||
}
|
||||
debugDirectory, ok := receipt["debug_directory"].(string)
|
||||
if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) {
|
||||
t.Fatalf("debug_directory = %q", receipt["debug_directory"])
|
||||
}
|
||||
if strings.Contains(stdout.String(), "complete:") || strings.Contains(stdout.String(), "debug=") {
|
||||
t.Fatalf("machine stdout contains human reporting: %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultReportsSuccessfulRejection(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
configBytes, err := os.ReadFile(roots.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - generic/always_reject\n"))
|
||||
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
harness := newStateTestHarness()
|
||||
opts := harness.options()
|
||||
if err := alwaysreject.RegisterTyped[stateTestArtifact](opts.Registries.Validators, stateTestArtifactKind); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.Catalog = catalogFromRegistries(opts.Registries)
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--chunk_cache", "bypass", "--json",
|
||||
}, &stdout, &stderr, opts)
|
||||
if code != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
receipt := decodeRunResultDocument(t, stdout.String())
|
||||
if got := receipt["normalized_output_count"]; got != float64(0) {
|
||||
t.Fatalf("normalized_output_count = %v", got)
|
||||
}
|
||||
if got := receipt["rejected_output_count"]; got != float64(1) {
|
||||
t.Fatalf("rejected_output_count = %v", got)
|
||||
}
|
||||
if got := receipt["validation_status"]; got != "rejected" {
|
||||
t.Fatalf("validation_status = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultIsAbsentForSyntaxAndRuntimeFailures(t *testing.T) {
|
||||
t.Run("syntax", func(t *testing.T) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{"run", "sample", "--json"}, &stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("runtime", func(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
harness := newStateTestHarness()
|
||||
harness.extractErr = errors.New("injected extraction failure")
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--chunk_cache", "bypass", "--json",
|
||||
}, &stdout, &stderr, harness.options())
|
||||
if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunResultDeliveryFailureRetainsPublishedBundles(t *testing.T) {
|
||||
roots := newStateTestRoots(t)
|
||||
writerErr := errors.New("result writer sentinel")
|
||||
stdout := &resultDeliveryWriter{err: writerErr}
|
||||
var stderr bytes.Buffer
|
||||
code := RunWithOptions([]string{
|
||||
"run", "sample", "--config", roots.config, "--input", roots.input,
|
||||
"--chunk_cache", "bypass", "--debug", "--json",
|
||||
}, stdout, &stderr, newStateTestHarness().options())
|
||||
if code != 1 || !strings.Contains(stderr.String(), "write run result") || strings.Contains(stderr.String(), writerErr.Error()) {
|
||||
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
||||
}
|
||||
if stdout.accepted.Len() != 0 {
|
||||
t.Fatalf("accepted stdout = %q", stdout.accepted.String())
|
||||
}
|
||||
assertStateTestOutput(t, roots.output)
|
||||
debugBundle := onlyChildDir(t, roots.debug)
|
||||
report := readStateTestRunReport(t, debugBundle)
|
||||
if !report.Succeeded {
|
||||
t.Fatalf("debug report = %#v, want successful persisted run", report)
|
||||
}
|
||||
if strings.Contains(readAllFiles(t, debugBundle), writerErr.Error()) {
|
||||
t.Fatalf("debug bundle contains result writer error")
|
||||
}
|
||||
}
|
||||
|
||||
func decodeRunResultDocument(t *testing.T, stdout string) map[string]any {
|
||||
t.Helper()
|
||||
if strings.Count(stdout, "\n") != 1 {
|
||||
t.Fatalf("stdout = %q, want one JSON document", stdout)
|
||||
}
|
||||
var receipt map[string]any
|
||||
if err := json.Unmarshal([]byte(stdout), &receipt); err != nil {
|
||||
t.Fatalf("decode run result: %v; stdout=%q", err, stdout)
|
||||
}
|
||||
return receipt
|
||||
}
|
||||
|
||||
type resultDeliveryWriter struct {
|
||||
err error
|
||||
accepted bytes.Buffer
|
||||
}
|
||||
|
||||
func (w *resultDeliveryWriter) Write(content []byte) (int, error) {
|
||||
if w.err != nil {
|
||||
return 0, w.err
|
||||
}
|
||||
return w.accepted.Write(content)
|
||||
}
|
||||
Reference in New Issue
Block a user