200 lines
7.3 KiB
Go
200 lines
7.3 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/always_reject"
|
|
)
|
|
|
|
func TestMaintainedMinimalInvocationEmitsRunResult(t *testing.T) {
|
|
outputRoot := filepath.Join(t.TempDir(), "output")
|
|
var stdout, stderr strings.Builder
|
|
code := RunWithOptions([]string{
|
|
"run", "dnd-session",
|
|
"--config", repositoryPath("examples", "dnd-minimal.config.yml"),
|
|
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
|
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot, "--json",
|
|
}, &stdout, &stderr, productionRunOptions(t, &productionFakeLLMClient{}))
|
|
if code != 0 || stderr.Len() != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
receipt := decodeRunResultDocument(t, stdout.String())
|
|
if got := receipt["schema_version"]; got != "notarius.run-result.v1" {
|
|
t.Fatalf("schema_version = %q", got)
|
|
}
|
|
if got := receipt["run_id"]; got != productionRunID {
|
|
t.Fatalf("run_id = %q", got)
|
|
}
|
|
if got := receipt["pipeline_id"]; got != "dnd-session" {
|
|
t.Fatalf("pipeline_id = %q", got)
|
|
}
|
|
if got := receipt["index_file"]; got != "index.json" {
|
|
t.Fatalf("index_file = %q", got)
|
|
}
|
|
if got := receipt["normalized_output_count"]; got != float64(1) {
|
|
t.Fatalf("normalized_output_count = %v", got)
|
|
}
|
|
if got := receipt["rejected_output_count"]; got != float64(0) {
|
|
t.Fatalf("rejected_output_count = %v", got)
|
|
}
|
|
if got := receipt["warning_count"]; got != float64(0) {
|
|
t.Fatalf("warning_count = %v", got)
|
|
}
|
|
if got := receipt["validation_status"]; got != "approved" {
|
|
t.Fatalf("validation_status = %q", got)
|
|
}
|
|
|
|
outputDirectory, ok := receipt["output_directory"].(string)
|
|
if !ok || !filepath.IsAbs(outputDirectory) || outputDirectory != filepath.Join(outputRoot, productionRunID) {
|
|
t.Fatalf("output_directory = %q", receipt["output_directory"])
|
|
}
|
|
indexFile := receipt["index_file"].(string)
|
|
assertFile(t, filepath.Join(outputDirectory, indexFile))
|
|
}
|
|
|
|
func TestRunResultReportsWarningsAndDebugBundle(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
harness := newStateTestHarness()
|
|
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}}
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{
|
|
"run", "sample", "--config", roots.config, "--input", roots.input,
|
|
"--chunk_cache", "bypass", "--debug", "--json",
|
|
}, &stdout, &stderr, harness.options())
|
|
if code != 0 || !strings.Contains(stderr.String(), "1 warning(s)") {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
receipt := decodeRunResultDocument(t, stdout.String())
|
|
if got := receipt["warning_count"]; got != float64(1) {
|
|
t.Fatalf("warning_count = %v", got)
|
|
}
|
|
debugDirectory, ok := receipt["debug_directory"].(string)
|
|
if !ok || !filepath.IsAbs(debugDirectory) || debugDirectory != onlyChildDir(t, roots.debug) {
|
|
t.Fatalf("debug_directory = %q", receipt["debug_directory"])
|
|
}
|
|
if strings.Contains(stdout.String(), "complete:") || strings.Contains(stdout.String(), "debug=") {
|
|
t.Fatalf("machine stdout contains human reporting: %q", stdout.String())
|
|
}
|
|
}
|
|
|
|
func TestRunResultReportsSuccessfulRejection(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
configBytes, err := os.ReadFile(roots.config)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
configBytes = []byte(replaceRequiredOnce(t, string(configBytes), " normalize: test/normalize\n", " normalize:\n module: test/normalize\n validators:\n - generic/always_reject\n"))
|
|
if err := os.WriteFile(roots.config, configBytes, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
harness := newStateTestHarness()
|
|
opts := harness.options()
|
|
if err := alwaysreject.RegisterTyped[stateTestArtifact](opts.Registries.Validators, stateTestArtifactKind); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
opts.Catalog = catalogFromRegistries(opts.Registries)
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{
|
|
"run", "sample", "--config", roots.config, "--input", roots.input,
|
|
"--chunk_cache", "bypass", "--json",
|
|
}, &stdout, &stderr, opts)
|
|
if code != 0 || stderr.Len() != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
receipt := decodeRunResultDocument(t, stdout.String())
|
|
if got := receipt["normalized_output_count"]; got != float64(0) {
|
|
t.Fatalf("normalized_output_count = %v", got)
|
|
}
|
|
if got := receipt["rejected_output_count"]; got != float64(1) {
|
|
t.Fatalf("rejected_output_count = %v", got)
|
|
}
|
|
if got := receipt["validation_status"]; got != "rejected" {
|
|
t.Fatalf("validation_status = %q", got)
|
|
}
|
|
}
|
|
|
|
func TestRunResultIsAbsentForSyntaxAndRuntimeFailures(t *testing.T) {
|
|
t.Run("syntax", func(t *testing.T) {
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"run", "sample", "--json"}, &stdout, &stderr, newStateTestHarness().options())
|
|
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
|
|
t.Run("runtime", func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
harness := newStateTestHarness()
|
|
harness.extractErr = errors.New("injected extraction failure")
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{
|
|
"run", "sample", "--config", roots.config, "--input", roots.input,
|
|
"--chunk_cache", "bypass", "--json",
|
|
}, &stdout, &stderr, harness.options())
|
|
if code != 1 || stdout.Len() != 0 || stderr.Len() == 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestRunResultDeliveryFailureRetainsPublishedBundles(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
writerErr := errors.New("result writer sentinel")
|
|
stdout := &resultDeliveryWriter{err: writerErr}
|
|
var stderr bytes.Buffer
|
|
code := RunWithOptions([]string{
|
|
"run", "sample", "--config", roots.config, "--input", roots.input,
|
|
"--chunk_cache", "bypass", "--debug", "--json",
|
|
}, stdout, &stderr, newStateTestHarness().options())
|
|
if code != 1 || !strings.Contains(stderr.String(), "write run result") || strings.Contains(stderr.String(), writerErr.Error()) {
|
|
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
|
}
|
|
if stdout.accepted.Len() != 0 {
|
|
t.Fatalf("accepted stdout = %q", stdout.accepted.String())
|
|
}
|
|
assertStateTestOutput(t, roots.output)
|
|
debugBundle := onlyChildDir(t, roots.debug)
|
|
report := readStateTestRunReport(t, debugBundle)
|
|
if !report.Succeeded {
|
|
t.Fatalf("debug report = %#v, want successful persisted run", report)
|
|
}
|
|
if strings.Contains(readAllFiles(t, debugBundle), writerErr.Error()) {
|
|
t.Fatalf("debug bundle contains result writer error")
|
|
}
|
|
}
|
|
|
|
func decodeRunResultDocument(t *testing.T, stdout string) map[string]any {
|
|
t.Helper()
|
|
if strings.Count(stdout, "\n") != 1 {
|
|
t.Fatalf("stdout = %q, want one JSON document", stdout)
|
|
}
|
|
var receipt map[string]any
|
|
if err := json.Unmarshal([]byte(stdout), &receipt); err != nil {
|
|
t.Fatalf("decode run result: %v; stdout=%q", err, stdout)
|
|
}
|
|
return receipt
|
|
}
|
|
|
|
type resultDeliveryWriter struct {
|
|
err error
|
|
accepted bytes.Buffer
|
|
}
|
|
|
|
func (w *resultDeliveryWriter) Write(content []byte) (int, error) {
|
|
if w.err != nil {
|
|
return 0, w.err
|
|
}
|
|
return w.accepted.Write(content)
|
|
}
|