983 lines
40 KiB
Go
983 lines
40 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
)
|
|
|
|
const stateTestDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
|
|
|
func TestRunStateSurfaceMatrix(t *testing.T) {
|
|
for _, debug := range []bool{false, true} {
|
|
for _, resume := range []bool{false, true} {
|
|
for _, mode := range []string{"auto", "bypass", "refresh"} {
|
|
name := fmt.Sprintf("debug=%t/resume=%t/cache=%s", debug, resume, mode)
|
|
t.Run(name, func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
harness := newStateTestHarness()
|
|
opts := harness.options()
|
|
var storeRoots []string
|
|
opts.ChunkPlanStoreFactory = func(root string) (pipeline.ChunkPlanStore, error) {
|
|
storeRoots = append(storeRoots, root)
|
|
return chunkplan.NewFilesystemStore(root)
|
|
}
|
|
result := runStateTest(t, roots, opts, debug, resume, mode)
|
|
if result.code != 0 {
|
|
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
|
}
|
|
assertStateTestOutput(t, roots.output)
|
|
if mode == "bypass" {
|
|
assertAbsent(t, roots.plans)
|
|
if len(storeRoots) != 0 {
|
|
t.Fatalf("chunk plan store roots = %v, want none", storeRoots)
|
|
}
|
|
} else {
|
|
assertFile(t, filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json"))
|
|
if len(storeRoots) != 1 || storeRoots[0] != roots.plans {
|
|
t.Fatalf("chunk plan store roots = %v, want [%q]", storeRoots, roots.plans)
|
|
}
|
|
}
|
|
assertAnyFile(t, roots.checkpoints)
|
|
assertRestrictedTree(t, roots.checkpoints)
|
|
if debug {
|
|
bundle := onlyChildDir(t, roots.debug)
|
|
assertFile(t, filepath.Join(bundle, "summary", "invocation.json"))
|
|
assertAnyFile(t, filepath.Join(bundle, "trace"))
|
|
assertRestrictedTree(t, roots.debug)
|
|
} else {
|
|
assertAbsent(t, roots.debug)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunKeepsStateRootsIndependentAndReusesSelectedCheckpointRoot(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
harness := newStateTestHarness()
|
|
first := runStateTest(t, roots, harness.options(), true, false, "auto")
|
|
if first.code != 0 {
|
|
t.Fatalf("first run code=%d stderr=%q", first.code, first.stderr)
|
|
}
|
|
planPath := filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:"), "plan.json")
|
|
initialPlan, err := os.ReadFile(planPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
firstBundle := onlyChildDir(t, roots.debug)
|
|
|
|
second := runStateTest(t, roots, harness.options(), false, false, "auto")
|
|
if second.code != 0 {
|
|
t.Fatalf("second run code=%d stderr=%q", second.code, second.stderr)
|
|
}
|
|
if harness.chunkCalls != 1 {
|
|
t.Fatalf("chunk calls after debug toggle = %d, want 1", harness.chunkCalls)
|
|
}
|
|
if harness.extractCalls != 2 {
|
|
t.Fatalf("extract calls after two recording-only runs = %d, want 2", harness.extractCalls)
|
|
}
|
|
if got, err := os.ReadFile(planPath); err != nil || !bytes.Equal(got, initialPlan) {
|
|
t.Fatalf("chunk plan changed after debug toggle: %v", err)
|
|
}
|
|
if _, err := os.Stat(firstBundle); err != nil {
|
|
t.Fatalf("initial debug bundle was removed: %v", err)
|
|
}
|
|
|
|
checkpointRoot := roots.checkpoints
|
|
extractCallsBeforeResume := harness.extractCalls
|
|
seed := runStateTest(t, roots, harness.options(), false, true, "auto")
|
|
if seed.code != 0 {
|
|
t.Fatalf("checkpoint seed code=%d stderr=%q", seed.code, seed.stderr)
|
|
}
|
|
if harness.extractCalls != extractCallsBeforeResume {
|
|
t.Fatalf("extract calls after reusing recording-only checkpoint = %d, want %d", harness.extractCalls, extractCallsBeforeResume)
|
|
}
|
|
extractCalls := harness.extractCalls
|
|
checkpointFiles := readTree(t, checkpointRoot)
|
|
reused := runStateTest(t, roots, harness.options(), false, true, "auto")
|
|
if reused.code != 0 {
|
|
t.Fatalf("checkpoint reuse code=%d stderr=%q", reused.code, reused.stderr)
|
|
}
|
|
if harness.extractCalls != extractCalls {
|
|
t.Fatalf("extract calls after checkpoint reuse = %d, want %d", harness.extractCalls, extractCalls)
|
|
}
|
|
if got := readTree(t, checkpointRoot); !sameFiles(got, checkpointFiles) {
|
|
t.Fatal("reused checkpoint was rewritten")
|
|
}
|
|
}
|
|
|
|
func TestRunRecomputesOnlyAfterExplicitChunkPlanRemoval(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
harness := newStateTestHarness()
|
|
first := runStateTest(t, roots, harness.options(), true, false, "auto")
|
|
if first.code != 0 {
|
|
t.Fatalf("first run code=%d stderr=%q", first.code, first.stderr)
|
|
}
|
|
firstOutput := onlyChildDir(t, roots.output)
|
|
firstBundle := onlyChildDir(t, roots.debug)
|
|
entry := filepath.Join(roots.plans, strings.TrimPrefix(stateTestDigest, "sha256:"))
|
|
if err := os.RemoveAll(entry); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
second := runStateTest(t, roots, harness.options(), false, false, "auto")
|
|
if second.code != 0 {
|
|
t.Fatalf("second run code=%d stderr=%q", second.code, second.stderr)
|
|
}
|
|
if harness.chunkCalls != 2 {
|
|
t.Fatalf("chunk calls = %d, want 2 after removing exact cache entry", harness.chunkCalls)
|
|
}
|
|
assertFile(t, filepath.Join(firstOutput, "result.json"))
|
|
assertFile(t, filepath.Join(firstBundle, "summary", "run-report.json"))
|
|
}
|
|
|
|
func TestRunRetainsDebugBundlesAcrossFailures(t *testing.T) {
|
|
t.Run("configuration failure precedes allocation", func(t *testing.T) {
|
|
root := filepath.Join(t.TempDir(), "debug")
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"run", "sample", "--config", filepath.Join(t.TempDir(), "missing.yml"), "--input", "missing", "--debug", "--debug-dir", root}, &stdout, &stderr, newStateTestHarness().options())
|
|
if code != 1 || !strings.Contains(stderr.String(), "config file") {
|
|
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
|
}
|
|
assertAbsent(t, root)
|
|
})
|
|
|
|
for _, failure := range []struct {
|
|
name string
|
|
expected string
|
|
setup func(*testing.T, stateTestRoots, *stateTestHarness) Options
|
|
}{
|
|
{"resolution", "pipeline \"missing\"", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options { return h.options() }},
|
|
{"pipeline", "synthetic extraction failure", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options {
|
|
h.extractErr = errors.New("synthetic extraction failure")
|
|
return h.options()
|
|
}},
|
|
{"output", "create output parent", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options {
|
|
if err := os.WriteFile(roots.output, []byte("not a directory"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return h.options()
|
|
}},
|
|
{"summary", "write debug invocation metadata", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options {
|
|
opts := h.options()
|
|
opts.DebugRecorderFactory = func(traceRoot string) (pipeline.DebugRecorder, error) {
|
|
if err := os.RemoveAll(filepath.Join(filepath.Dir(traceRoot), "summary")); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := os.WriteFile(filepath.Join(filepath.Dir(traceRoot), "summary"), []byte("blocked"), 0o600); err != nil {
|
|
return nil, err
|
|
}
|
|
return frameworkdebug.NewFilesystemRecorder(traceRoot)
|
|
}
|
|
return opts
|
|
}},
|
|
{"trace", "trace unavailable", func(t *testing.T, roots stateTestRoots, h *stateTestHarness) Options {
|
|
opts := h.options()
|
|
opts.DebugRecorderFactory = func(string) (pipeline.DebugRecorder, error) { return failingDebugRecorder{}, nil }
|
|
return opts
|
|
}},
|
|
} {
|
|
t.Run(failure.name, func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
harness := newStateTestHarness()
|
|
opts := failure.setup(t, roots, harness)
|
|
failureStderr := ""
|
|
if failure.name == "resolution" {
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"run", "missing", "--config", roots.config, "--input", roots.input, "--debug"}, &stdout, &stderr, opts)
|
|
if code != 1 {
|
|
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
|
}
|
|
failureStderr = stderr.String()
|
|
} else {
|
|
result := runStateTest(t, roots, opts, true, false, "bypass")
|
|
if result.code != 1 {
|
|
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
|
}
|
|
failureStderr = result.stderr
|
|
}
|
|
if !strings.Contains(failureStderr, failure.expected) || !strings.Contains(failureStderr, "debug=") {
|
|
t.Fatalf("stderr=%q, want %q and debug path", failureStderr, failure.expected)
|
|
}
|
|
bundle := onlyChildDir(t, roots.debug)
|
|
if !strings.Contains(readAllFiles(t, bundle), "synthetic") && failure.name == "pipeline" {
|
|
t.Fatal("pipeline failure was not retained in debug bundle")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunDebugArtifactsRedactSecretsButRetainApplicationData(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
t.Setenv("STATE_TEST_UNRELATED_ENV", "HOST_ONLY_SENTINEL")
|
|
if err := os.WriteFile(filepath.Join(filepath.Dir(roots.input), "unrelated.txt"), []byte("HOST_ONLY_FILE_SENTINEL"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
harness := newStateTestHarness()
|
|
result := runStateTest(t, roots, harness.options(), true, false, "bypass")
|
|
if result.code != 0 {
|
|
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
|
}
|
|
bundle := onlyChildDir(t, roots.debug)
|
|
summary := readAllFiles(t, filepath.Join(bundle, "summary"))
|
|
trace := readAllFiles(t, filepath.Join(bundle, "trace"))
|
|
for _, forbidden := range []string{"sk-secretvalue", "Bearer secretvalue", "HOST_ONLY_SENTINEL", "HOST_ONLY_FILE_SENTINEL"} {
|
|
if strings.Contains(summary, forbidden) || strings.Contains(trace, forbidden) {
|
|
t.Fatalf("debug bundle contains %q", forbidden)
|
|
}
|
|
}
|
|
if strings.Contains(summary, "application content") {
|
|
t.Fatal("summary contains raw application input")
|
|
}
|
|
if !strings.Contains(trace, "application content") {
|
|
t.Fatal("trace does not retain expected application input")
|
|
}
|
|
}
|
|
|
|
func TestRunRedactsSensitiveModuleOptionsFromConfigAndPipelineSummaries(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
data, err := os.ReadFile(roots.config)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
configText := replaceRequiredOnce(t, string(data), " input: test/input\n", ` input:
|
|
module: test/input
|
|
options:
|
|
api_key: CONFIG_SUMMARY_SECRET_SENTINEL
|
|
safe: SAFE_OPTION_SENTINEL
|
|
nested:
|
|
- - password: PIPELINE_SUMMARY_SECRET_SENTINEL
|
|
neighbor: SAFE_NESTED_OPTION_SENTINEL
|
|
`)
|
|
if err := os.WriteFile(roots.config, []byte(configText), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
result := runStateTest(t, roots, newStateTestHarness().options(), true, false, "bypass")
|
|
if result.code != 0 {
|
|
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
|
}
|
|
summaryRoot := filepath.Join(onlyChildDir(t, roots.debug), "summary")
|
|
for _, name := range []string{"effective-config.json", "resolved-pipeline.json"} {
|
|
contents, err := os.ReadFile(filepath.Join(summaryRoot, name))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
text := string(contents)
|
|
for _, secret := range []string{"CONFIG_SUMMARY_SECRET_SENTINEL", "PIPELINE_SUMMARY_SECRET_SENTINEL"} {
|
|
if strings.Contains(text, secret) {
|
|
t.Fatalf("%s contains %q: %s", name, secret, text)
|
|
}
|
|
}
|
|
for _, retained := range []string{"[REDACTED]", "SAFE_OPTION_SENTINEL", "SAFE_NESTED_OPTION_SENTINEL"} {
|
|
if !strings.Contains(text, retained) {
|
|
t.Fatalf("%s does not contain %q: %s", name, retained, text)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunUsesOneInjectedIdentityForDebugOutputAndManifest(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
harness := newStateTestHarness()
|
|
opts := harness.options()
|
|
const runID = "run-1000000000-11111111111111111111111111111111"
|
|
opts.RunIDGenerator = func(time.Time) (string, error) { return runID, nil }
|
|
|
|
result := runStateTest(t, roots, opts, true, false, "bypass")
|
|
if result.code != 0 {
|
|
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
|
}
|
|
outputPath := filepath.Join(roots.output, runID)
|
|
debugPath := filepath.Join(roots.debug, runID)
|
|
assertFile(t, filepath.Join(outputPath, "result.json"))
|
|
assertFile(t, filepath.Join(debugPath, "summary", "run-manifest.json"))
|
|
if !strings.Contains(result.stdout, "output="+outputPath) || !strings.Contains(result.stdout, "debug="+debugPath) {
|
|
t.Fatalf("stdout=%q, want shared run identity", result.stdout)
|
|
}
|
|
data, err := os.ReadFile(filepath.Join(debugPath, "summary", "run-manifest.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var manifest artifacts.RunManifest
|
|
if err := json.Unmarshal(data, &manifest); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if manifest.RunID != runID {
|
|
t.Fatalf("manifest run ID = %q, want %q", manifest.RunID, runID)
|
|
}
|
|
wantStartedAt := time.Unix(1, 0).UTC()
|
|
if manifest.StartedAt == nil || !manifest.StartedAt.Equal(wantStartedAt) {
|
|
t.Fatalf("manifest started at = %v, want %v", manifest.StartedAt, wantStartedAt)
|
|
}
|
|
var invocation debugbundle.Invocation
|
|
readStateTestSummaryJSON(t, debugPath, "invocation.json", &invocation)
|
|
if invocation.RunID != runID || !invocation.StartedAt.Equal(wantStartedAt) {
|
|
t.Fatalf("debug invocation identity = %#v, want run %q at %v", invocation, runID, wantStartedAt)
|
|
}
|
|
report := readStateTestRunReport(t, debugPath)
|
|
if !report.Succeeded || report.RunID != runID || report.PipelineID != "sample" || report.OutputPath != outputPath || report.DebugPath != debugPath || report.OutputCount != 1 || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != "approved" {
|
|
t.Fatalf("success report = %#v", report)
|
|
}
|
|
if !strings.Contains(result.stdout, "outputs=1 rejected=0") {
|
|
t.Fatalf("stdout=%q, want report counts", result.stdout)
|
|
}
|
|
}
|
|
|
|
func TestRunWritesTerminalArtifactsForResolutionPipelineAndOutputFailures(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
pipelineID string
|
|
wantError string
|
|
wantOutputs int
|
|
wantValidation string
|
|
configureFailure func(*testing.T, stateTestRoots, *stateTestHarness)
|
|
}{
|
|
{name: "resolution", pipelineID: "missing", wantError: `pipeline "missing"`},
|
|
{name: "pipeline", pipelineID: "sample", wantError: "synthetic extraction failure", wantValidation: "failed", configureFailure: func(_ *testing.T, _ stateTestRoots, h *stateTestHarness) {
|
|
h.extractErr = errors.New("synthetic extraction failure")
|
|
}},
|
|
{name: "output", pipelineID: "sample", wantError: "create output parent", wantOutputs: 1, wantValidation: "approved", configureFailure: func(t *testing.T, roots stateTestRoots, _ *stateTestHarness) {
|
|
if err := os.WriteFile(roots.output, []byte("not a directory"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
harness := newStateTestHarness()
|
|
if tc.configureFailure != nil {
|
|
tc.configureFailure(t, roots, harness)
|
|
}
|
|
opts := harness.options()
|
|
var stdout, stderr bytes.Buffer
|
|
args := []string{"run", tc.pipelineID, "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}
|
|
code := RunWithOptions(args, &stdout, &stderr, opts)
|
|
if code != 1 || !strings.Contains(stderr.String(), tc.wantError) {
|
|
t.Fatalf("code=%d stderr=%q", code, stderr.String())
|
|
}
|
|
bundlePath := onlyChildDir(t, roots.debug)
|
|
runID := filepath.Base(bundlePath)
|
|
report := readStateTestRunReport(t, bundlePath)
|
|
if report.Succeeded || report.RunID != runID || report.PipelineID != tc.pipelineID || report.OutputPath != filepath.Join(roots.output, runID) || report.DebugPath != bundlePath || report.OutputCount != tc.wantOutputs || report.RejectedCount != 0 || report.WarningCount != 0 || report.ValidationStatus != tc.wantValidation {
|
|
t.Fatalf("failure report = %#v", report)
|
|
}
|
|
errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log"))
|
|
if err != nil || !strings.Contains(string(errorLog), tc.wantError) {
|
|
t.Fatalf("error log = %q, %v", errorLog, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunRetainsPartialPipelineOutcomeInFailureSummary(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
harness := newStateTestHarness()
|
|
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "partial-warning", Message: "warning retained before failure"}}
|
|
harness.extractErr = errors.New("synthetic partial pipeline failure")
|
|
|
|
result := runStateTest(t, roots, harness.options(), true, true, "bypass")
|
|
if result.code != 1 {
|
|
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
|
}
|
|
bundlePath := onlyChildDir(t, roots.debug)
|
|
report := readStateTestRunReport(t, bundlePath)
|
|
if report.Succeeded || report.OutputCount != 0 || report.RejectedCount != 0 || report.WarningCount != 1 || report.ValidationStatus != "failed" {
|
|
t.Fatalf("partial failure report = %#v", report)
|
|
}
|
|
|
|
var manifest artifacts.RunManifest
|
|
readStateTestSummaryJSON(t, bundlePath, "run-manifest.json", &manifest)
|
|
if manifest.RunID != report.RunID || manifest.PipelineID != "sample" || manifest.ValidationStatus != "failed" {
|
|
t.Fatalf("partial manifest = %#v", manifest)
|
|
}
|
|
var warnings []contracts.Warning
|
|
readStateTestSummaryJSON(t, bundlePath, "warnings.json", &warnings)
|
|
if len(warnings) != 1 || warnings[0].ReasonCode != "partial-warning" {
|
|
t.Fatalf("partial warnings = %#v", warnings)
|
|
}
|
|
var events []pipeline.CheckpointEvent
|
|
readStateTestSummaryJSON(t, bundlePath, "checkpoint-events.json", &events)
|
|
if len(events) == 0 || events[0].Stage != "source" {
|
|
t.Fatalf("partial checkpoint events = %#v, want retained source decision", events)
|
|
}
|
|
var chunkPlan artifacts.ChunkPlanSummary
|
|
readStateTestSummaryJSON(t, bundlePath, "chunk-plan.json", &chunkPlan)
|
|
if chunkPlan.Mode != "bypass" || chunkPlan.ValidationStatus == "not_run" {
|
|
t.Fatalf("partial chunk plan = %#v", chunkPlan)
|
|
}
|
|
}
|
|
|
|
func TestRunTerminalPersistenceFailuresDoNotRecurseOrHidePrimaryError(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
reportErr error
|
|
errorLogErr error
|
|
wantSecondary string
|
|
}{
|
|
{name: "run report", reportErr: errors.New("injected run report failure"), wantSecondary: "injected run report failure"},
|
|
{name: "error log", errorLogErr: errors.New("injected error log failure"), wantSecondary: "injected error log failure"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
harness := newStateTestHarness()
|
|
harness.extractErr = errors.New("primary pipeline failure")
|
|
opts := harness.options()
|
|
var terminal *recordingTerminalWriter
|
|
opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter {
|
|
terminal = &recordingTerminalWriter{delegate: delegate, reportErr: tc.reportErr, errorLogErr: tc.errorLogErr}
|
|
return terminal
|
|
}
|
|
|
|
result := runStateTest(t, roots, opts, true, false, "bypass")
|
|
if result.code != 1 {
|
|
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
|
}
|
|
if terminal == nil {
|
|
t.Fatal("terminal writer was not constructed")
|
|
}
|
|
if terminal.reportCalls != 1 || terminal.errorLogCalls != 1 {
|
|
t.Fatalf("terminal calls = report:%d error:%d", terminal.reportCalls, terminal.errorLogCalls)
|
|
}
|
|
primaryIndex := strings.Index(result.stderr, "primary pipeline failure")
|
|
secondaryIndex := strings.Index(result.stderr, tc.wantSecondary)
|
|
debugIndex := strings.Index(result.stderr, "debug=")
|
|
if primaryIndex < 0 || secondaryIndex <= primaryIndex || debugIndex <= secondaryIndex {
|
|
t.Fatalf("stderr order = %q", result.stderr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunReportFailureOnSuccessIsTerminalizedWithoutRetry(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
opts := newStateTestHarness().options()
|
|
var terminal *recordingTerminalWriter
|
|
opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter {
|
|
terminal = &recordingTerminalWriter{delegate: delegate, reportErr: errors.New("injected success report failure")}
|
|
return terminal
|
|
}
|
|
|
|
result := runStateTest(t, roots, opts, true, false, "bypass")
|
|
if result.code != 1 || !strings.Contains(result.stderr, "write debug run report") || !strings.Contains(result.stderr, "injected success report failure") {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
|
}
|
|
if terminal == nil {
|
|
t.Fatal("terminal writer was not constructed")
|
|
}
|
|
if terminal.reportCalls != 1 || terminal.errorLogCalls != 1 {
|
|
t.Fatalf("terminal calls = report:%d error:%d", terminal.reportCalls, terminal.errorLogCalls)
|
|
}
|
|
if result.stdout != "" {
|
|
t.Fatalf("stdout=%q, want no success message", result.stdout)
|
|
}
|
|
bundlePath := onlyChildDir(t, roots.debug)
|
|
errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log"))
|
|
if err != nil || !strings.Contains(string(errorLog), "injected success report failure") {
|
|
t.Fatalf("error log = %q, %v", errorLog, err)
|
|
}
|
|
}
|
|
|
|
func TestRunWithoutDebugDoesNotUseTerminalSummaryWriter(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
harness := newStateTestHarness()
|
|
harness.extractErr = errors.New("non-debug pipeline failure")
|
|
opts := harness.options()
|
|
factoryCalls := 0
|
|
opts.DebugTerminalFactory = func(delegate *debugbundle.SummaryWriter) DebugTerminalWriter {
|
|
factoryCalls++
|
|
return delegate
|
|
}
|
|
|
|
result := runStateTest(t, roots, opts, false, false, "bypass")
|
|
if result.code != 1 || !strings.Contains(result.stderr, "non-debug pipeline failure") {
|
|
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
|
}
|
|
if factoryCalls != 0 {
|
|
t.Fatalf("terminal summary factory calls = %d, want 0", factoryCalls)
|
|
}
|
|
assertAbsent(t, roots.debug)
|
|
}
|
|
|
|
func TestRunRefusesExistingOutputDirectoryWithoutChangingIt(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
const runID = "run-1000000000-22222222222222222222222222222222"
|
|
runPath := filepath.Join(roots.output, runID)
|
|
if err := os.MkdirAll(filepath.Join(runPath, "nested"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(runPath, "sentinel"), []byte("existing output"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(runPath, "nested", "data"), []byte("preserve me"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := readTree(t, runPath)
|
|
opts := newStateTestHarness().options()
|
|
opts.RunIDGenerator = func(time.Time) (string, error) { return runID, nil }
|
|
|
|
result := runStateTest(t, roots, opts, true, false, "bypass")
|
|
if result.code != 1 || !strings.Contains(result.stderr, "output run directory") || !strings.Contains(result.stderr, "already exists") {
|
|
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
|
}
|
|
if after := readTree(t, runPath); !sameFiles(after, before) {
|
|
t.Fatalf("existing output changed: before=%v after=%v", before, after)
|
|
}
|
|
bundlePath := filepath.Join(roots.debug, runID)
|
|
report := readStateTestRunReport(t, bundlePath)
|
|
if report.Succeeded || report.RunID != runID || report.OutputPath != runPath || report.DebugPath != bundlePath || report.OutputCount != 1 || report.ValidationStatus != "approved" {
|
|
t.Fatalf("output collision report = %#v", report)
|
|
}
|
|
errorLog, err := os.ReadFile(filepath.Join(bundlePath, "summary", "error.log"))
|
|
if err != nil || !strings.Contains(string(errorLog), "already exists") {
|
|
t.Fatalf("output collision error log = %q, %v", errorLog, err)
|
|
}
|
|
}
|
|
|
|
func TestRepeatedRunIdentityCannotOverwriteFirstOutput(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
const runID = "run-1000000000-33333333333333333333333333333333"
|
|
harness := newStateTestHarness()
|
|
opts := harness.options()
|
|
opts.RunIDGenerator = func(time.Time) (string, error) { return runID, nil }
|
|
|
|
first := runStateTest(t, roots, opts, false, false, "bypass")
|
|
if first.code != 0 {
|
|
t.Fatalf("first code=%d stderr=%q", first.code, first.stderr)
|
|
}
|
|
runPath := filepath.Join(roots.output, runID)
|
|
before := readTree(t, runPath)
|
|
second := runStateTest(t, roots, opts, false, false, "bypass")
|
|
if second.code != 1 || !strings.Contains(second.stderr, "already exists") {
|
|
t.Fatalf("second code=%d stderr=%q", second.code, second.stderr)
|
|
}
|
|
if after := readTree(t, runPath); !sameFiles(after, before) {
|
|
t.Fatalf("first output changed: before=%v after=%v", before, after)
|
|
}
|
|
}
|
|
|
|
func TestRunRefusesExistingDebugBundleWithoutChangingIt(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
const runID = "run-1000000000-44444444444444444444444444444444"
|
|
bundlePath := filepath.Join(roots.debug, runID)
|
|
if err := os.MkdirAll(bundlePath, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sentinelPath := filepath.Join(bundlePath, "sentinel")
|
|
if err := os.WriteFile(sentinelPath, []byte("existing debug"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
opts := newStateTestHarness().options()
|
|
opts.RunIDGenerator = func(time.Time) (string, error) { return runID, nil }
|
|
|
|
result := runStateTest(t, roots, opts, true, false, "bypass")
|
|
if result.code != 1 || !strings.Contains(result.stderr, "debug bundle") || !strings.Contains(result.stderr, "already exists") {
|
|
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
|
}
|
|
if got, err := os.ReadFile(sentinelPath); err != nil || string(got) != "existing debug" {
|
|
t.Fatalf("sentinel = %q, %v", got, err)
|
|
}
|
|
assertAbsent(t, roots.output)
|
|
}
|
|
|
|
func TestRunIDGenerationFailurePrecedesDebugAllocation(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
opts := newStateTestHarness().options()
|
|
opts.RunIDGenerator = func(time.Time) (string, error) { return "", errors.New("random source unavailable") }
|
|
|
|
result := runStateTest(t, roots, opts, true, false, "bypass")
|
|
if result.code != 1 || !strings.Contains(result.stderr, "generate run ID: random source unavailable") {
|
|
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
|
}
|
|
assertAbsent(t, roots.debug)
|
|
assertAbsent(t, roots.output)
|
|
}
|
|
|
|
func TestRunRejectsUnsafeGeneratedIdentityBeforePathUse(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
opts := newStateTestHarness().options()
|
|
opts.RunIDGenerator = func(time.Time) (string, error) { return "../outside", nil }
|
|
|
|
result := runStateTest(t, roots, opts, true, false, "bypass")
|
|
if result.code != 1 || !strings.Contains(result.stderr, "invalid generated run ID") || !strings.Contains(result.stderr, "one safe path component") {
|
|
t.Fatalf("code=%d stderr=%q", result.code, result.stderr)
|
|
}
|
|
assertAbsent(t, roots.debug)
|
|
assertAbsent(t, roots.output)
|
|
}
|
|
|
|
type stateTestRoots struct{ config, input, output, plans, checkpoints, debug string }
|
|
|
|
func newStateTestRoots(t *testing.T) stateTestRoots {
|
|
t.Helper()
|
|
base := t.TempDir()
|
|
roots := stateTestRoots{input: filepath.Join(base, "input.txt"), output: filepath.Join(base, "output"), plans: filepath.Join(base, "plans"), checkpoints: filepath.Join(base, "checkpoints"), debug: filepath.Join(base, "debug")}
|
|
if err := os.WriteFile(roots.input, []byte("application content Bearer secretvalue sk-secretvalue"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
roots.config = filepath.Join(base, "config.yml")
|
|
config := fmt.Sprintf("version: 3\noutput:\n directory: %q\ncache:\n chunk_plans:\n directory: %q\n mode: auto\n checkpoints:\n enabled: true\n directory: %q\ndebug:\n directory: %q\npipelines:\n sample:\n input: test/input\n chunk: test/chunk\n artifacts:\n items:\n extract: test/extract\n merge: test/merge\n normalize: test/normalize\n output: test/output\n", roots.output, roots.plans, roots.checkpoints, roots.debug)
|
|
if err := os.WriteFile(roots.config, []byte(config), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return roots
|
|
}
|
|
|
|
type stateTestResult struct {
|
|
code int
|
|
stdout, stderr string
|
|
}
|
|
|
|
func runStateTest(t *testing.T, roots stateTestRoots, opts Options, debug, resume bool, mode string) stateTestResult {
|
|
t.Helper()
|
|
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", mode}
|
|
if debug {
|
|
args = append(args, "--debug")
|
|
}
|
|
if resume {
|
|
args = append(args, "--resume")
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
return stateTestResult{RunWithOptions(args, &stdout, &stderr, opts), stdout.String(), stderr.String()}
|
|
}
|
|
|
|
func assertStateTestOutput(t *testing.T, root string) {
|
|
t.Helper()
|
|
output := onlyChildDir(t, root)
|
|
data, err := os.ReadFile(filepath.Join(output, "result.json"))
|
|
if err != nil || string(data) != "{\"ok\":true}\n" {
|
|
t.Fatalf("output = %q, %v", data, err)
|
|
}
|
|
}
|
|
|
|
func onlyChildDir(t *testing.T, root string) string {
|
|
t.Helper()
|
|
entries, err := os.ReadDir(root)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var dirs []string
|
|
for _, entry := range entries {
|
|
if entry.IsDir() {
|
|
dirs = append(dirs, filepath.Join(root, entry.Name()))
|
|
}
|
|
}
|
|
if len(dirs) != 1 {
|
|
t.Fatalf("directories in %q = %v, want one", root, dirs)
|
|
}
|
|
return dirs[0]
|
|
}
|
|
|
|
func assertFile(t *testing.T, path string) {
|
|
t.Helper()
|
|
if info, err := os.Stat(path); err != nil || info.IsDir() {
|
|
t.Fatalf("file %q: %v", path, err)
|
|
}
|
|
}
|
|
func assertAbsent(t *testing.T, path string) {
|
|
t.Helper()
|
|
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
|
t.Fatalf("%q exists or stat failed: %v", path, err)
|
|
}
|
|
}
|
|
func assertAnyFile(t *testing.T, root string) {
|
|
t.Helper()
|
|
if text := readAllFiles(t, root); text == "" {
|
|
t.Fatalf("no files under %q", root)
|
|
}
|
|
}
|
|
|
|
func readAllFiles(t *testing.T, root string) string {
|
|
t.Helper()
|
|
var content strings.Builder
|
|
if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if entry.IsDir() {
|
|
return nil
|
|
}
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
content.Write(data)
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return content.String()
|
|
}
|
|
|
|
func readStateTestRunReport(t *testing.T, bundlePath string) debugbundle.RunReport {
|
|
t.Helper()
|
|
var report debugbundle.RunReport
|
|
readStateTestSummaryJSON(t, bundlePath, "run-report.json", &report)
|
|
return report
|
|
}
|
|
|
|
func readStateTestSummaryJSON(t *testing.T, bundlePath, name string, target any) {
|
|
t.Helper()
|
|
data, err := os.ReadFile(filepath.Join(bundlePath, "summary", name))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := json.Unmarshal(data, target); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func assertRestrictedTree(t *testing.T, root string) {
|
|
t.Helper()
|
|
if runtime.GOOS == "windows" {
|
|
return
|
|
}
|
|
if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
want := os.FileMode(0o600)
|
|
if info.IsDir() {
|
|
want = 0o700
|
|
}
|
|
if info.Mode().Perm() != want {
|
|
return fmt.Errorf("%s has mode %o, want %o", path, info.Mode().Perm(), want)
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func readTree(t *testing.T, root string) map[string][]byte {
|
|
t.Helper()
|
|
files := map[string][]byte{}
|
|
if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if entry.IsDir() {
|
|
return nil
|
|
}
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
relative, err := filepath.Rel(root, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
files[relative] = data
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return files
|
|
}
|
|
func sameFiles(left, right map[string][]byte) bool {
|
|
if len(left) != len(right) {
|
|
return false
|
|
}
|
|
for path, data := range left {
|
|
if !bytes.Equal(data, right[path]) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
type stateTestHarness struct {
|
|
mu sync.Mutex
|
|
chunkCalls, extractCalls int
|
|
runIDCalls uint64
|
|
extractErr error
|
|
chunkWarnings []contracts.Warning
|
|
moduleProfiles []string
|
|
sessionIDs []string
|
|
outputWarnings []contracts.Warning
|
|
includeWarnings bool
|
|
}
|
|
|
|
func newStateTestHarness() *stateTestHarness { return &stateTestHarness{} }
|
|
func (h *stateTestHarness) options() Options {
|
|
registries := pipeline.Registries{Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()}
|
|
if err := pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, stateTestCodec{}); err != nil {
|
|
panic(err)
|
|
}
|
|
if err := registries.Inputs.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "test/input", Stage: pipeline.StageInput, Provides: []string{"source"}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.InputAdapter, error) { return stateTestInput{}, nil }); err != nil {
|
|
panic(err)
|
|
}
|
|
if err := registries.Chunkers.RegisterBuilderWithSpec(pipeline.ModuleSpec{Key: "test/chunk", Stage: pipeline.StageChunk, Requires: []string{"source"}, Provides: []string{"chunks"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "cache-reference"}}}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.Chunker, error) { return stateTestChunker{h}, nil }); err != nil {
|
|
panic(err)
|
|
}
|
|
if err := pipeline.RegisterExtractor(registries.Extractors, pipeline.ModuleSpec{Key: "test/extract", Stage: pipeline.StageExtract, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Extractor[stateTestArtifact], error) { return stateTestExtractor{h}, nil }); err != nil {
|
|
panic(err)
|
|
}
|
|
if err := pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "test/merge", Stage: pipeline.StageMerge, Requires: []string{"artifact"}, Provides: []string{"merged"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Merger[stateTestArtifact], error) { return stateTestMerger{harness: h}, nil }); err != nil {
|
|
panic(err)
|
|
}
|
|
if err := pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "test/normalize", Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ArtifactKind: stateTestArtifactKind}, func() (contracts.Normalizer[stateTestArtifact], error) { return stateTestNormalizer{harness: h}, nil }); err != nil {
|
|
panic(err)
|
|
}
|
|
if err := registries.Outputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "test/output", Stage: pipeline.StageOutput, Requires: []string{"normalized"}, Provides: []string{"output"}}, func() (contracts.OutputEncoder, error) {
|
|
return stateTestOutput{harness: h, includeWarnings: h.includeWarnings}, nil
|
|
}); err != nil {
|
|
panic(err)
|
|
}
|
|
return Options{Catalog: catalogFromRegistries(registries), Registries: registries, LookupEnv: emptyLookup, Now: func() time.Time { return time.Unix(1, 0) }, RunIDGenerator: func(startedAt time.Time) (string, error) {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
h.runIDCalls++
|
|
return fmt.Sprintf("run-%d-%032x", startedAt.UnixNano(), h.runIDCalls), nil
|
|
}, UserCacheDir: func() (string, error) { return "", errors.New("unexpected user cache lookup") }, LLMClientFactory: func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
|
return nil, nil, nil
|
|
}}
|
|
}
|
|
|
|
type stateTestInput struct{}
|
|
|
|
func (stateTestInput) Key() string { return "test/input" }
|
|
func (stateTestInput) Parse(_ context.Context, req contracts.ParseRequest) (*source.SourceDocument, error) {
|
|
return &source.SourceDocument{ID: "source", Kind: "text", Format: "text/plain", Digest: stateTestDigest, Units: []source.SourceUnit{{ID: 1, Kind: "text", Text: string(req.Raw), Ref: source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}}}, nil
|
|
}
|
|
|
|
type stateTestChunker struct{ harness *stateTestHarness }
|
|
|
|
func (stateTestChunker) Key() string { return "test/chunk" }
|
|
func (stateTestChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
|
func (c stateTestChunker) Plan(_ context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
|
|
c.harness.mu.Lock()
|
|
c.harness.moduleProfiles = append(c.harness.moduleProfiles, req.LLMProfile)
|
|
c.harness.sessionIDs = append(c.harness.sessionIDs, req.SessionID)
|
|
c.harness.mu.Unlock()
|
|
c.harness.mu.Lock()
|
|
c.harness.chunkCalls++
|
|
c.harness.mu.Unlock()
|
|
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: 1, EndUnitID: 1}}}, Warnings: append([]contracts.Warning(nil), c.harness.chunkWarnings...)}, nil
|
|
}
|
|
|
|
const stateTestArtifactKind contracts.ArtifactKind = "test/artifact"
|
|
|
|
type stateTestArtifact struct {
|
|
Value string `json:"value"`
|
|
}
|
|
type stateTestCodec struct{}
|
|
|
|
func (stateTestCodec) Kind() contracts.ArtifactKind { return stateTestArtifactKind }
|
|
func (stateTestCodec) Schema() contracts.ArtifactSchema {
|
|
return contracts.ArtifactSchema{ID: "test.artifact", Name: "test_artifact", Version: "v1", JSONSchema: []byte(`{"type":"object"}`)}
|
|
}
|
|
func (stateTestCodec) MediaType() string { return "application/json" }
|
|
func (stateTestCodec) EncodeCandidate(v stateTestArtifact) ([]byte, error) {
|
|
return []byte(`{"value":"ok"}`), nil
|
|
}
|
|
func (stateTestCodec) Encode(v stateTestArtifact) ([]byte, error) {
|
|
return []byte(`{"value":"ok"}`), nil
|
|
}
|
|
func (stateTestCodec) Decode([]byte) (stateTestArtifact, error) {
|
|
return stateTestArtifact{Value: "ok"}, nil
|
|
}
|
|
|
|
type stateTestExtractor struct{ harness *stateTestHarness }
|
|
|
|
func (stateTestExtractor) Key() string { return "test/extract" }
|
|
func (stateTestExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
|
func (e stateTestExtractor) Extract(_ context.Context, req contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[stateTestArtifact], error) {
|
|
e.harness.mu.Lock()
|
|
defer e.harness.mu.Unlock()
|
|
e.harness.extractCalls++
|
|
e.harness.moduleProfiles = append(e.harness.moduleProfiles, req.LLMProfile)
|
|
e.harness.sessionIDs = append(e.harness.sessionIDs, req.SessionID)
|
|
if e.harness.extractErr != nil {
|
|
return contracts.TypedExtractionResult[stateTestArtifact]{}, e.harness.extractErr
|
|
}
|
|
return contracts.TypedExtractionResult[stateTestArtifact]{Value: stateTestArtifact{Value: "ok"}}, nil
|
|
}
|
|
|
|
type stateTestMerger struct{ harness *stateTestHarness }
|
|
|
|
func (stateTestMerger) Key() string { return "test/merge" }
|
|
func (m stateTestMerger) Merge(_ context.Context, req contracts.TypedMergeRequest[stateTestArtifact]) (contracts.TypedMergeResult[stateTestArtifact], error) {
|
|
m.harness.mu.Lock()
|
|
m.harness.moduleProfiles = append(m.harness.moduleProfiles, req.LLMProfile)
|
|
m.harness.sessionIDs = append(m.harness.sessionIDs, req.SessionID)
|
|
m.harness.mu.Unlock()
|
|
return contracts.TypedMergeResult[stateTestArtifact]{Value: req.ExtractOutputs[0].Value}, nil
|
|
}
|
|
|
|
type stateTestNormalizer struct{ harness *stateTestHarness }
|
|
|
|
func (stateTestNormalizer) Key() string { return "test/normalize" }
|
|
func (stateTestNormalizer) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
|
func (n stateTestNormalizer) Normalize(_ context.Context, req contracts.TypedNormalizeRequest[stateTestArtifact]) (contracts.TypedNormalizeResult[stateTestArtifact], error) {
|
|
n.harness.mu.Lock()
|
|
n.harness.moduleProfiles = append(n.harness.moduleProfiles, req.LLMProfile)
|
|
n.harness.sessionIDs = append(n.harness.sessionIDs, req.SessionID)
|
|
n.harness.mu.Unlock()
|
|
return contracts.TypedNormalizeResult[stateTestArtifact]{Value: req.MergeOutput.Value}, nil
|
|
}
|
|
|
|
type stateTestOutput struct {
|
|
harness *stateTestHarness
|
|
includeWarnings bool
|
|
}
|
|
|
|
func (o stateTestOutput) Key() string { return "test/output" }
|
|
func (o stateTestOutput) Encode(_ context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
|
o.harness.mu.Lock()
|
|
o.harness.outputWarnings = append([]contracts.Warning(nil), req.Warnings...)
|
|
o.harness.mu.Unlock()
|
|
data := []byte("{\"ok\":true}\n")
|
|
if o.includeWarnings && len(req.Warnings) > 0 {
|
|
data = []byte(fmt.Sprintf("{\"ok\":true,\"warnings\":%q}\n", req.Warnings[0].ReasonCode))
|
|
}
|
|
return contracts.OutputResult{Files: []contracts.OutputFile{{Name: "result.json", Bytes: data}}}, nil
|
|
}
|
|
|
|
type failingDebugRecorder struct{}
|
|
|
|
func (failingDebugRecorder) Enabled() bool { return true }
|
|
func (failingDebugRecorder) WriteJSON(string, any) error { return errors.New("trace unavailable") }
|
|
func (failingDebugRecorder) WriteBytes(string, []byte) error { return errors.New("trace unavailable") }
|
|
|
|
type recordingTerminalWriter struct {
|
|
delegate DebugTerminalWriter
|
|
reportErr, errorLogErr error
|
|
reportCalls, errorLogCalls int
|
|
}
|
|
|
|
func (w *recordingTerminalWriter) WriteRunReport(report debugbundle.RunReport) error {
|
|
w.reportCalls++
|
|
if w.reportErr != nil {
|
|
return w.reportErr
|
|
}
|
|
return w.delegate.WriteRunReport(report)
|
|
}
|
|
|
|
func (w *recordingTerminalWriter) WriteError(message string) error {
|
|
w.errorLogCalls++
|
|
if w.errorLogErr != nil {
|
|
return w.errorLogErr
|
|
}
|
|
return w.delegate.WriteError(message)
|
|
}
|