631 lines
27 KiB
Go
631 lines
27 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
)
|
|
|
|
func TestRunControlsRejectSyntaxWithoutAllocatingState(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
args func(stateTestRoots) []string
|
|
}{
|
|
{name: "missing pipeline", args: func(roots stateTestRoots) []string {
|
|
return []string{"run", "--config", roots.config, "--input", roots.input}
|
|
}},
|
|
{name: "missing input", args: func(roots stateTestRoots) []string {
|
|
return []string{"run", "sample", "--config", roots.config}
|
|
}},
|
|
{name: "unknown flag", args: func(roots stateTestRoots) []string {
|
|
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--unknown"}
|
|
}},
|
|
{name: "blank output directory", args: func(roots stateTestRoots) []string {
|
|
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--output-dir", ""}
|
|
}},
|
|
{name: "blank debug directory", args: func(roots stateTestRoots) []string {
|
|
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", ""}
|
|
}},
|
|
{name: "debug directory without debug", args: func(roots stateTestRoots) []string {
|
|
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", filepath.Join(filepath.Dir(roots.debug), "requested-debug")}
|
|
}},
|
|
{name: "blank session ID", args: func(roots stateTestRoots) []string {
|
|
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--session-id", ""}
|
|
}},
|
|
{name: "multiple pipeline IDs", args: func(roots stateTestRoots) []string {
|
|
return []string{"run", "sample", "extra", "--config", roots.config, "--input", roots.input}
|
|
}},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions(tt.args(roots), &stdout, &stderr, newStateTestHarness().options())
|
|
if code != 2 || stdout.Len() != 0 || stderr.Len() == 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
assertAbsent(t, roots.output)
|
|
assertAbsent(t, roots.debug)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRecomputeStepCLIContract(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
configure func(*testing.T, stateTestRoots)
|
|
flags []string
|
|
wantCode int
|
|
wantOutput string
|
|
wantError string
|
|
}{
|
|
{
|
|
name: "explicit step",
|
|
configure: func(t *testing.T, roots stateTestRoots) {
|
|
replaceStateTestConfigLine(t, roots.config, " artifacts:\n items:\n extract: test/extract\n merge: test/merge\n normalize: test/normalize\n", " steps:\n - id: chosen\n artifacts:\n items:\n extract: test/extract\n merge: test/merge\n normalize: test/normalize\n")
|
|
},
|
|
flags: []string{"--resume", "--recompute-step", "chosen"},
|
|
wantCode: 0,
|
|
wantOutput: "outputs=1",
|
|
},
|
|
{name: "implicit default step", flags: []string{"--resume", "--recompute-step", "default"}, wantCode: 0, wantOutput: "outputs=1"},
|
|
{name: "repeated flag", flags: []string{"--resume", "--recompute-step", "default", "--recompute-step", "default"}, wantCode: 2, wantError: "specified only once"},
|
|
{name: "empty step", flags: []string{"--resume", "--recompute-step", ""}, wantCode: 2, wantError: "must not be empty"},
|
|
{name: "unknown step", flags: []string{"--resume", "--recompute-step", "missing"}, wantCode: 1, wantError: "unknown pipeline step"},
|
|
{name: "without resume", flags: []string{"--recompute-step", "default"}, wantCode: 2, wantError: "requires --resume"},
|
|
{
|
|
name: "checkpoint recording disabled",
|
|
configure: func(t *testing.T, roots stateTestRoots) {
|
|
replaceStateTestConfigLine(t, roots.config, " enabled: true\n", " enabled: false\n")
|
|
},
|
|
flags: []string{"--resume", "--recompute-step", "default"}, wantCode: 1, wantError: "cache.checkpoints.enabled",
|
|
},
|
|
{name: "with only", flags: []string{"--resume", "--recompute-step", "default", "--only", "items"}, wantCode: 2, wantError: "cannot be combined with --only"},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
if tt.configure != nil {
|
|
tt.configure(t, roots)
|
|
}
|
|
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}
|
|
args = append(args, tt.flags...)
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions(args, &stdout, &stderr, newStateTestHarness().options())
|
|
if code != tt.wantCode || (tt.wantOutput != "" && !strings.Contains(stdout.String(), tt.wantOutput)) || (tt.wantError != "" && !strings.Contains(stderr.String(), tt.wantError)) {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunValidFailuresClassifyAndReportDebug(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
args func(stateTestRoots) []string
|
|
wantError string
|
|
wantDebug bool
|
|
}{
|
|
{name: "unknown pipeline", args: func(roots stateTestRoots) []string {
|
|
return []string{"run", "missing", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}
|
|
}, wantError: `pipeline "missing"`},
|
|
{name: "unknown lane", args: func(roots stateTestRoots) []string {
|
|
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--only", "missing", "--chunk_cache", "bypass", "--debug"}
|
|
}, wantError: `lane "missing"`, wantDebug: true},
|
|
{name: "unreadable input", args: func(roots stateTestRoots) []string {
|
|
return []string{"run", "sample", "--config", roots.config, "--input", filepath.Join(filepath.Dir(roots.input), "unreadable.txt"), "--chunk_cache", "bypass", "--debug"}
|
|
}, wantError: "read input", wantDebug: true},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions(tt.args(roots), &stdout, &stderr, newStateTestHarness().options())
|
|
if code != 1 || stdout.Len() != 0 || !strings.Contains(stderr.String(), tt.wantError) {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
if tt.wantDebug {
|
|
if !strings.Contains(stderr.String(), "debug=") {
|
|
t.Fatalf("stderr=%q, want debug path", stderr.String())
|
|
}
|
|
onlyChildDir(t, roots.debug)
|
|
} else {
|
|
assertAbsent(t, roots.debug)
|
|
}
|
|
assertAbsent(t, roots.output)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunOnlyExecutesSelectedLanes(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
data, err := os.ReadFile(roots.config)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
data = []byte(replaceRequiredOnce(t, string(data), " output: test/output\n", " other:\n extract: test/extract\n output: test/output\n"))
|
|
if err := os.WriteFile(roots.config, data, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
harness := newStateTestHarness()
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--only", "items", "--chunk_cache", "bypass"}, &stdout, &stderr, harness.options())
|
|
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || stderr.Len() != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
harness.mu.Lock()
|
|
extractCalls := harness.extractCalls
|
|
harness.mu.Unlock()
|
|
if extractCalls != 1 {
|
|
t.Fatalf("extract calls = %d, want only the selected lane", extractCalls)
|
|
}
|
|
}
|
|
|
|
func TestRunStateRootsHonorEnvironmentFlagsAndDefaults(t *testing.T) {
|
|
t.Run("environment roots", func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
environmentOutput := filepath.Join(t.TempDir(), "environment-output")
|
|
environmentDebug := filepath.Join(t.TempDir(), "environment-debug")
|
|
opts := newStateTestHarness().options()
|
|
opts.LookupEnv = lookupRunContractEnv(map[string]string{
|
|
"NOTARIUS_OUTPUT_DIR": environmentOutput,
|
|
"NOTARIUS_DEBUG_DIR": environmentDebug,
|
|
})
|
|
result := runWithStateRoots(t, roots, opts, nil)
|
|
if result.code != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
|
}
|
|
assertFile(t, filepath.Join(environmentOutput, filepath.Base(onlyChildDir(t, environmentOutput)), "result.json"))
|
|
onlyChildDir(t, environmentDebug)
|
|
assertAbsent(t, roots.output)
|
|
assertAbsent(t, roots.debug)
|
|
})
|
|
|
|
t.Run("command flags override environment", func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
environmentOutput := filepath.Join(t.TempDir(), "environment-output")
|
|
environmentDebug := filepath.Join(t.TempDir(), "environment-debug")
|
|
flagOutput := filepath.Join(t.TempDir(), "flag-output")
|
|
flagDebug := filepath.Join(t.TempDir(), "flag-debug")
|
|
opts := newStateTestHarness().options()
|
|
opts.LookupEnv = lookupRunContractEnv(map[string]string{
|
|
"NOTARIUS_OUTPUT_DIR": environmentOutput,
|
|
"NOTARIUS_DEBUG_DIR": environmentDebug,
|
|
})
|
|
result := runWithStateRoots(t, roots, opts, []string{"--output-dir", flagOutput, "--debug-dir", flagDebug})
|
|
if result.code != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
|
}
|
|
assertFile(t, filepath.Join(flagOutput, filepath.Base(onlyChildDir(t, flagOutput)), "result.json"))
|
|
onlyChildDir(t, flagDebug)
|
|
assertAbsent(t, environmentOutput)
|
|
assertAbsent(t, environmentDebug)
|
|
})
|
|
|
|
t.Run("built-in roots", func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
data, err := os.ReadFile(roots.config)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
text := string(data)
|
|
text = replaceRequiredOnce(t, text, fmt.Sprintf(" directory: %q\n", roots.output), "")
|
|
text = replaceRequiredOnce(t, text, fmt.Sprintf(" directory: %q\n", roots.debug), "")
|
|
if err := os.WriteFile(roots.config, []byte(text), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
workDir := t.TempDir()
|
|
t.Chdir(workDir)
|
|
opts := newStateTestHarness().options()
|
|
result := runWithStateRoots(t, roots, opts, nil)
|
|
if result.code != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", result.code, result.stdout, result.stderr)
|
|
}
|
|
assertFile(t, filepath.Join(workDir, "notarius-output", filepath.Base(onlyChildDir(t, filepath.Join(workDir, "notarius-output"))), "result.json"))
|
|
onlyChildDir(t, filepath.Join(workDir, "notarius-debug"))
|
|
})
|
|
}
|
|
|
|
func TestRunLLMProfileOverrideAndValidationUseInjectedBoundaries(t *testing.T) {
|
|
t.Run("one effective profile reaches the factory and modules", func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
profileDir := writeRunContractProfiles(t, "override-profile")
|
|
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
|
|
harness := newStateTestHarness()
|
|
var factoryProfiles []string
|
|
opts := harness.options()
|
|
var factoryOverrides []LLMRuntimeOverrides
|
|
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
|
factoryProfiles = append(factoryProfiles, profileID)
|
|
factoryOverrides = append(factoryOverrides, overrides)
|
|
return nil, nil, nil
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", "override-profile"}, &stdout, &stderr, opts)
|
|
if code != 0 || stderr.Len() != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
if len(factoryProfiles) != 1 || factoryProfiles[0] != "override-profile" {
|
|
t.Fatalf("factory profiles = %#v, want one override profile", factoryProfiles)
|
|
}
|
|
if len(factoryOverrides) != 1 || factoryOverrides[0].ReasoningEffort != nil {
|
|
t.Fatalf("factory overrides = %#v, want inherited reasoning", factoryOverrides)
|
|
}
|
|
harness.mu.Lock()
|
|
profiles := append([]string(nil), harness.moduleProfiles...)
|
|
harness.mu.Unlock()
|
|
if len(profiles) < 4 {
|
|
t.Fatalf("module profiles = %#v, want chunk and lane stage requests", profiles)
|
|
}
|
|
for _, profile := range profiles {
|
|
if profile != "override-profile" {
|
|
t.Fatalf("module profiles = %#v, want override on every request", profiles)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("validator profile remains distinct", func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
profileDir := writeRunContractProfiles(t, "override-profile", "validator-profile")
|
|
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
|
|
harness := newStateTestHarness()
|
|
var validatorProfiles []string
|
|
opts := harness.options()
|
|
registerRunContractValidator(t, &opts, &validatorProfiles)
|
|
factoryProfiles := []string{}
|
|
opts.LLMClientFactory = func(_ context.Context, _ config.Config, profileID string, _ LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
|
factoryProfiles = append(factoryProfiles, profileID)
|
|
return nil, nil, nil
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", "override-profile"}, &stdout, &stderr, opts)
|
|
if code != 0 || stderr.Len() != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
if len(factoryProfiles) != 1 || factoryProfiles[0] != "" {
|
|
t.Fatalf("factory profiles = %#v, want one call without a unique profile", factoryProfiles)
|
|
}
|
|
if len(validatorProfiles) != 1 || validatorProfiles[0] != "validator-profile" {
|
|
t.Fatalf("validator profiles = %#v, want configured validator profile", validatorProfiles)
|
|
}
|
|
})
|
|
|
|
t.Run("unknown profile is rejected without factory access", func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
profileDir := writeRunContractProfiles(t, "override-profile")
|
|
prependRunContractConfig(t, roots, fmt.Sprintf("promptkit:\n profile_dir: %q\n", profileDir))
|
|
factoryCalls := 0
|
|
opts := newStateTestHarness().options()
|
|
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
|
factoryCalls++
|
|
return nil, nil, nil
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--llm-profile", "missing-profile"}, &stdout, &stderr, opts)
|
|
if code != 1 || !strings.Contains(stderr.String(), "not configured") || factoryCalls != 0 || stdout.Len() != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q factoryCalls=%d", code, stdout.String(), stderr.String(), factoryCalls)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestRunReasoningEffortOverrideReachesFactory(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
flags []string
|
|
wantValue string
|
|
wantSet bool
|
|
}{
|
|
{name: "inherit"},
|
|
{name: "replace", flags: []string{"--reasoning-effort", " focused "}, wantValue: "focused", wantSet: true},
|
|
{name: "clear", flags: []string{"--clear-reasoning-effort"}, wantSet: true},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
opts := newStateTestHarness().options()
|
|
var got []LLMRuntimeOverrides
|
|
opts.LLMClientFactory = func(_ context.Context, _ config.Config, _ string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
|
got = append(got, overrides)
|
|
return nil, nil, nil
|
|
}
|
|
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, tt.flags...)
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions(args, &stdout, &stderr, opts); code != 0 || stderr.Len() != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
if len(got) != 1 {
|
|
t.Fatalf("factory overrides = %#v, want one call", got)
|
|
}
|
|
if !tt.wantSet {
|
|
if got[0].ReasoningEffort != nil {
|
|
t.Fatalf("reasoning effort = %q, want inherit", *got[0].ReasoningEffort)
|
|
}
|
|
return
|
|
}
|
|
if got[0].ReasoningEffort == nil || *got[0].ReasoningEffort != tt.wantValue {
|
|
t.Fatalf("reasoning effort = %#v, want %q", got[0].ReasoningEffort, tt.wantValue)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunReasoningEffortOverrideRejectsInvalidSyntax(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
flags []string
|
|
wantError string
|
|
}{
|
|
{
|
|
name: "mutually exclusive controls",
|
|
flags: []string{"--reasoning-effort", "focused", "--clear-reasoning-effort"},
|
|
wantError: "cannot be combined",
|
|
},
|
|
{
|
|
name: "empty replacement",
|
|
flags: []string{"--reasoning-effort", " "},
|
|
wantError: "must not be empty",
|
|
},
|
|
{
|
|
name: "duplicate replacement",
|
|
flags: []string{"--reasoning-effort", "low", "--reasoning-effort", "high"},
|
|
wantError: "may be specified only once",
|
|
},
|
|
{
|
|
name: "missing replacement",
|
|
flags: []string{"--reasoning-effort"},
|
|
wantError: "flag needs an argument",
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input}, tt.flags...)
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions(args, &stdout, &stderr, newStateTestHarness().options())
|
|
if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), tt.wantError) {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
assertNoRunState(t, roots)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestReasoningEffortOverrideSeparatesCheckpointIdentities(t *testing.T) {
|
|
replacement := " focused "
|
|
cleared := ""
|
|
states := []struct {
|
|
name string
|
|
overrides LLMRuntimeOverrides
|
|
wantValue string
|
|
wantSet bool
|
|
}{
|
|
{name: "inherit"},
|
|
{name: "replace", overrides: LLMRuntimeOverrides{ReasoningEffort: &replacement}, wantValue: "focused", wantSet: true},
|
|
{name: "clear", overrides: LLMRuntimeOverrides{ReasoningEffort: &cleared}, wantValue: "<cleared>", wantSet: true},
|
|
}
|
|
digests := make(map[string]string, len(states))
|
|
for _, state := range states {
|
|
fingerprints := runtimeOverrideFingerprints("", "", state.overrides)
|
|
var value string
|
|
var found bool
|
|
for _, fingerprint := range fingerprints {
|
|
if fingerprint.Name == "reasoning_effort_override" {
|
|
value, found = fingerprint.Value, true
|
|
}
|
|
}
|
|
if found != state.wantSet || (found && value != state.wantValue) {
|
|
t.Fatalf("%s fingerprint found=%t value=%q, want found=%t value=%q", state.name, found, value, state.wantSet, state.wantValue)
|
|
}
|
|
identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
|
|
Pipeline: pipeline.ResolvedPipeline{ID: "sample", Digest: "sha256:pipeline", Input: pipeline.Binding("test/input")},
|
|
RawInputDigest: "sha256:input",
|
|
RuntimeOverrides: fingerprints,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
digests[state.name] = identity.Digest
|
|
}
|
|
if digests["inherit"] == digests["replace"] || digests["inherit"] == digests["clear"] || digests["replace"] == digests["clear"] {
|
|
t.Fatalf("checkpoint identity digests are not distinct: %#v", digests)
|
|
}
|
|
}
|
|
|
|
func TestEffectiveLLMProfileIDsAreSortedDeduplicatedAndLLMOnly(t *testing.T) {
|
|
resolved := pipeline.ResolvedPipeline{
|
|
Input: pipeline.ModuleBinding{LLMProfile: "input-profile"},
|
|
Chunk: pipeline.ModuleBinding{LLMProfile: " zeta "},
|
|
Steps: []pipeline.ResolvedPipelineStep{{
|
|
ID: "default",
|
|
ArtifactLanes: []pipeline.ResolvedArtifactLane{{
|
|
Extract: pipeline.ModuleBinding{LLMProfile: "alpha"},
|
|
Merge: pipeline.ModuleBinding{LLMProfile: "zeta"},
|
|
Normalize: pipeline.ModuleBinding{LLMProfile: " gamma "},
|
|
}},
|
|
}},
|
|
ValidatorChains: []pipeline.ResolvedValidatorChain{{Validators: []pipeline.ResolvedValidator{
|
|
{Binding: pipeline.ModuleBinding{LLMProfile: "deterministic-profile"}, ExecutionClass: contracts.ExecutionClassDeterministic},
|
|
{Binding: pipeline.ModuleBinding{LLMProfile: "beta"}, ExecutionClass: contracts.ExecutionClassLLMBacked},
|
|
}}},
|
|
Output: pipeline.ModuleBinding{LLMProfile: "output-profile"},
|
|
}
|
|
got := effectiveLLMProfileIDs(resolved)
|
|
want := []string{"alpha", "beta", "gamma", "zeta"}
|
|
if strings.Join(got, ",") != strings.Join(want, ",") {
|
|
t.Fatalf("effective profiles = %#v, want %#v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestRunSessionIDUsesExplicitValueOrSourceDocumentID(t *testing.T) {
|
|
for _, tt := range []struct {
|
|
name string
|
|
args []string
|
|
want string
|
|
}{
|
|
{name: "source document", want: "source"},
|
|
{name: "explicit trimmed value", args: []string{"--session-id", " explicit-session "}, want: "explicit-session"},
|
|
} {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
harness := newStateTestHarness()
|
|
args := append([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, tt.args...)
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions(args, &stdout, &stderr, harness.options())
|
|
if code != 0 || stderr.Len() != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
harness.mu.Lock()
|
|
sessions := append([]string(nil), harness.sessionIDs...)
|
|
harness.mu.Unlock()
|
|
if len(sessions) < 4 {
|
|
t.Fatalf("session IDs = %#v, want all prompt-facing module requests", sessions)
|
|
}
|
|
for _, session := range sessions {
|
|
if session != tt.want {
|
|
t.Fatalf("session IDs = %#v, want %q", sessions, tt.want)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunFactoryAndPreparationFailuresAreProcessFailures(t *testing.T) {
|
|
t.Run("LLM factory", func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
opts := newStateTestHarness().options()
|
|
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
|
return nil, nil, errors.New("injected LLM factory failure")
|
|
}
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
|
|
if code != 1 || !strings.Contains(stderr.String(), "injected LLM factory failure") || stdout.Len() != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
|
|
t.Run("pipeline preparation", func(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
data, err := os.ReadFile(roots.config)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
data = []byte(replaceRequiredOnce(t, string(data), "extract: test/extract", "extract: test/failing-extract"))
|
|
if err := os.WriteFile(roots.config, data, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
opts := newStateTestHarness().options()
|
|
if err := pipeline.RegisterExtractorBuilder(opts.Registries.Extractors, pipeline.ModuleSpec{Key: "test/failing-extract", Stage: pipeline.StageExtract, Requires: []string{"chunks"}, Provides: []string{"artifact"}, ArtifactKind: stateTestArtifactKind}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.Extractor[stateTestArtifact], error) {
|
|
return nil, errors.New("injected extractor construction failure")
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
opts.Catalog = catalogFromRegistries(opts.Registries)
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass"}, &stdout, &stderr, opts)
|
|
if code != 1 || !strings.Contains(stderr.String(), "injected extractor construction failure") || stdout.Len() != 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestRunWarningsRemainSuccessfulAndReachDurableSurfaces(t *testing.T) {
|
|
roots := newStateTestRoots(t)
|
|
harness := newStateTestHarness()
|
|
harness.includeWarnings = true
|
|
harness.chunkWarnings = []contracts.Warning{{Scope: "chunk", ReasonCode: "contract-warning", Message: "warning retained"}}
|
|
var stdout, stderr bytes.Buffer
|
|
code := RunWithOptions([]string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}, &stdout, &stderr, harness.options())
|
|
if code != 0 || !strings.Contains(stdout.String(), "outputs=1") || !strings.Contains(stderr.String(), "1 warning(s)") {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
outputPath := filepath.Join(onlyChildDir(t, roots.output), "result.json")
|
|
output, err := os.ReadFile(outputPath)
|
|
if err != nil || !strings.Contains(string(output), "contract-warning") {
|
|
t.Fatalf("durable output = %q, %v", output, err)
|
|
}
|
|
bundle := onlyChildDir(t, roots.debug)
|
|
var warnings []contracts.Warning
|
|
readStateTestSummaryJSON(t, bundle, "warnings.json", &warnings)
|
|
if len(warnings) != 1 || warnings[0].ReasonCode != "contract-warning" {
|
|
t.Fatalf("debug warnings = %#v", warnings)
|
|
}
|
|
}
|
|
|
|
func runWithStateRoots(t *testing.T, roots stateTestRoots, opts Options, extra []string) stateTestResult {
|
|
t.Helper()
|
|
args := []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--chunk_cache", "bypass", "--debug"}
|
|
args = append(args, extra...)
|
|
var stdout, stderr bytes.Buffer
|
|
return stateTestResult{code: RunWithOptions(args, &stdout, &stderr, opts), stdout: stdout.String(), stderr: stderr.String()}
|
|
}
|
|
|
|
func lookupRunContractEnv(values map[string]string) func(string) (string, bool) {
|
|
return func(name string) (string, bool) {
|
|
value, ok := values[name]
|
|
return value, ok
|
|
}
|
|
}
|
|
|
|
func prependRunContractConfig(t *testing.T, roots stateTestRoots, prefix string) {
|
|
t.Helper()
|
|
data, err := os.ReadFile(roots.config)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(roots.config, append([]byte(prefix), data...), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func writeRunContractProfiles(t *testing.T, ids ...string) string {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
for _, id := range ids {
|
|
profile := fmt.Sprintf("id: %s\nendpoint: http://127.0.0.1:1/v1\nmodel: %s-model\n", id, id)
|
|
if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(profile), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return dir
|
|
}
|
|
|
|
func registerRunContractValidator(t *testing.T, opts *Options, profiles *[]string) {
|
|
t.Helper()
|
|
if err := pipeline.RegisterTypedValidatorBuilder(opts.Registries.Validators, stateTestArtifactKind, pipeline.ValidatorSpec{Key: "run-contract-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}, func(map[string]any) error { return nil }, func(pipeline.BuildRequest) (contracts.TypedValidator[stateTestArtifact], error) {
|
|
return runContractValidator{profiles: profiles}, nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := opts.Registries.ValidatorChains.Register(pipeline.ValidatorChainMapping{Stage: pipeline.StageExtract, Module: "test/extract", Validators: []pipeline.ModuleBinding{{Module: "run-contract-validator", LLMProfile: "validator-profile"}}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
opts.Catalog = catalogFromRegistries(opts.Registries)
|
|
}
|
|
|
|
type runContractValidator struct {
|
|
profiles *[]string
|
|
}
|
|
|
|
func (v runContractValidator) Name() string { return "run-contract-validator" }
|
|
|
|
func (v runContractValidator) ExecutionClass() contracts.ExecutionClass {
|
|
return contracts.ExecutionClassLLMBacked
|
|
}
|
|
|
|
func (v runContractValidator) Validate(_ context.Context, req contracts.TypedValidationRequest[stateTestArtifact]) (contracts.ValidationResult, error) {
|
|
*v.profiles = append(*v.profiles, req.LLMProfile)
|
|
return contracts.ValidationResult{Approved: true}, nil
|
|
}
|