284 lines
10 KiB
Go
284 lines
10 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
)
|
|
|
|
func TestConfigKitReadOnlyCommandsLoadImportsAndOverlays(t *testing.T) {
|
|
t.Parallel()
|
|
root := t.TempDir()
|
|
path := filepath.Join(root, "notarius.yml")
|
|
writeConfigKitFile(t, filepath.Join(root, "fragment.yml"), "pipelines:\n imported:\n input: {module: input}\n")
|
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "pipelines:\n overlaid:\n input: {module: input}\n")
|
|
writeConfigKitFile(t, path, "version: 5\nconfigkit:\n imports:\n - path: fragment.yml\npipelines:\n base:\n input: {module: input}\n")
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions([]string{"pipelines", "list", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\noverlaid\n" {
|
|
t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "validate", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || !strings.Contains(stdout.String(), "is valid") {
|
|
t.Fatalf("validate = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
for _, selection := range [][]string{nil, {"--config-overlay", ""}, {"--config-overlay", "default"}} {
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
args := append([]string{"pipelines", "list", "--config", path}, selection...)
|
|
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\n" || stderr.Len() != 0 {
|
|
t.Fatalf("base list %q = %d, %q, %q", selection, code, stdout.String(), stderr.String())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPipelineInheritanceCommandsSelectAndInspectDerivedPipelines(t *testing.T) {
|
|
t.Parallel()
|
|
path := writeCommandConfigContent(t, `version: 5
|
|
pipelines:
|
|
" parent ":
|
|
input:
|
|
module: seriatim
|
|
artifacts:
|
|
spells:
|
|
extract:
|
|
module: dnd/spells
|
|
structured_output_repair_attempts: 2
|
|
child:
|
|
extends: " parent "
|
|
structured_output_repair_attempts: 0
|
|
`)
|
|
options := commandContractOptions(t)
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions([]string{"pipelines", "list", "--config", path}, &stdout, &stderr, options); code != 0 || stdout.String() != "child\nparent\n" || stderr.Len() != 0 {
|
|
t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "validate", "--config", path, "--pipeline", "child"}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), `is valid for pipeline "child"`) || stderr.Len() != 0 {
|
|
t.Fatalf("validate child = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "effective", "--config", path}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), "module: seriatim") || !strings.Contains(stdout.String(), "structured_output_repair_attempts: 0") || strings.Contains(stdout.String(), "extends:") || stderr.Len() != 0 {
|
|
t.Fatalf("effective = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--config", path, "/pipelines/child/input/module"}, &stdout, &stderr, options); code != 0 || !strings.Contains(stdout.String(), `path: "/pipelines/child/input/module"`) || !strings.Contains(stdout.String(), "parent") || stderr.Len() != 0 {
|
|
t.Fatalf("explain inherited input = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestPipelineInheritanceLoadFailuresUseRuntimeDiagnostics(t *testing.T) {
|
|
t.Parallel()
|
|
for _, test := range []struct {
|
|
name string
|
|
content string
|
|
args []string
|
|
want string
|
|
}{
|
|
{
|
|
name: "missing parent",
|
|
content: `version: 5
|
|
pipelines:
|
|
child:
|
|
extends: absent
|
|
`,
|
|
args: []string{"config", "validate", "--pipeline", "child"},
|
|
want: "inheritance.missing",
|
|
},
|
|
{
|
|
name: "cycle",
|
|
content: `version: 5
|
|
pipelines:
|
|
first:
|
|
extends: second
|
|
second:
|
|
extends: first
|
|
`,
|
|
args: []string{"config", "validate", "--pipeline", "first"},
|
|
want: "cycle",
|
|
},
|
|
{
|
|
name: "invalid directive type",
|
|
content: `version: 5
|
|
pipelines:
|
|
child:
|
|
extends: 7
|
|
`,
|
|
args: []string{"config", "validate", "--pipeline", "child"},
|
|
want: "extends",
|
|
},
|
|
{
|
|
name: "unselected missing parent",
|
|
content: `version: 5
|
|
pipelines:
|
|
parent:
|
|
input:
|
|
module: seriatim
|
|
child:
|
|
extends: parent
|
|
unrelated:
|
|
extends: absent
|
|
`,
|
|
args: []string{"config", "validate", "--pipeline", "child"},
|
|
want: "inheritance.missing",
|
|
},
|
|
{
|
|
name: "exact sibling mismatch",
|
|
content: `version: 5
|
|
pipelines:
|
|
parent:
|
|
input:
|
|
module: seriatim
|
|
child:
|
|
extends: "parent "
|
|
`,
|
|
args: []string{"pipelines", "list"},
|
|
want: "inheritance.missing",
|
|
},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
path := writeCommandConfigContent(t, test.content)
|
|
args := append([]string{}, test.args...)
|
|
args = append(args, "--config", path)
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 1 || stdout.Len() != 0 || stderr.Len() == 0 || !strings.Contains(stderr.String(), test.want) {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRetiredConfigProfileFlagUsesNormalUnknownFlagBehavior(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
for _, args := range [][]string{
|
|
{"run", "sample", "--config-profile=development"},
|
|
{"config", "validate", "--config-profile=development"},
|
|
{"config", "effective", "--config-profile=development"},
|
|
{"config", "explain", "--config-profile=development", ""},
|
|
{"pipelines", "list", "--config-profile=development"},
|
|
} {
|
|
t.Run(strings.Join(args[:2], " "), func(t *testing.T) {
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "not defined") {
|
|
t.Fatalf("args=%q code=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestConfigKitReadOnlyCommandsRejectVersionFour(t *testing.T) {
|
|
t.Parallel()
|
|
path := filepath.Join(t.TempDir(), "notarius.yml")
|
|
writeConfigKitFile(t, path, "version: 4\n")
|
|
var stdout, stderr bytes.Buffer
|
|
if code := runPipelinesList([]string{"--config", path}, &stdout, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "version") {
|
|
t.Fatalf("list = %d, %q", code, stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestConfigurationInspectionCommandsRenderSafeConfigKitViews(t *testing.T) {
|
|
t.Parallel()
|
|
root := t.TempDir()
|
|
path := filepath.Join(root, "notarius.yml")
|
|
writeConfigKitFile(t, path, `version: 5
|
|
concurrency:
|
|
total_llm: 4
|
|
pipelines:
|
|
"0":
|
|
input: {module: input}
|
|
`)
|
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "concurrency:\n total_llm: 3\n")
|
|
writeConfigKitFile(t, filepath.Join(root, "overlays", "retired.yml"), "pipelines: null\n")
|
|
|
|
providerCalled := false
|
|
opts := commandContractOptions(t)
|
|
opts.LLMClientFactory = func(context.Context, config.Config, string, LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
|
providerCalled = true
|
|
return nil, nil, errors.New("LLM client must not be constructed")
|
|
}
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
if code := RunWithOptions([]string{"config", "effective", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "total_llm: 3") || stderr.Len() != 0 {
|
|
t.Fatalf("effective = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
if providerCalled {
|
|
t.Fatal("config effective constructed an LLM client")
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-overlay", "development", "/concurrency/total_llm"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/concurrency/total_llm"`) || stderr.Len() != 0 {
|
|
t.Fatalf("explain effective = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--config", path, ""}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: ""`) || stderr.Len() != 0 {
|
|
t.Fatalf("explain root = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
|
|
stdout.Reset()
|
|
stderr.Reset()
|
|
if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-overlay", "retired", "/pipelines/0"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/pipelines/0"`) || stderr.Len() != 0 {
|
|
t.Fatalf("explain removed mapping key = %d, %q, %q", code, stdout.String(), stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestConfigExplainRejectsInvalidAndUnknownPointers(t *testing.T) {
|
|
t.Parallel()
|
|
path := writeCommandConfig(t, "session", "other")
|
|
for _, test := range []struct {
|
|
name string
|
|
pointer string
|
|
wantCode int
|
|
}{
|
|
{name: "missing", wantCode: 2},
|
|
{name: "malformed escape", pointer: "/pipelines~2session", wantCode: 2},
|
|
{name: "unknown", pointer: "/unknown", wantCode: 1},
|
|
{name: "invalid sequence index", pointer: "/pipelines/session/steps/+0", wantCode: 1},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
var stdout, stderr bytes.Buffer
|
|
args := []string{"config", "explain", "--config", path}
|
|
if test.name != "missing" {
|
|
args = append(args, test.pointer)
|
|
}
|
|
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != test.wantCode || stdout.Len() != 0 || stderr.Len() == 0 {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestConfigInspectionCommandsReportWriterFailures(t *testing.T) {
|
|
t.Parallel()
|
|
path := writeCommandConfig(t, "session", "other")
|
|
var stderr bytes.Buffer
|
|
if code := RunWithOptions([]string{"config", "effective", "--config", path}, failingInspectionWriter{}, &stderr, commandContractOptions(t)); code != 1 || !strings.Contains(stderr.String(), "write effective configuration") {
|
|
t.Fatalf("effective writer failure = %d, %q", code, stderr.String())
|
|
}
|
|
}
|
|
|
|
type failingInspectionWriter struct{}
|
|
|
|
func (failingInspectionWriter) Write([]byte) (int, error) {
|
|
return 0, io.ErrClosedPipe
|
|
}
|