210 lines
10 KiB
Go
210 lines
10 KiB
Go
package ckprompt
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"strings"
|
|
"testing"
|
|
|
|
"go.maximumdirect.net/ckconfig"
|
|
"go.maximumdirect.net/commandkit"
|
|
"go.maximumdirect.net/configkit"
|
|
"go.maximumdirect.net/promptkit"
|
|
)
|
|
|
|
func TestPromptConfigurationFailureCallbacks(t *testing.T) {
|
|
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("prompt-config", []byte("broken: [")))
|
|
var native *configkit.LoadError
|
|
if !errors.As(loadErr, &native) {
|
|
t.Fatalf("fixture: %v", loadErr)
|
|
}
|
|
wrapped := fmt.Errorf("load: %w", loadErr)
|
|
diag := commandkit.Explanation{Message: "configuration diagnostic"}
|
|
suite := Suite[struct{}, struct{}]{
|
|
ListPrompts: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]promptkit.PromptIdentity], error) {
|
|
return commandkit.Acquired[[]promptkit.PromptIdentity]{Explanation: diag}, wrapped
|
|
},
|
|
ListProfiles: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]string], error) {
|
|
return commandkit.Acquired[[]string]{Explanation: diag}, wrapped
|
|
},
|
|
ListBackends: func(context.Context, CatalogRequest, struct{}, struct{}) (commandkit.Acquired[[]string], error) {
|
|
return commandkit.Acquired[[]string]{Explanation: diag}, wrapped
|
|
},
|
|
ListNamespaces: func(context.Context, NamespaceRequest, struct{}, struct{}) (commandkit.Acquired[[]string], error) {
|
|
return commandkit.Acquired[[]string]{Explanation: diag}, wrapped
|
|
},
|
|
ListCollections: func(context.Context, ckconfig.Selection, struct{}, struct{}) (commandkit.Acquired[[]promptkit.CollectionInfo], error) {
|
|
return commandkit.Acquired[[]promptkit.CollectionInfo]{Explanation: diag}, wrapped
|
|
},
|
|
InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
|
|
return commandkit.Acquired[*promptkit.PromptInspection]{Explanation: diag}, wrapped
|
|
},
|
|
InspectProfile: func(context.Context, ProfileRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.ProfileInspection], error) {
|
|
return commandkit.Acquired[*promptkit.ProfileInspection]{Explanation: diag}, wrapped
|
|
},
|
|
InspectBackend: func(context.Context, BackendRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.BackendInspection], error) {
|
|
return commandkit.Acquired[*promptkit.BackendInspection]{Explanation: diag}, wrapped
|
|
},
|
|
InspectRequirements: func(context.Context, RequirementsRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.RequirementInspection], error) {
|
|
return commandkit.Acquired[*promptkit.RequirementInspection]{Explanation: diag}, wrapped
|
|
},
|
|
ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
|
|
return commandkit.Acquired[struct{}]{Explanation: diag}, wrapped
|
|
},
|
|
PrepareRequest: func(context.Context, PreparationRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PreparedRun], error) {
|
|
return commandkit.Acquired[*promptkit.PreparedRun]{Explanation: diag}, wrapped
|
|
},
|
|
}
|
|
app := newPromptTestApp(t, Command(suite))
|
|
for _, args := range [][]string{{"promptkit", "list", "prompts"}, {"promptkit", "list", "profiles"}, {"promptkit", "list", "backends"}, {"promptkit", "list", "namespaces"}, {"promptkit", "list", "collections"}, {"promptkit", "show", "prompt", "example"}, {"promptkit", "show", "profile", "example"}, {"promptkit", "show", "backend", "example"}, {"promptkit", "requirements", "show", "example"}, {"promptkit", "requirements", "validate", "example"}, {"promptkit", "show", "prepared", "example"}} {
|
|
t.Run(strings.Join(args, "_"), func(t *testing.T) {
|
|
var out, diagnostic bytes.Buffer
|
|
result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostic})
|
|
var got *configkit.LoadError
|
|
if result.Kind != commandkit.OutcomeUsage || result.Status != 2 || !errors.Is(result.Cause, loadErr) || !errors.As(result.Cause, &got) || got != native || !strings.Contains(diagnostic.String(), "configuration diagnostic") {
|
|
t.Fatalf("outcome=%#v diagnostics=%q", result, diagnostic.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPromptNativeConfigurationFailures(t *testing.T) {
|
|
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("native-config", []byte("broken: [")))
|
|
load := func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
|
return nil, fmt.Errorf("wrapped: %w", loadErr)
|
|
}
|
|
suite := FromEngine[struct{}, struct{}](load)
|
|
suite = WithRequestValidation(suite, load, nil)
|
|
suite = WithRequestPreparation(suite, load, nil)
|
|
app := newPromptTestApp(t, Command(suite))
|
|
for _, args := range [][]string{{"promptkit", "list", "prompts"}, {"promptkit", "show", "prompt", "example"}, {"promptkit", "requirements", "show", "example"}, {"promptkit", "requirements", "validate", "example"}, {"promptkit", "show", "prepared", "example"}} {
|
|
t.Run(strings.Join(args, "_"), func(t *testing.T) {
|
|
var diag bytes.Buffer
|
|
result := app.Execute(context.Background(), args, commandkit.Streams{Out: io.Discard, Err: &diag})
|
|
var native *configkit.LoadError
|
|
if result.Status != 2 || !errors.Is(result.Cause, loadErr) || !errors.As(result.Cause, &native) || !strings.Contains(diag.String(), "native-config") {
|
|
t.Fatalf("outcome=%#v diagnostics=%q", result, diag.String())
|
|
}
|
|
})
|
|
}
|
|
|
|
}
|
|
|
|
func TestBuilderConfigurationFailureDiagnostics(t *testing.T) {
|
|
_, malformed := configkit.Load[map[string]any](configkit.Bytes("builder-config", []byte("broken: [")))
|
|
path := configkit.Keys("version")
|
|
_, rejected := configkit.Load(configkit.Bytes("builder-config", []byte("version: 1\n")), configkit.Options[map[string]any]{
|
|
Definition: configkit.Definition[map[string]any]{Version: &configkit.VersionRule{
|
|
Path: path,
|
|
Check: func(_ configkit.Version, validation *configkit.Validation) error {
|
|
validation.Warning(path, "builder warning")
|
|
validation.Error(path, "builder rejection")
|
|
return nil
|
|
},
|
|
}},
|
|
})
|
|
for _, fixture := range []struct {
|
|
name string
|
|
cause error
|
|
code string
|
|
warning string
|
|
status int
|
|
}{
|
|
{"malformed", malformed, "yaml.invalid", "", 2},
|
|
{"rejected", rejected, "version.error", "builder warning", 2},
|
|
{"application failure", errors.New("builder unavailable"), "", "", 1},
|
|
} {
|
|
if fixture.cause == nil {
|
|
t.Fatalf("%s fixture succeeded", fixture.name)
|
|
}
|
|
for _, args := range [][]string{
|
|
{"promptkit", "requirements", "validate", "example"},
|
|
{"promptkit", "show", "prepared", "example"},
|
|
{"promptkit", "test", "reachability"},
|
|
{"promptkit", "test", "all"},
|
|
} {
|
|
for _, canceled := range []bool{false, true} {
|
|
name := fmt.Sprintf("%s/%s/canceled=%t", fixture.name, strings.Join(args, "_"), canceled)
|
|
t.Run(name, func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
loaderCalls, builderCalls := 0, 0
|
|
load := func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
|
|
loaderCalls++
|
|
return nil, errors.New("unexpected engine load")
|
|
}
|
|
wrapped := fmt.Errorf("builder load: %w", fixture.cause)
|
|
buildError := func() error {
|
|
builderCalls++
|
|
if canceled {
|
|
cancel()
|
|
}
|
|
return wrapped
|
|
}
|
|
suite := WithRequestValidation(Suite[struct{}, struct{}]{}, load, func(context.Context, ValidationRequest, struct{}, struct{}) (promptkit.RunRequest, error) {
|
|
return promptkit.RunRequest{}, buildError()
|
|
})
|
|
suite = WithRequestPreparation(suite, load, func(context.Context, PreparationRequest, struct{}, struct{}) (promptkit.RunRequest, error) {
|
|
return promptkit.RunRequest{}, buildError()
|
|
})
|
|
suite = WithSmokeTesting(suite, load, func(context.Context, SmokeRequest, struct{}, struct{}) (promptkit.SmokeTestRequest, error) {
|
|
return promptkit.SmokeTestRequest{}, buildError()
|
|
})
|
|
app := newPromptTestApp(t, Command(suite))
|
|
var out, diagnostic bytes.Buffer
|
|
result := app.Execute(ctx, append(append([]string{}, args...), "--verbose"), commandkit.Streams{Out: &out, Err: &diagnostic})
|
|
wantStatus, wantCalls := fixture.status, 1
|
|
if args[len(args)-1] == "all" && !canceled {
|
|
wantCalls = 4
|
|
}
|
|
if canceled {
|
|
wantStatus = 1
|
|
if !errors.Is(result.Cause, context.Canceled) {
|
|
t.Fatalf("lost cancellation: %#v", result)
|
|
}
|
|
}
|
|
if result.Status != wantStatus || !errors.Is(result.Cause, wrapped) || loaderCalls != 0 || builderCalls != wantCalls {
|
|
t.Fatalf("result=%#v loader calls=%d builder calls=%d", result, loaderCalls, builderCalls)
|
|
}
|
|
if fixture.code != "" {
|
|
var expected, retained *configkit.LoadError
|
|
if !errors.As(fixture.cause, &expected) || !errors.As(result.Cause, &retained) || retained != expected {
|
|
t.Fatalf("lost loading error: %#v", result)
|
|
}
|
|
if !strings.Contains(diagnostic.String(), fixture.code) || !strings.Contains(diagnostic.String(), "builder-config") || !strings.Contains(diagnostic.String(), fixture.warning) {
|
|
t.Fatalf("diagnostics: %q", diagnostic.String())
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPromptConfigurationFailureCancellation(t *testing.T) {
|
|
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("invalid", []byte("broken: [")))
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
calls := 0
|
|
suite := Suite[struct{}, struct{}]{InspectPrompt: func(context.Context, PromptRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.PromptInspection], error) {
|
|
calls++
|
|
cancel()
|
|
return commandkit.Acquired[*promptkit.PromptInspection]{}, loadErr
|
|
}}
|
|
app := newPromptTestApp(t, Command(suite))
|
|
var out bytes.Buffer
|
|
result := app.Execute(ctx, []string{"promptkit", "show", "prompt", "example"}, commandkit.Streams{Out: &out, Err: io.Discard})
|
|
if result.Status != 1 || !errors.Is(result.Cause, context.Canceled) || !errors.Is(result.Cause, loadErr) || out.Len() != 0 || calls != 1 {
|
|
t.Fatalf("outcome=%#v calls=%d output=%q", result, calls, out.String())
|
|
}
|
|
ctx, cancel = context.WithCancel(context.Background())
|
|
cancel()
|
|
calls = 0
|
|
result = app.Execute(ctx, []string{"promptkit", "show", "prompt", "example"}, commandkit.Streams{Out: &out, Err: io.Discard})
|
|
if result.Status != 1 || !errors.Is(result.Cause, context.Canceled) || calls != 0 || out.Len() != 0 {
|
|
t.Fatalf("pre-canceled=%#v calls=%d", result, calls)
|
|
}
|
|
}
|