252 lines
10 KiB
Go
252 lines
10 KiB
Go
package ckprompt
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"go.maximumdirect.net/ckconfig"
|
|
commandkit "go.maximumdirect.net/commandkit"
|
|
promptkit "go.maximumdirect.net/promptkit"
|
|
"go.yaml.in/yaml/v4"
|
|
)
|
|
|
|
func TestSmokeAggregateCoordinatesConcreteRequestsAndRetainsErrors(t *testing.T) {
|
|
type input struct{ identity *int }
|
|
type state struct{ identity *int }
|
|
|
|
inputIdentity, stateIdentity := 1, 2
|
|
wantKinds := []promptkit.SmokeTestKind{
|
|
promptkit.SmokeTestReachability,
|
|
promptkit.SmokeTestAuthentication,
|
|
promptkit.SmokeTestTextInference,
|
|
promptkit.SmokeTestStructuredOutput,
|
|
}
|
|
nativeErr := errors.New("authentication smoke failed")
|
|
newInputCalls, checkCalls, newStateCalls := 0, 0, 0
|
|
var gotKinds []promptkit.SmokeTestKind
|
|
var firstInput *input
|
|
var sharedState *state
|
|
suite := Suite[*input, *state]{
|
|
NewInput: func() *input {
|
|
newInputCalls++
|
|
return &input{identity: &inputIdentity}
|
|
},
|
|
NewState: func() *state {
|
|
newStateCalls++
|
|
return &state{identity: &stateIdentity}
|
|
},
|
|
CheckInput: func(_ ckconfig.Selection, value *input) error {
|
|
checkCalls++
|
|
firstInput = value
|
|
return nil
|
|
},
|
|
SmokeTest: func(_ context.Context, request SmokeRequest, value *input, shared *state) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
|
gotKinds = append(gotKinds, request.Request.Kind)
|
|
if firstInput != value || value.identity != &inputIdentity || shared.identity != &stateIdentity {
|
|
t.Errorf("input/state were not shared: input=%p state=%#v", value, shared)
|
|
}
|
|
if sharedState == nil {
|
|
sharedState = shared
|
|
} else if sharedState != shared {
|
|
t.Errorf("state pointer changed between smoke tests: first=%p current=%p", sharedState, shared)
|
|
}
|
|
if request.Request.Execution == nil || request.Request.Execution.Model != "base-model" || request.Request.Execution.Temperature == nil || *request.Request.Execution.Temperature != 0.25 || request.Request.TransientRetries == nil || *request.Request.TransientRetries != 3 {
|
|
t.Errorf("seed for %s = %#v", request.Request.Kind, request.Request)
|
|
}
|
|
if len(gotKinds) == 1 {
|
|
request.Request.Execution.Model = "builder mutation"
|
|
*request.Request.Execution.Temperature = 0.9
|
|
*request.Request.TransientRetries = 11
|
|
}
|
|
result := &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication, Supported: true, Completed: true}
|
|
if request.Request.Kind == promptkit.SmokeTestAuthentication {
|
|
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: result}, nativeErr
|
|
}
|
|
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: result}, nil
|
|
},
|
|
}
|
|
app := newPromptTestApp(t, Command(suite))
|
|
var output bytes.Buffer
|
|
outcome := app.Execute(context.Background(), []string{
|
|
"promptkit", "test", "all", "--llm-model=base-model", "--llm-temperature=0.25", "--llm-transient-retries=3",
|
|
}, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, nativeErr) {
|
|
t.Fatalf("aggregate outcome = %#v", outcome)
|
|
}
|
|
if newInputCalls != 1 || checkCalls != 1 || newStateCalls != 1 || !reflect.DeepEqual(gotKinds, wantKinds) {
|
|
t.Fatalf("invocation calls = input %d, check %d, state %d, kinds %#v", newInputCalls, checkCalls, newStateCalls, gotKinds)
|
|
}
|
|
|
|
var document struct {
|
|
Tests []struct {
|
|
Kind string `yaml:"kind"`
|
|
Report *struct {
|
|
Kind string `yaml:"kind"`
|
|
} `yaml:"report"`
|
|
Error string `yaml:"error"`
|
|
} `yaml:"tests"`
|
|
}
|
|
if err := yaml.Unmarshal(output.Bytes(), &document); err != nil {
|
|
t.Fatalf("decode aggregate report %q: %v", output.String(), err)
|
|
}
|
|
if len(document.Tests) != len(wantKinds) {
|
|
t.Fatalf("aggregate entries = %#v", document.Tests)
|
|
}
|
|
for index, test := range document.Tests {
|
|
if test.Kind != string(wantKinds[index]) || test.Report == nil || test.Report.Kind != string(promptkit.SmokeTestAuthentication) {
|
|
t.Errorf("aggregate entry %d = %#v", index, test)
|
|
}
|
|
if index == 1 && test.Error != nativeErr.Error() {
|
|
t.Errorf("authentication entry error = %q", test.Error)
|
|
}
|
|
if index != 1 && test.Error != "" {
|
|
t.Errorf("unexpected entry error at %d: %q", index, test.Error)
|
|
}
|
|
}
|
|
}
|
|
|
|
type cancelProgressWriter struct {
|
|
bytes.Buffer
|
|
cancel context.CancelFunc
|
|
}
|
|
|
|
func (writer *cancelProgressWriter) Write(p []byte) (int, error) {
|
|
writer.cancel()
|
|
return writer.Buffer.Write(p)
|
|
}
|
|
|
|
func TestSmokeAggregateCancellationDuringProgress(t *testing.T) {
|
|
providerErr := errors.New("provider failed")
|
|
for _, test := range []struct {
|
|
name string
|
|
err error
|
|
cancelAt int
|
|
}{
|
|
{name: "partial success", cancelAt: 1},
|
|
{name: "partial provider failure", err: providerErr, cancelAt: 1},
|
|
{name: "last completed test", cancelAt: 4},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
var kinds []promptkit.SmokeTestKind
|
|
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
|
|
SmokeTest: func(_ context.Context, request SmokeRequest, _, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
|
kinds = append(kinds, request.Request.Kind)
|
|
acquired := commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind, Completed: true}}
|
|
if len(kinds) == test.cancelAt {
|
|
acquired.Diagnostics = []commandkit.Diagnostic{{Severity: commandkit.SeverityProgress, Write: func(w io.Writer) error { _, err := io.WriteString(w, "progress evidence\n"); return err }}}
|
|
acquired.Explanation = commandkit.Explanation{Message: "provider explanation"}
|
|
return acquired, test.err
|
|
}
|
|
return acquired, nil
|
|
},
|
|
}))
|
|
var output bytes.Buffer
|
|
diagnostics := &cancelProgressWriter{cancel: cancel}
|
|
outcome := app.Execute(ctx, []string{"promptkit", "test", "all", "--progress", "--format=json"}, commandkit.Streams{Out: &output, Err: diagnostics})
|
|
if len(kinds) != test.cancelAt || !strings.Contains(diagnostics.String(), "progress evidence") {
|
|
t.Fatalf("kinds=%v diagnostics=%q", kinds, diagnostics.String())
|
|
}
|
|
if test.cancelAt < 4 {
|
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || (test.err != nil && (!errors.Is(outcome.Cause, test.err) || !strings.Contains(diagnostics.String(), "provider explanation"))) {
|
|
t.Fatalf("partial outcome=%#v diagnostics=%q", outcome, diagnostics.String())
|
|
}
|
|
} else if outcome.Kind != commandkit.OutcomeSuccess || outcome.Cause != nil {
|
|
t.Fatalf("completed outcome=%#v", outcome)
|
|
}
|
|
var report struct {
|
|
Tests []struct {
|
|
Kind string `json:"kind"`
|
|
Report *struct {
|
|
Completed bool `json:"completed"`
|
|
} `json:"report"`
|
|
} `json:"tests"`
|
|
}
|
|
if err := json.Unmarshal(output.Bytes(), &report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(report.Tests) != len(kinds) {
|
|
t.Fatalf("report=%s", output.String())
|
|
}
|
|
for i, entry := range report.Tests {
|
|
if entry.Kind != string(kinds[i]) || entry.Report == nil || !entry.Report.Completed {
|
|
t.Fatalf("report entry=%#v", entry)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSmokeAggregateCancellationRetainsPartialResults(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
t.Cleanup(cancel)
|
|
operationErr := errors.New("second smoke operation failed")
|
|
var gotKinds []promptkit.SmokeTestKind
|
|
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
|
|
SmokeTest: func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
|
gotKinds = append(gotKinds, request.Request.Kind)
|
|
if len(gotKinds) == 2 {
|
|
cancel()
|
|
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, operationErr
|
|
}
|
|
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil
|
|
},
|
|
}))
|
|
var output bytes.Buffer
|
|
outcome := app.Execute(ctx, []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.Cause, context.Canceled) {
|
|
t.Fatalf("partial aggregate outcome = %#v", outcome)
|
|
}
|
|
if !reflect.DeepEqual(gotKinds, []promptkit.SmokeTestKind{promptkit.SmokeTestReachability, promptkit.SmokeTestAuthentication}) {
|
|
t.Fatalf("callbacks after cancellation = %#v", gotKinds)
|
|
}
|
|
var document struct {
|
|
Tests []struct {
|
|
Kind string `yaml:"kind"`
|
|
} `yaml:"tests"`
|
|
}
|
|
if err := yaml.Unmarshal(output.Bytes(), &document); err != nil {
|
|
t.Fatalf("decode partial aggregate report %q: %v", output.String(), err)
|
|
}
|
|
if len(document.Tests) != 2 || document.Tests[0].Kind != string(promptkit.SmokeTestReachability) || document.Tests[1].Kind != string(promptkit.SmokeTestAuthentication) {
|
|
t.Fatalf("partial aggregate entries = %#v", document.Tests)
|
|
}
|
|
}
|
|
|
|
func TestSmokeAggregateIsolatesProviderParameters(t *testing.T) {
|
|
calls := 0
|
|
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
|
|
SmokeTest: func(_ context.Context, request SmokeRequest, _, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
|
calls++
|
|
params := request.Request.Execution.ExtraParams
|
|
value := params["nested"].(map[string]any)["array"].([]any)[0].(map[string]any)
|
|
if value["number"] != int64(1) {
|
|
t.Errorf("call %d: %#v", calls, value)
|
|
}
|
|
value["number"] = int64(9)
|
|
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, nil
|
|
},
|
|
}))
|
|
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all", `--llm-extra-params={"nested":{"array":[{"number":1}]}}`}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
|
if outcome.Kind != commandkit.OutcomeSuccess || calls != 4 {
|
|
t.Fatalf("outcome=%#v calls=%d", outcome, calls)
|
|
}
|
|
}
|
|
|
|
func TestCopyProviderParamsPreservesEmptyAndNilValues(t *testing.T) {
|
|
if copyProviderParams(nil) != nil {
|
|
t.Fatal("nil object became non-nil")
|
|
}
|
|
copied := copyProviderParams(map[string]any{"emptyObject": map[string]any{}, "nilObject": map[string]any(nil), "emptyArray": []any{}, "nilArray": []any(nil), "null": nil, "decimal": float64(1.5)})
|
|
if copied["emptyObject"].(map[string]any) == nil || copied["nilObject"].(map[string]any) != nil || copied["emptyArray"].([]any) == nil || copied["nilArray"].([]any) != nil || copied["null"] != nil || copied["decimal"] != float64(1.5) {
|
|
t.Fatalf("copied = %#v", copied)
|
|
}
|
|
}
|