Verify PromptKit smoke outcomes and delivery
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# Dependency upgrades, discovery, and smoke testing implementation plan
|
||||
|
||||
Status: Stages 1–13 complete; Stage 14 is next.
|
||||
Status: Stages 1–14 complete; Stage 15 is next.
|
||||
|
||||
## Authority and baseline
|
||||
|
||||
@@ -694,6 +694,14 @@ provider smoke-test correctness belongs upstream.
|
||||
**Complete when:** realistic smoke failures preserve their reports, native error
|
||||
identity, and delivery outcomes without a parallel diagnostic test suite.
|
||||
|
||||
**Completed:** 2026-10-17. Added outcome and delivery coverage for false/unknown
|
||||
observations, error-only and report-plus-error results, cancellation before and
|
||||
after acquisition, and retained aggregate results across cancellation. Tests
|
||||
also cover error and short report writes, encoding failure, diagnostic writer
|
||||
failure, operation plus delivery errors, ordered diagnostics, and joined
|
||||
`errors.Is`/`errors.As` identity. The full ckprompt suite passed with
|
||||
`GOTOOLCHAIN=local go test -count=1 ./...`.
|
||||
|
||||
## Stage 15: Demonstrate the new interfaces in the maintained consumer
|
||||
|
||||
**Primary files:** `integrations/ckprompt/examples/composed`, its README,
|
||||
|
||||
175
integrations/ckprompt/smoke_outcome_test.go
Normal file
175
integrations/ckprompt/smoke_outcome_test.go
Normal file
@@ -0,0 +1,175 @@
|
||||
package ckprompt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
commandkit "gitea.maximumdirect.net/eric/CommandKit"
|
||||
promptkit "gitea.maximumdirect.net/eric/promptkit"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
func TestSmokeTestFindingsDoNotChangeSuccessfulOutcome(t *testing.T) {
|
||||
falseValue := false
|
||||
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
|
||||
SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
||||
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{
|
||||
Kind: promptkit.SmokeTestReachability, Supported: false, Applicable: nil, Completed: false,
|
||||
Facts: promptkit.SmokeTestFacts{EndpointReachable: &falseValue},
|
||||
}}, nil
|
||||
},
|
||||
}))
|
||||
var output bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "reachability"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Cause != nil {
|
||||
t.Fatalf("false and unknown findings changed outcome: %#v", outcome)
|
||||
}
|
||||
var report struct {
|
||||
Supported bool `yaml:"supported"`
|
||||
Applicable *bool `yaml:"applicable"`
|
||||
Completed bool `yaml:"completed"`
|
||||
Facts struct {
|
||||
EndpointReachable *bool `yaml:"endpoint_reachable"`
|
||||
} `yaml:"facts"`
|
||||
}
|
||||
if err := yaml.Unmarshal(output.Bytes(), &report); err != nil {
|
||||
t.Fatalf("decode smoke findings %q: %v", output.String(), err)
|
||||
}
|
||||
if report.Supported || report.Applicable != nil || report.Completed || report.Facts.EndpointReachable == nil || *report.Facts.EndpointReachable {
|
||||
t.Fatalf("native findings were not retained: %#v", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeTestCancellationAfterAcquisitionStillDeliversReport(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
calls := 0
|
||||
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
|
||||
SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
||||
calls++
|
||||
cancel()
|
||||
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication}}, nil
|
||||
},
|
||||
}))
|
||||
var output bytes.Buffer
|
||||
outcome := app.Execute(ctx, []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || calls != 1 || output.Len() == 0 {
|
||||
t.Fatalf("post-acquisition cancellation = (%#v, %q), calls=%d", outcome, output.String(), calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeTestCancellationBeforeAcquisitionSkipsProvider(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
calls, stateCalls := 0, 0
|
||||
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
|
||||
NewState: func() struct{} { stateCalls++; return struct{}{} },
|
||||
SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
||||
calls++
|
||||
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{}}, nil
|
||||
},
|
||||
}))
|
||||
var output bytes.Buffer
|
||||
outcome := app.Execute(ctx, []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || calls != 0 || stateCalls != 0 || output.Len() != 0 {
|
||||
t.Fatalf("pre-acquisition cancellation = (%#v, %q), calls/state=%d/%d", outcome, output.String(), calls, stateCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSmokeTestEncodingAndWriterFailuresRetainOutcomeFields(t *testing.T) {
|
||||
operationErr := errors.New("native operation failed")
|
||||
diagnostic := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
|
||||
_, err := io.WriteString(writer, "smoke warning\n")
|
||||
return err
|
||||
}}
|
||||
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
|
||||
SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
||||
return commandkit.Acquired[*promptkit.SmokeTestReport]{
|
||||
Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication},
|
||||
Diagnostics: []commandkit.Diagnostic{diagnostic},
|
||||
}, operationErr
|
||||
},
|
||||
}))
|
||||
writeErr := errors.New("smoke report writer failed")
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
writer io.Writer
|
||||
want error
|
||||
}{
|
||||
{name: "writer error", writer: partialReportWriter{err: writeErr}, want: writeErr},
|
||||
{name: "short write", writer: partialReportWriter{}, want: io.ErrShortWrite},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var diagnosticOutput bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput})
|
||||
warningIndex := strings.Index(diagnosticOutput.String(), "smoke warning")
|
||||
operationIndex := strings.Index(diagnosticOutput.String(), operationErr.Error())
|
||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.DeliveryError, test.want) || !outcome.ReportIncomplete || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || warningIndex < 0 || operationIndex < warningIndex {
|
||||
t.Fatalf("report delivery = %#v, diagnostics=%q", outcome, diagnosticOutput.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
diagnosticErr := errors.New("smoke diagnostic writer failed")
|
||||
var output bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: partialReportWriter{err: diagnosticErr}})
|
||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.DeliveryError, diagnosticErr) || outcome.ReportIncomplete || outcome.DiagnosticDelivery != commandkit.DeliveryIncomplete || output.Len() == 0 {
|
||||
t.Fatalf("diagnostic delivery = (%#v, %q)", outcome, output.String())
|
||||
}
|
||||
|
||||
encodeApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
|
||||
SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
||||
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{
|
||||
Target: promptkit.SmokeTestTarget{EffectiveSettings: promptkit.ExecutionTarget{ExtraParams: map[string]any{"unsupported": func() {}}}},
|
||||
}}, nil
|
||||
},
|
||||
}))
|
||||
output.Reset()
|
||||
outcome = encodeApp.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil || !errors.Is(outcome.DeliveryError, outcome.Cause) || outcome.ReportIncomplete || output.Len() != 0 {
|
||||
t.Fatalf("encoding failure = (%#v, %q)", outcome, output.String())
|
||||
}
|
||||
}
|
||||
|
||||
type smokeTypedError struct{ text string }
|
||||
|
||||
func (err *smokeTypedError) Error() string { return err.text }
|
||||
|
||||
func TestSmokeAggregatePreservesMultipleTypedCauses(t *testing.T) {
|
||||
firstErr := &smokeTypedError{text: "reachability failed"}
|
||||
secondErr := &smokeTypedError{text: "authentication failed"}
|
||||
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
|
||||
SmokeTest: func(_ context.Context, request SmokeRequest, _ struct{}, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
|
||||
var err error
|
||||
switch request.Request.Kind {
|
||||
case promptkit.SmokeTestReachability:
|
||||
err = firstErr
|
||||
case promptkit.SmokeTestAuthentication:
|
||||
err = secondErr
|
||||
}
|
||||
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, err
|
||||
},
|
||||
}))
|
||||
var output bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard})
|
||||
var retained *smokeTypedError
|
||||
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, firstErr) || !errors.Is(outcome.Cause, secondErr) || !errors.As(outcome.Cause, &retained) || retained != firstErr {
|
||||
t.Fatalf("joined aggregate errors lost identity: %#v, retained=%#v", outcome, retained)
|
||||
}
|
||||
var document struct {
|
||||
Tests []struct {
|
||||
Kind string `yaml:"kind"`
|
||||
Error string `yaml:"error"`
|
||||
} `yaml:"tests"`
|
||||
}
|
||||
if err := yaml.Unmarshal(output.Bytes(), &document); err != nil {
|
||||
t.Fatalf("decode aggregate errors %q: %v", output.String(), err)
|
||||
}
|
||||
if len(document.Tests) != 4 || document.Tests[0].Kind != string(promptkit.SmokeTestReachability) || document.Tests[0].Error != firstErr.Error() || document.Tests[1].Kind != string(promptkit.SmokeTestAuthentication) || document.Tests[1].Error != secondErr.Error() || document.Tests[2].Error != "" || document.Tests[3].Error != "" {
|
||||
t.Fatalf("aggregate error entries = %#v", document.Tests)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user