Verify smoke and composed configuration outcomes

This commit is contained in:
2026-11-16 18:06:45 +00:00
parent 2bd216f0d3
commit 20d2924338
4 changed files with 109 additions and 3 deletions

View File

@@ -252,7 +252,13 @@ packages using the aligned workspace.
## Stage 4 — Verify smoke aggregation and configuration composition
Status — pending.
Status — complete. Verified individual and aggregate smoke outcomes, mixed
failures and cancellation with retained reports and causes, and all five
operations through the composed PromptKit configuration scope, including the
right comparison side and pre-canceled invocations. Updated the existing
comparison and composed-example expectations for input diagnostics. With Go
1.27.0, `GOTOOLCHAIN=local GOPROXY=off go test ./integrations/ckprompt/...`
passed in the aligned workspace using a temporary Go build cache.
In `integrations/ckprompt/smoke_command.go`, complete the context argument updates
and apply the shared classification to individual failures and the final joined

View File

@@ -154,7 +154,7 @@ func TestPublicConfigDiffUsesPromptKitPairAndPreparedInputs(t *testing.T) {
var diagnostics bytes.Buffer
failure := app.Execute(context.Background(), []string{"config", "diff", "--scope", "promptkit", "--config", "bad-left.yml", "--against-config", "right.yml", "--campaign", "spring", "--against-campaign", "spring"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
if failure.Kind != commandkit.OutcomeOperational || !strings.Contains(diagnostics.String(), `Configuration scope "promptkit" (left):`) {
if failure.Kind != commandkit.OutcomeUsage || !strings.Contains(diagnostics.String(), `Configuration scope "promptkit" (left):`) {
t.Fatalf("installed load failure = %#v, diagnostics %q", failure, diagnostics.String())
}
var loadFailure *configkit.LoadError

View File

@@ -0,0 +1,100 @@
package ckprompt
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"strings"
"testing"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
func TestComposedConfigurationInputOutcomes(t *testing.T) {
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("prompt-invalid", []byte("broken: [")))
engine := adapterEngine(t)
calls := 0
scope := ConfigScope(func(_ context.Context, selection ckconfig.Selection, _, _ struct{}) (*promptkit.Engine, error) {
calls++
if selection.Config.Value == "good" {
return engine, nil
}
return nil, fmt.Errorf("wrapped: %w", loadErr)
})
app := newPromptTestApp(t, ckconfig.Command(ckconfig.Suite[struct{}, struct{}]{Scopes: []ckconfig.Scope[struct{}, struct{}]{scope}}))
cases := []struct {
args []string
report string
side string
}{
{[]string{"config", "show"}, "", ""},
{[]string{"config", "sources", "--format=json"}, `"sources"`, ""},
{[]string{"config", "validate", "--details", "--format=json"}, `"code": "yaml.invalid"`, ""},
{[]string{"config", "explain", "--list"}, "", ""},
{[]string{"config", "diff"}, "", "left"},
{[]string{"config", "diff", "--config=good", "--against-config=bad"}, "", "right"},
}
for _, tc := range cases {
t.Run(strings.Join(tc.args, "_"), func(t *testing.T) {
var out, diag bytes.Buffer
args := append(append([]string{}, tc.args...), "--scope=promptkit")
result := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diag})
var native *configkit.LoadError
if result.Status != 2 || !errors.Is(result.Cause, loadErr) || !errors.As(result.Cause, &native) || !strings.Contains(diag.String(), "prompt-invalid") || (tc.report != "" && !strings.Contains(out.String(), tc.report)) || (tc.side != "" && !strings.Contains(diag.String(), "("+tc.side+")")) {
t.Fatalf("result=%#v output=%q diagnostics=%q", result, out.String(), diag.String())
}
})
}
for _, tc := range cases {
ctx, cancel := context.WithCancel(context.Background())
cancel()
before := calls
var out bytes.Buffer
args := append(append([]string{}, tc.args...), "--scope=promptkit")
result := app.Execute(ctx, args, commandkit.Streams{Out: &out, Err: io.Discard})
if result.Status != 1 || !errors.Is(result.Cause, context.Canceled) || calls != before || out.Len() != 0 {
t.Fatalf("canceled %v: %#v output=%q", tc.args, result, out.String())
}
}
}
func TestSmokeConfigurationFailureOutcomes(t *testing.T) {
_, first := configkit.Load[map[string]any](configkit.Bytes("first", []byte("broken: [")))
_, second := configkit.Load[map[string]any](configkit.Bytes("second", []byte("other: [")))
operational := errors.New("provider failure")
for _, tc := range []struct {
name, command string
failures []error
cancel bool
want int
}{
{"single", "reachability", []error{first}, false, 2},
{"all input", "all", []error{first, second, first, second}, false, 2},
{"mixed", "all", []error{first, operational, second, first}, false, 1},
{"cancellation", "all", []error{first}, true, 1},
} {
t.Run(tc.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
calls := 0
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{SmokeTest: func(_ context.Context, req SmokeRequest, _, _ struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
err := tc.failures[calls]
calls++
if tc.cancel {
cancel()
}
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: req.Request.Kind}, Explanation: commandkit.Explanation{Message: fmt.Sprintf("context-%d", calls)}}, err
}}))
var out, diag bytes.Buffer
result := app.Execute(ctx, []string{"promptkit", "test", tc.command, "--format=json"}, commandkit.Streams{Out: &out, Err: &diag})
if result.Status != tc.want || calls != len(tc.failures) || !errors.Is(result.Cause, first) || (tc.name == "all input" && !errors.Is(result.Cause, second)) || (tc.name == "mixed" && (!errors.Is(result.Cause, second) || !errors.Is(result.Cause, operational))) || (tc.cancel && !errors.Is(result.Cause, context.Canceled)) || out.Len() == 0 || !strings.Contains(diag.String(), "context-1") {
t.Fatalf("outcome=%#v calls=%d report=%q diagnostics=%q", result, calls, out.String(), diag.String())
}
})
}
}

View File

@@ -170,7 +170,7 @@ func TestComposedIndependentOverridesAndFailureEvidence(t *testing.T) {
}
outcome = app.Execute(context.Background(), []string{"config", "validate", "--scope", "app", "--campaign", "spring", "--limit=0", "--set", `/default_campaign="autumn"`}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil {
if outcome.Kind != commandkit.OutcomeUsage || outcome.Cause == nil {
t.Fatalf("overlapping native overrides: %#v", outcome)
}
loads := 0