base-layer/commandkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Verify native PromptKit request validation

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-12 22:15:06 +00:00
parent 7fc5423688
commit f5c1c69ccb
2 changed files with 245 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
docs/roadmap/implementation.md
View File

@@ -1,6 +1,6 @@
# PromptKit request validation implementation plan
Status: Stages 1 and 2 complete; implementation is ongoing.
Status: Stages 1–3 complete; implementation is ongoing.
## Authority, scope, and baseline
@@ -196,6 +196,17 @@ selected builder-before-loader lifecycle.
**Complete when:** the adapter's native authority, builder forwarding, failure
handling, and cancellation boundaries have meaningful public-boundary evidence.
**Completed:** 2026-10-12. Added real-engine adapter coverage for native success,
missing required input, invalid profile, and an unloadable file reference. The
builder test supplies a required inline artifact and verifies context, selection,
typed input, and state reach both callbacks; validation succeeds without a model
call. Tests cover nil engines, retained `*configkit.LoadError` diagnostics and
identity, cancellation before and after building, cancellation during and after
loading, and PromptKit's native artifact-load category when an artifact reader
returns cancellation. Focused adapter tests and
`GOTOOLCHAIN=local go test -count=1 ./...` passed from
`integrations/ckprompt`.
## Stage 4: Implement the validate command and requirements group composition
**Primary files:** `integrations/ckprompt/command.go`, a new private

233
integrations/ckprompt/validation_adapters_test.go
View File

@@ -5,12 +5,45 @@ import (
"errors"
"strings"
"testing"
"testing/fstest"
commandkit "gitea.maximumdirect.net/eric/CommandKit"
"gitea.maximumdirect.net/eric/CommandKit/integrations/ckconfig"
configkit "gitea.maximumdirect.net/eric/ConfigKit"
promptkit "gitea.maximumdirect.net/eric/promptkit"
)
type validationFailOnGenerate struct{ calls int }
func (client *validationFailOnGenerate) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.calls++
return nil, errors.New("request validation must not generate")
}
type validationCancelArtifactReader struct{ cancel context.CancelFunc }
func (reader validationCancelArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
reader.cancel()
return nil, context.Canceled
}
func validationTestEngine(t *testing.T, configuration string, options ...promptkit.Option) *promptkit.Engine {
t.Helper()
inputs, err := promptkit.FilesystemInputs("request validation fixture", fstest.MapFS{
"config/prompt.yml": &fstest.MapFile{Data: []byte(configuration)},
}, "config")
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: inputs}},
}}, options...)
if err != nil {
t.Fatalf("construct PromptKit validation fixture: %v", err)
}
return engine
}
func TestWithRequestValidationPreservesSuiteAndValidatesSeedWithoutBuilder(t *testing.T) {
type contextKey struct{}
type input struct{ value string }
@@ -98,3 +131,203 @@ func TestWithRequestValidationRetainsNilLoaderIssueIndependently(t *testing.T) {
t.Fatalf("installation did not report retained issue after capabilities were cleared: %v", err)
}
}
func TestWithRequestValidationDelegatesNativeOutcomesWithoutModelCalls(t *testing.T) {
client := &validationFailOnGenerate{}
engine := validationTestEngine(t, adapterPrompt, promptkit.WithLLMClient(client))
selection := ckconfig.Selection{Config: commandkit.Supplied[string]{Value: "selected.yml", Set: true}}
suite := WithRequestValidation(Suite[struct{}, struct{}]{}, func(_ context.Context, gotSelection ckconfig.Selection, _, _ struct{}) (*promptkit.Engine, error) {
if gotSelection != selection {
t.Errorf("loader selection = %#v, want %#v", gotSelection, selection)
}
return engine, nil
}, nil)
for _, test := range []struct {
name string
request promptkit.RunRequest
want error
}{
{name: "success", request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "default", Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.Inline("release planning")}}},
{name: "missing required input", request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "default"}, want: promptkit.ErrPromptRender},
{name: "invalid profile selection", request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "missing", Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.Inline("release planning")}}, want: promptkit.ErrProfileNotFound},
{name: "unloadable artifact", request: promptkit.RunRequest{PromptID: "briefing", ProfileID: "default", Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.File("missing-topic.txt")}}, want: promptkit.ErrArtifactLoad},
} {
t.Run(test.name, func(t *testing.T) {
nativeErr := engine.ValidateRequest(context.Background(), test.request)
acquired, adapterErr := suite.ValidateRequest(context.Background(), ValidationRequest{Selection: selection, Request: test.request}, struct{}{}, struct{}{})
if test.want == nil {
if nativeErr != nil || adapterErr != nil {
t.Fatalf("native error = %v, adapter error = %v", nativeErr, adapterErr)
}
if acquired.Value != (struct{}{}) {
t.Fatalf("successful acquisition = %#v", acquired)
}
return
}
if !errors.Is(nativeErr, test.want) || !errors.Is(adapterErr, test.want) {
t.Fatalf("native error = %v, adapter error = %v, want both to match %v", nativeErr, adapterErr, test.want)
}
})
}
if client.calls != 0 {
t.Fatalf("request validation made %d model calls", client.calls)
}
}
func TestWithRequestValidationBuilderOutputReachesNativeValidation(t *testing.T) {
type contextKey struct{}
type input struct{ campaign string }
type state struct{ token string }
client := &validationFailOnGenerate{}
engine := validationTestEngine(t, adapterPrompt, promptkit.WithLLMClient(client))
ctx := context.WithValue(context.Background(), contextKey{}, "validation invocation")
selection := ckconfig.Selection{Overlay: commandkit.Supplied[string]{Value: "campaign.yml", Set: true}}
seed := promptkit.RunRequest{PromptID: "briefing", ProfileID: "default", Vars: map[string]string{"topic": "seed topic"}}
wantInput, wantState := input{campaign: "spring"}, state{token: "shared"}
builderCalls, loaderCalls := 0, 0
suite := WithRequestValidation(Suite[input, state]{}, func(gotContext context.Context, gotSelection ckconfig.Selection, gotInput input, gotState state) (*promptkit.Engine, error) {
loaderCalls++
if gotContext != ctx || gotContext.Value(contextKey{}) != "validation invocation" || gotSelection != selection || gotInput != wantInput || gotState != wantState {
t.Errorf("loader args = (%v, %#v, %#v, %#v)", gotContext, gotSelection, gotInput, gotState)
}
return engine, nil
}, func(gotContext context.Context, request ValidationRequest, gotInput input, gotState state) (promptkit.RunRequest, error) {
builderCalls++
if gotContext != ctx || gotContext.Value(contextKey{}) != "validation invocation" || request.Selection != selection || request.Request.PromptID != seed.PromptID || request.Request.Vars["topic"] != "seed topic" || gotInput != wantInput || gotState != wantState {
t.Errorf("builder args = (%v, %#v, %#v, %#v)", gotContext, request, gotInput, gotState)
}
built := request.Request
built.Inputs = map[string]promptkit.ArtifactRef{"topic": promptkit.Inline("required artifact from builder")}
built.SessionID = "builder-session"
return built, nil
})
acquired, err := suite.ValidateRequest(ctx, ValidationRequest{Selection: selection, Request: seed}, wantInput, wantState)
if err != nil || acquired.Value != (struct{}{}) || builderCalls != 1 || loaderCalls != 1 {
t.Fatalf("built request validation = (%#v, %v), builder calls = %d, loader calls = %d", acquired, err, builderCalls, loaderCalls)
}
if client.calls != 0 {
t.Fatalf("request validation made %d model calls", client.calls)
}
}
func TestWithRequestValidationRetainsNilEngineAndLoadDiagnostics(t *testing.T) {
request := ValidationRequest{Request: promptkit.RunRequest{PromptID: "empty", ProfileID: "default"}}
nilEngineSuite := WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
return nil, nil
}, nil)
acquired, err := nilEngineSuite.ValidateRequest(context.Background(), request, struct{}{}, struct{}{})
if err == nil || !strings.Contains(err.Error(), "nil engine") || len(acquired.Diagnostics) != 0 {
t.Fatalf("nil engine result = (%#v, %v)", acquired, err)
}
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("invalid validation configuration", []byte("broken: [")))
if loadErr == nil {
t.Fatal("invalid configuration unexpectedly loaded")
}
loadFailure := WithRequestValidation(Suite[struct{}, struct{}]{}, func(context.Context, ckconfig.Selection, struct{}, struct{}) (*promptkit.Engine, error) {
return nil, loadErr
}, nil)
acquired, err = loadFailure.ValidateRequest(context.Background(), request, struct{}{}, struct{}{})
var retained *configkit.LoadError
if !errors.Is(err, loadErr) || !errors.As(err, &retained) || retained != loadErr || len(acquired.Diagnostics) == 0 {
t.Fatalf("load failure = (%#v, %v), want original ConfigKit error and diagnostics", acquired, err)
}
}
func TestWithRequestValidationCancellationBoundaries(t *testing.T) {
engine := adapterEngine(t)
request := ValidationRequest{Request: promptkit.RunRequest{PromptID: "missing", ProfileID: "default"}}
t.Run("before builder", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
builderCalls, loaderCalls := 0, 0
suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
loaderCalls++
return engine, nil
}, func(context.Context, ValidationRequest, int, int) (promptkit.RunRequest, error) {
builderCalls++
return request.Request, nil
})
acquired, err := suite.ValidateRequest(ctx, request, 0, 0)
if !errors.Is(err, context.Canceled) || builderCalls != 0 || loaderCalls != 0 || len(acquired.Diagnostics) != 0 {
t.Fatalf("pre-build cancellation = (%#v, %v), builder calls=%d, loader calls=%d", acquired, err, builderCalls, loaderCalls)
}
})
t.Run("after builder", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
builderCalls, loaderCalls := 0, 0
suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
loaderCalls++
return engine, nil
}, func(context.Context, ValidationRequest, int, int) (promptkit.RunRequest, error) {
builderCalls++
cancel()
return request.Request, nil
})
acquired, err := suite.ValidateRequest(ctx, request, 0, 0)
if !errors.Is(err, context.Canceled) || builderCalls != 1 || loaderCalls != 0 || len(acquired.Diagnostics) != 0 {
t.Fatalf("post-build cancellation = (%#v, %v), builder calls=%d, loader calls=%d", acquired, err, builderCalls, loaderCalls)
}
})
t.Run("during load retains load failure", func(t *testing.T) {
_, loadErr := configkit.Load[map[string]any](configkit.Bytes("cancelled validation configuration", []byte("broken: [")))
if loadErr == nil {
t.Fatal("invalid configuration unexpectedly loaded")
}
ctx, cancel := context.WithCancel(context.Background())
loaderCalls := 0
suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
loaderCalls++
cancel()
return nil, loadErr
}, nil)
acquired, err := suite.ValidateRequest(ctx, request, 0, 0)
var retained *configkit.LoadError
if !errors.Is(err, loadErr) || !errors.As(err, &retained) || retained != loadErr || len(acquired.Diagnostics) == 0 || loaderCalls != 1 {
t.Fatalf("cancelled load = (%#v, %v), loader calls=%d", acquired, err, loaderCalls)
}
})
t.Run("after successful load skips native validation", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
loaderCalls := 0
suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
loaderCalls++
cancel()
return engine, nil
}, nil)
acquired, err := suite.ValidateRequest(ctx, request, 0, 0)
if !errors.Is(err, context.Canceled) || loaderCalls != 1 || len(acquired.Diagnostics) != 0 {
t.Fatalf("post-load cancellation = (%#v, %v), loader calls=%d", acquired, err, loaderCalls)
}
})
t.Run("native artifact cancellation retains PromptKit category", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
client := &validationFailOnGenerate{}
engine := validationTestEngine(t, adapterPrompt, promptkit.WithArtifactReader(validationCancelArtifactReader{cancel: cancel}), promptkit.WithLLMClient(client))
suite := WithRequestValidation(Suite[int, int]{}, func(context.Context, ckconfig.Selection, int, int) (*promptkit.Engine, error) {
return engine, nil
}, nil)
request := ValidationRequest{Request: promptkit.RunRequest{
PromptID: "briefing",
ProfileID: "default",
Inputs: map[string]promptkit.ArtifactRef{"topic": promptkit.File("cancel-topic.txt")},
}}
acquired, err := suite.ValidateRequest(ctx, request, 0, 0)
if !errors.Is(err, promptkit.ErrArtifactLoad) || len(acquired.Diagnostics) != 0 {
t.Fatalf("native artifact cancellation = (%#v, %v), want PromptKit artifact-load category", acquired, err)
}
if client.calls != 0 {
t.Fatalf("request validation made %d model calls", client.calls)
}
})
}