base-layer/commandkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Migrate PromptKit validation and smoke reporting

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-26 03:35:39 +00:00
parent 4ace616bc2
commit 401232a1b0
6 changed files with 158 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
docs/roadmap/implementation.md
View File

@@ -1,6 +1,6 @@
# Invocation reporting implementation plan
Status: Stages 1–14 complete. Stage 15 is next.
Status: Stages 1–15 complete. Stage 16 is next.
## Objective and authority
@@ -569,7 +569,7 @@ Stage 20 will validate the assembled module.
## Stage 15: Migrate request validation and smoke commands
Status: pending.
Status: complete.
Primary files:
[validation_command.go](../../integrations/ckprompt/validation_command.go) and
@@ -599,6 +599,15 @@ bindings, reports alongside errors, diagnostic ordering, cancellation,
render/write failures, and quiet semantics. Test CommandKit coordination,
not whether PromptKit's validation or smoke checks implement their domain rules.
Concrete request validation and PromptKit smoke commands now submit passive
messages through the reporter and render confirmation, native, and aggregate
reports inside managed output. Smoke reports remain available alongside
operation failures, with native explanations and causes retained separately;
aggregate collection order and shared input/state lifetimes remain covered.
`GOTOOLCHAIN=local go test -count=1 .` passes in `integrations/ckprompt`. The
module-wide `go test ./...` reaches and passes that package but cannot compile
`examples/composed`, whose legacy API callers are scheduled for Stage 17.
## Stage 16: Maintain a core reporting consumer
Status: pending.

112
integrations/ckprompt/smoke_command.go
View File

@@ -184,14 +184,14 @@ func checkSmokeFormat(format string) error {
func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I]) commandkit.Outcome {
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure(err, nil)
return definitionFailure("PromptKit smoke testing was canceled.", commandkit.Explanation{}, err)
}
var state S
if suite.NewState != nil {
state = suite.NewState()
}
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure(err, nil)
return definitionFailure("PromptKit smoke testing was canceled.", commandkit.Explanation{}, err)
}
if input.kind == "all" {
return runSmokeAggregate(invocation, suite, input, state)
@@ -199,6 +199,7 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S],
request := SmokeRequest{Selection: input.selection, Request: input.request}
acquired, operationErr := suite.SmokeTest(invocation.Context, request, input.selected, state)
queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics)
if acquired.Value == nil && operationErr == nil {
operationErr = errors.New("PromptKit smoke test returned neither a report nor an error")
}
@@ -210,14 +211,21 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S],
}
}
if acquired.Value == nil {
return definitionFailure(operationErr, acquired.Diagnostics)
return definitionFailure(smokeFailureMessage(input.kind), acquired.Explanation, operationErr)
}
output, encodeErr := encodeSmokeReport(acquired.Value)
if encodeErr != nil {
return smokeDeliveryFailure(operationErr, encodeErr, acquired.Diagnostics, false)
_ = invocation.Reporter().RenderOutput(func(writer io.Writer) error {
output, err := encodeSmokeReport(acquired.Value)
if err != nil {
return err
}
_, err = writer.Write(output)
return err
})
if operationErr != nil {
return definitionFailure(smokeFailureMessage(input.kind), acquired.Explanation, operationErr)
}
return writeSmokeReport(invocation, output, acquired.Diagnostics, operationErr)
return commandkit.Success()
}
func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I], state S) commandkit.Outcome {
@@ -228,13 +236,13 @@ func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I
promptkit.SmokeTestStructuredOutput,
}
results := make([]smokeResult, 0, len(kinds))
var diagnostics []commandkit.Diagnostic
var operationErrors []error
var explanations []commandkit.Explanation
for _, kind := range kinds {
if err := promptKitContextError(invocation.Context); err != nil {
if len(results) == 0 {
return definitionFailure(err, diagnostics)
return definitionFailure("PromptKit aggregate smoke testing was canceled.", commandkit.Explanation{}, err)
}
operationErrors = append(operationErrors, err)
break
@@ -253,10 +261,13 @@ func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I
}
}
queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics)
results = append(results, smokeResult{Kind: kind, Report: acquired.Value, Err: operationErr})
diagnostics = append(diagnostics, acquired.Diagnostics...)
if operationErr != nil {
operationErrors = append(operationErrors, operationErr)
if acquired.Explanation.Message != "" || len(acquired.Explanation.Details) > 0 {
explanations = append(explanations, acquired.Explanation)
}
}
if err := promptKitContextError(invocation.Context); err != nil {
break
@@ -264,13 +275,57 @@ func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I
}
if len(results) == 0 {
return definitionFailure(errors.Join(operationErrors...), diagnostics)
return definitionFailure("PromptKit aggregate smoke testing did not produce any results.", combineSmokeExplanations(explanations), errors.Join(operationErrors...))
}
output, encodeErr := encodeSmokeResults(results)
if encodeErr != nil {
return smokeDeliveryFailure(errors.Join(operationErrors...), encodeErr, diagnostics, false)
_ = invocation.Reporter().RenderOutput(func(writer io.Writer) error {
output, err := encodeSmokeResults(results)
if err != nil {
return err
}
_, err = writer.Write(output)
return err
})
if len(operationErrors) > 0 {
return definitionFailure("One or more PromptKit smoke tests failed.", combineSmokeExplanations(explanations), errors.Join(operationErrors...))
}
return writeSmokeReport(invocation, output, diagnostics, errors.Join(operationErrors...))
return commandkit.Success()
}
func smokeFailureMessage(kind string) string {
return fmt.Sprintf("PromptKit %s smoke testing failed.", kind)
}
func combineSmokeExplanations(explanations []commandkit.Explanation) commandkit.Explanation {
var selected []commandkit.Explanation
for _, explanation := range explanations {
if explanation.Message != "" || len(explanation.Details) > 0 {
selected = append(selected, explanation)
}
}
if len(selected) == 0 {
return commandkit.Explanation{}
}
if len(selected) == 1 {
return selected[0]
}
combined := commandkit.Explanation{Message: selected[0].Message}
combined.Details = append(combined.Details, selected[0].Details...)
for _, explanation := range selected[1:] {
if explanation.Message != "" {
message := explanation.Message
combined.Details = append(combined.Details, func(writer io.Writer) error {
return writeSmokeExplanationMessage(writer, message)
})
}
combined.Details = append(combined.Details, explanation.Details...)
}
return combined
}
func writeSmokeExplanationMessage(writer io.Writer, message string) error {
_, err := io.WriteString(writer, message+"\n")
return err
}
func smokeRequestForKind(seed promptkit.SmokeTestRequest, kind promptkit.SmokeTestKind) promptkit.SmokeTestRequest {
@@ -312,30 +367,3 @@ func copyString(value *string) *string {
copy := *value
return &copy
}
func writeSmokeReport(invocation commandkit.Invocation, output []byte, diagnostics []commandkit.Diagnostic, operationErr error) commandkit.Outcome {
written, deliveryErr := invocation.Streams.Out.Write(output)
if deliveryErr == nil && written != len(output) {
deliveryErr = io.ErrShortWrite
}
if deliveryErr != nil {
return smokeDeliveryFailure(operationErr, deliveryErr, diagnostics, true)
}
if operationErr != nil {
return definitionFailure(operationErr, diagnostics)
}
outcome := commandkit.Success()
outcome.Diagnostics = diagnostics
return outcome
}
func smokeDeliveryFailure(operationErr, deliveryErr error, diagnostics []commandkit.Diagnostic, reportStarted bool) commandkit.Outcome {
cause := operationErr
if cause == nil {
cause = deliveryErr
}
outcome := definitionFailure(cause, diagnostics)
outcome.DeliveryError = deliveryErr
outcome.ReportIncomplete = reportStarted
return outcome
}

28
integrations/ckprompt/smoke_command_test.go
View File

@@ -105,16 +105,36 @@ func TestSmokeTestCommandBindsRequestAndRendersFindings(t *testing.T) {
func TestSmokeTestCommandDeliversReportWithOperationError(t *testing.T) {
sentinel := errors.New("native smoke operation failed")
warning := commandkit.MessageDiagnostic(commandkit.SeverityWarning, "independent smoke warning")
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication, Completed: true}}, sentinel
return commandkit.Acquired[*promptkit.SmokeTestReport]{
Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication, Completed: true},
Diagnostics: []commandkit.Diagnostic{warning},
Explanation: commandkit.Explanation{Details: []commandkit.Detail{func(writer io.Writer) error {
_, err := io.WriteString(writer, "Native authentication context\n")
return err
}}},
}, sentinel
},
}))
var output bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard})
var output, diagnosticOutput bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnosticOutput})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, sentinel) || output.Len() == 0 {
t.Fatalf("report-plus-error outcome = (%#v, %q)", outcome, output.String())
}
warningAt := strings.Index(diagnosticOutput.String(), "independent smoke warning")
explanationAt := strings.Index(diagnosticOutput.String(), "Native authentication context")
if warningAt < 0 || explanationAt < 0 || warningAt >= explanationAt || strings.Contains(diagnosticOutput.String(), sentinel.Error()) {
t.Fatalf("smoke warning and native terminal context were not separate and ordered: %q", diagnosticOutput.String())
}
output.Reset()
diagnosticOutput.Reset()
outcome = app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnosticOutput})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, sentinel) || output.Len() != 0 || !strings.Contains(diagnosticOutput.String(), "Native authentication context") || strings.Contains(diagnosticOutput.String(), "independent smoke warning") {
t.Fatalf("quiet report-plus-error = (%#v, out=%q, err=%q)", outcome, output.String(), diagnosticOutput.String())
}
app = newPromptTestApp(t, Command(Suite[struct{}, struct{}]{
SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) {
@@ -154,7 +174,7 @@ func TestSmokeTestCommandRetainsShortReportWrite(t *testing.T) {
},
}))
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: smokeShortWriter{}, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, io.ErrShortWrite) || !errors.Is(outcome.DeliveryError, io.ErrShortWrite) || !outcome.ReportIncomplete {
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Output.WriteError, io.ErrShortWrite) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete {
t.Fatalf("short report delivery = %#v", outcome)
}
}

26
integrations/ckprompt/smoke_outcome_test.go
View File

@@ -105,10 +105,9 @@ func TestSmokeTestEncodingAndWriterFailuresRetainOutcomeFields(t *testing.T) {
} {
t.Run(test.name, func(t *testing.T) {
var diagnosticOutput bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput})
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--verbose"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput})
warningIndex := strings.Index(diagnosticOutput.String(), "smoke warning")
operationIndex := strings.Index(diagnosticOutput.String(), operationErr.Error())
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.DeliveryError, test.want) || !outcome.ReportIncomplete || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || warningIndex < 0 || operationIndex < warningIndex {
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.Presentation.Output.WriteError, test.want) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete || warningIndex < 0 || strings.Contains(diagnosticOutput.String(), operationErr.Error()) {
t.Fatalf("report delivery = %#v, diagnostics=%q", outcome, diagnosticOutput.String())
}
})
@@ -116,8 +115,8 @@ func TestSmokeTestEncodingAndWriterFailuresRetainOutcomeFields(t *testing.T) {
diagnosticErr := errors.New("smoke diagnostic writer failed")
var output bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: partialReportWriter{err: diagnosticErr}})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.DeliveryError, diagnosticErr) || outcome.ReportIncomplete || outcome.DiagnosticDelivery != commandkit.DeliveryIncomplete || output.Len() == 0 {
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--verbose"}, commandkit.Streams{Out: &output, Err: partialReportWriter{err: diagnosticErr}})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.Presentation.Diagnostics.WriteError, diagnosticErr) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete || output.Len() == 0 {
t.Fatalf("diagnostic delivery = (%#v, %q)", outcome, output.String())
}
@@ -130,7 +129,7 @@ func TestSmokeTestEncodingAndWriterFailuresRetainOutcomeFields(t *testing.T) {
}))
output.Reset()
outcome = encodeApp.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil || !errors.Is(outcome.DeliveryError, outcome.Cause) || outcome.ReportIncomplete || output.Len() != 0 {
if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause != nil || outcome.Presentation.Output.RenderError == nil || output.Len() != 0 {
t.Fatalf("encoding failure = (%#v, %q)", outcome, output.String())
}
}
@@ -151,15 +150,24 @@ func TestSmokeAggregatePreservesMultipleTypedCauses(t *testing.T) {
case promptkit.SmokeTestAuthentication:
err = secondErr
}
return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, err
acquired := commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}
if err != nil {
acquired.Explanation = commandkit.Explanation{Message: string(request.Request.Kind) + " native context"}
}
return acquired, err
},
}))
var output bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard})
var output, diagnosticOutput bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: &diagnosticOutput})
var retained *smokeTypedError
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, firstErr) || !errors.Is(outcome.Cause, secondErr) || !errors.As(outcome.Cause, &retained) || retained != firstErr {
t.Fatalf("joined aggregate errors lost identity: %#v, retained=%#v", outcome, retained)
}
firstContext := strings.Index(diagnosticOutput.String(), "reachability native context")
secondContext := strings.Index(diagnosticOutput.String(), "authentication native context")
if firstContext < 0 || secondContext < 0 || firstContext >= secondContext {
t.Fatalf("aggregate native explanations were not retained in order: %q", diagnosticOutput.String())
}
var document struct {
Tests []struct {
Kind string `yaml:"kind"`

20
integrations/ckprompt/validation_command.go
View File

@@ -2,6 +2,8 @@ package ckprompt
import (
"errors"
"fmt"
"io"
commandkit "gitea.maximumdirect.net/eric/CommandKit"
promptkit "gitea.maximumdirect.net/eric/promptkit"
@@ -102,26 +104,26 @@ func newValidationCommand[I, S any](suite Suite[I, S]) commandkit.Command {
func runRequestValidation[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *validationCommandInput[I]) commandkit.Outcome {
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure(err, nil)
return definitionFailure("PromptKit request validation was canceled.", commandkit.Explanation{}, err)
}
var state S
if suite.NewState != nil {
state = suite.NewState()
}
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure(err, nil)
return definitionFailure("PromptKit request validation was canceled.", commandkit.Explanation{}, err)
}
acquired, err := suite.ValidateRequest(invocation.Context, input.request, input.selected, state)
queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics)
if err != nil {
return definitionFailure(err, acquired.Diagnostics)
return definitionFailure(fmt.Sprintf("Unable to validate the PromptKit request for prompt %q.", input.promptID), acquired.Explanation, err)
}
if err := promptKitContextError(invocation.Context); err != nil {
return definitionFailure(err, acquired.Diagnostics)
return definitionFailure(fmt.Sprintf("Validation of the PromptKit request for prompt %q was canceled.", input.promptID), acquired.Explanation, err)
}
output, err := encodePromptYAMLBytes(requestValidationReport{PromptID: input.promptID, Valid: true})
if err != nil {
return definitionFailure(err, acquired.Diagnostics)
}
return writePromptReport(invocation, output, acquired.Diagnostics)
_ = invocation.Reporter().RenderOutput(func(writer io.Writer) error {
return encodePromptYAML(writer, requestValidationReport{PromptID: input.promptID, Valid: true})
})
return commandkit.Success()
}

72
integrations/ckprompt/validation_command_test.go
View File

@@ -98,7 +98,7 @@ func TestValidationCommandIsAvailableIndependentlyAndOwnsItsFamily(t *testing.T)
}}))
var help bytes.Buffer
outcome := validationOnly.Execute(context.Background(), []string{"help", "promptkit", "requirements"}, commandkit.Streams{Out: &help, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "validate") || strings.Contains(help.String(), "show") {
if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "validate") || strings.Contains(help.String(), "\n show ") {
t.Fatalf("validation-only availability = (%#v, %q)", outcome, help.String())
}
@@ -359,41 +359,32 @@ func TestValidationCommandUsesNativeOutcomesAndMinimalConfirmation(t *testing.T)
}
func TestValidationCommandRetainsProviderDiagnosticsAndClassifiesFailures(t *testing.T) {
warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
_, err := io.WriteString(writer, "validation warning\n")
return err
}}
withWarning := func(diagnostics []commandkit.Diagnostic) bool {
for _, diagnostic := range diagnostics {
if diagnostic.Severity == commandkit.SeverityWarning && diagnostic.Write != nil {
var output bytes.Buffer
if diagnostic.Write(&output) == nil && output.String() == "validation warning\n" {
return true
}
}
}
return false
}
warning := commandkit.MessageDiagnostic(commandkit.SeverityWarning, "validation warning")
var diagnostics bytes.Buffer
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil
}}))
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || !withWarning(outcome.Diagnostics) || diagnostics.String() != "validation warning\n" {
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete || !strings.Contains(diagnostics.String(), "validation warning") {
t.Fatalf("warning-only success = %#v, stderr=%q", outcome, diagnostics.String())
}
var quietReport, quietDiagnostics bytes.Buffer
quietOutcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--quiet"}, commandkit.Streams{Out: &quietReport, Err: &quietDiagnostics})
if quietOutcome.Kind != commandkit.OutcomeSuccess || quietOutcome.Presentation.Output.State != commandkit.DeliverySuppressed || quietOutcome.Presentation.Diagnostics.State != commandkit.DeliverySuppressed || quietReport.Len() != 0 || quietDiagnostics.Len() != 0 {
t.Fatalf("quiet validation = %#v, report=%q diagnostics=%q", quietOutcome, quietReport.String(), quietDiagnostics.String())
}
providerErr := errors.New("validation provider failed")
failedApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr
}}))
diagnostics.Reset()
outcome = failedApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, providerErr) || !withWarning(outcome.Diagnostics) || outcome.DiagnosticDelivery != commandkit.DeliveryComplete {
outcome = failedApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, providerErr) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete {
t.Fatalf("provider failure = %#v, stderr=%q", outcome, diagnostics.String())
}
if strings.Count(diagnostics.String(), "validation warning") != 1 || strings.Count(diagnostics.String(), providerErr.Error()) != 1 {
t.Fatalf("provider diagnostics were lost or duplicated: stderr=%q", diagnostics.String())
if strings.Count(diagnostics.String(), "validation warning") != 1 || strings.Contains(diagnostics.String(), providerErr.Error()) || !strings.Contains(diagnostics.String(), "Unable to validate the PromptKit request") {
t.Fatalf("provider warning or terminal explanation was incorrect: stderr=%q", diagnostics.String())
}
builderErr := errors.New("request builder failed")
@@ -418,16 +409,13 @@ func TestValidationCommandRetainsProviderDiagnosticsAndClassifiesFailures(t *tes
}, nil)))
outcome = loadApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
var retained *configkit.LoadError
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, loadErr) || !errors.As(outcome.Cause, &retained) || retained != loadErr || len(outcome.Diagnostics) == 0 {
if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, loadErr) || !errors.As(outcome.Cause, &retained) || retained != loadErr || outcome.Explanation.Message == "" && len(outcome.Explanation.Details) == 0 {
t.Fatalf("loader failure = %#v, want retained ConfigKit error and diagnostics", outcome)
}
}
func TestValidationCommandCancellationPreventsAcquisitionAndReport(t *testing.T) {
warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error {
_, err := io.WriteString(writer, "validation warning\n")
return err
}}
warning := commandkit.MessageDiagnostic(commandkit.SeverityWarning, "validation warning")
calls := 0
app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) {
calls++
@@ -448,8 +436,9 @@ func TestValidationCommandCancellationPreventsAcquisitionAndReport(t *testing.T)
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil
}}))
output.Reset()
outcome = canceledApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: &output, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || !withValidationWarning(outcome.Diagnostics) || output.Len() != 0 {
var diagnostics bytes.Buffer
outcome = canceledApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || !strings.Contains(diagnostics.String(), "validation warning") || output.Len() != 0 {
t.Fatalf("post-acquisition cancellation = %#v, output=%q", outcome, output.String())
}
@@ -459,8 +448,9 @@ func TestValidationCommandCancellationPreventsAcquisitionAndReport(t *testing.T)
cancel()
return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr
}}))
outcome = errorApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, providerErr) || !withValidationWarning(outcome.Diagnostics) {
var errorDiagnostics bytes.Buffer
outcome = errorApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: &errorDiagnostics})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, providerErr) || !strings.Contains(errorDiagnostics.String(), "validation warning") {
t.Fatalf("provider error after cancellation = %#v", outcome)
}
}
@@ -484,28 +474,16 @@ func TestValidationCommandPreservesReportAndDiagnosticDeliveryFailures(t *testin
} {
t.Run(test.name, func(t *testing.T) {
var diagnosticOutput bytes.Buffer
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput})
if outcome.Kind != commandkit.OutcomeOperational || !outcome.ReportIncomplete || !errors.Is(outcome.Cause, test.want) || !errors.Is(outcome.DeliveryError, test.want) || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || !strings.Contains(diagnosticOutput.String(), "validation warning") {
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Output.WriteError, test.want) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete || !strings.Contains(diagnosticOutput.String(), "validation warning") {
t.Fatalf("report delivery = %#v, diagnostics=%q", outcome, diagnosticOutput.String())
}
})
}
diagnosticErr := errors.New("validation diagnostic writer failed")
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: partialReportWriter{err: diagnosticErr}})
if outcome.Kind != commandkit.OutcomeOperational || outcome.ReportIncomplete || !errors.Is(outcome.DeliveryError, diagnosticErr) || outcome.DiagnosticDelivery != commandkit.DeliveryIncomplete {
outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: partialReportWriter{err: diagnosticErr}})
if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Diagnostics.WriteError, diagnosticErr) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete {
t.Fatalf("diagnostic delivery = %#v", outcome)
}
}
func withValidationWarning(diagnostics []commandkit.Diagnostic) bool {
for _, diagnostic := range diagnostics {
if diagnostic.Severity == commandkit.SeverityWarning && diagnostic.Write != nil {
var output bytes.Buffer
if diagnostic.Write(&output) == nil && output.String() == "validation warning\n" {
return true
}
}
}
return false
}