From 401232a1b055f2620d2ea431e0914157f2f88f54 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 26 Oct 2026 03:35:39 +0000 Subject: [PATCH] Migrate PromptKit validation and smoke reporting --- docs/roadmap/implementation.md | 13 +- integrations/ckprompt/smoke_command.go | 112 +++++++++++------- integrations/ckprompt/smoke_command_test.go | 28 ++++- integrations/ckprompt/smoke_outcome_test.go | 26 ++-- integrations/ckprompt/validation_command.go | 20 ++-- .../ckprompt/validation_command_test.go | 72 ++++------- 6 files changed, 158 insertions(+), 113 deletions(-) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 89bbdea..6e960b6 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Invocation reporting implementation plan -Status: Stages 1–14 complete. Stage 15 is next. +Status: Stages 1–15 complete. Stage 16 is next. ## Objective and authority @@ -569,7 +569,7 @@ Stage 20 will validate the assembled module. ## Stage 15: Migrate request validation and smoke commands -Status: pending. +Status: complete. Primary files: [validation_command.go](../../integrations/ckprompt/validation_command.go) and @@ -599,6 +599,15 @@ bindings, reports alongside errors, diagnostic ordering, cancellation, render/write failures, and quiet semantics. Test CommandKit coordination, not whether PromptKit's validation or smoke checks implement their domain rules. +Concrete request validation and PromptKit smoke commands now submit passive +messages through the reporter and render confirmation, native, and aggregate +reports inside managed output. Smoke reports remain available alongside +operation failures, with native explanations and causes retained separately; +aggregate collection order and shared input/state lifetimes remain covered. +`GOTOOLCHAIN=local go test -count=1 .` passes in `integrations/ckprompt`. The +module-wide `go test ./...` reaches and passes that package but cannot compile +`examples/composed`, whose legacy API callers are scheduled for Stage 17. + ## Stage 16: Maintain a core reporting consumer Status: pending. diff --git a/integrations/ckprompt/smoke_command.go b/integrations/ckprompt/smoke_command.go index b185c73..4b88ab0 100644 --- a/integrations/ckprompt/smoke_command.go +++ b/integrations/ckprompt/smoke_command.go @@ -184,14 +184,14 @@ func checkSmokeFormat(format string) error { func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I]) commandkit.Outcome { if err := promptKitContextError(invocation.Context); err != nil { - return definitionFailure(err, nil) + return definitionFailure("PromptKit smoke testing was canceled.", commandkit.Explanation{}, err) } var state S if suite.NewState != nil { state = suite.NewState() } if err := promptKitContextError(invocation.Context); err != nil { - return definitionFailure(err, nil) + return definitionFailure("PromptKit smoke testing was canceled.", commandkit.Explanation{}, err) } if input.kind == "all" { return runSmokeAggregate(invocation, suite, input, state) @@ -199,6 +199,7 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S], request := SmokeRequest{Selection: input.selection, Request: input.request} acquired, operationErr := suite.SmokeTest(invocation.Context, request, input.selected, state) + queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics) if acquired.Value == nil && operationErr == nil { operationErr = errors.New("PromptKit smoke test returned neither a report nor an error") } @@ -210,14 +211,21 @@ func runSmokeTest[I, S any](invocation commandkit.Invocation, suite Suite[I, S], } } if acquired.Value == nil { - return definitionFailure(operationErr, acquired.Diagnostics) + return definitionFailure(smokeFailureMessage(input.kind), acquired.Explanation, operationErr) } - output, encodeErr := encodeSmokeReport(acquired.Value) - if encodeErr != nil { - return smokeDeliveryFailure(operationErr, encodeErr, acquired.Diagnostics, false) + _ = invocation.Reporter().RenderOutput(func(writer io.Writer) error { + output, err := encodeSmokeReport(acquired.Value) + if err != nil { + return err + } + _, err = writer.Write(output) + return err + }) + if operationErr != nil { + return definitionFailure(smokeFailureMessage(input.kind), acquired.Explanation, operationErr) } - return writeSmokeReport(invocation, output, acquired.Diagnostics, operationErr) + return commandkit.Success() } func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *smokeCommandInput[I], state S) commandkit.Outcome { @@ -228,13 +236,13 @@ func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I promptkit.SmokeTestStructuredOutput, } results := make([]smokeResult, 0, len(kinds)) - var diagnostics []commandkit.Diagnostic var operationErrors []error + var explanations []commandkit.Explanation for _, kind := range kinds { if err := promptKitContextError(invocation.Context); err != nil { if len(results) == 0 { - return definitionFailure(err, diagnostics) + return definitionFailure("PromptKit aggregate smoke testing was canceled.", commandkit.Explanation{}, err) } operationErrors = append(operationErrors, err) break @@ -253,10 +261,13 @@ func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I } } + queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics) results = append(results, smokeResult{Kind: kind, Report: acquired.Value, Err: operationErr}) - diagnostics = append(diagnostics, acquired.Diagnostics...) if operationErr != nil { operationErrors = append(operationErrors, operationErr) + if acquired.Explanation.Message != "" || len(acquired.Explanation.Details) > 0 { + explanations = append(explanations, acquired.Explanation) + } } if err := promptKitContextError(invocation.Context); err != nil { break @@ -264,13 +275,57 @@ func runSmokeAggregate[I, S any](invocation commandkit.Invocation, suite Suite[I } if len(results) == 0 { - return definitionFailure(errors.Join(operationErrors...), diagnostics) + return definitionFailure("PromptKit aggregate smoke testing did not produce any results.", combineSmokeExplanations(explanations), errors.Join(operationErrors...)) } - output, encodeErr := encodeSmokeResults(results) - if encodeErr != nil { - return smokeDeliveryFailure(errors.Join(operationErrors...), encodeErr, diagnostics, false) + _ = invocation.Reporter().RenderOutput(func(writer io.Writer) error { + output, err := encodeSmokeResults(results) + if err != nil { + return err + } + _, err = writer.Write(output) + return err + }) + if len(operationErrors) > 0 { + return definitionFailure("One or more PromptKit smoke tests failed.", combineSmokeExplanations(explanations), errors.Join(operationErrors...)) } - return writeSmokeReport(invocation, output, diagnostics, errors.Join(operationErrors...)) + return commandkit.Success() +} + +func smokeFailureMessage(kind string) string { + return fmt.Sprintf("PromptKit %s smoke testing failed.", kind) +} + +func combineSmokeExplanations(explanations []commandkit.Explanation) commandkit.Explanation { + var selected []commandkit.Explanation + for _, explanation := range explanations { + if explanation.Message != "" || len(explanation.Details) > 0 { + selected = append(selected, explanation) + } + } + if len(selected) == 0 { + return commandkit.Explanation{} + } + if len(selected) == 1 { + return selected[0] + } + + combined := commandkit.Explanation{Message: selected[0].Message} + combined.Details = append(combined.Details, selected[0].Details...) + for _, explanation := range selected[1:] { + if explanation.Message != "" { + message := explanation.Message + combined.Details = append(combined.Details, func(writer io.Writer) error { + return writeSmokeExplanationMessage(writer, message) + }) + } + combined.Details = append(combined.Details, explanation.Details...) + } + return combined +} + +func writeSmokeExplanationMessage(writer io.Writer, message string) error { + _, err := io.WriteString(writer, message+"\n") + return err } func smokeRequestForKind(seed promptkit.SmokeTestRequest, kind promptkit.SmokeTestKind) promptkit.SmokeTestRequest { @@ -312,30 +367,3 @@ func copyString(value *string) *string { copy := *value return © } - -func writeSmokeReport(invocation commandkit.Invocation, output []byte, diagnostics []commandkit.Diagnostic, operationErr error) commandkit.Outcome { - written, deliveryErr := invocation.Streams.Out.Write(output) - if deliveryErr == nil && written != len(output) { - deliveryErr = io.ErrShortWrite - } - if deliveryErr != nil { - return smokeDeliveryFailure(operationErr, deliveryErr, diagnostics, true) - } - if operationErr != nil { - return definitionFailure(operationErr, diagnostics) - } - outcome := commandkit.Success() - outcome.Diagnostics = diagnostics - return outcome -} - -func smokeDeliveryFailure(operationErr, deliveryErr error, diagnostics []commandkit.Diagnostic, reportStarted bool) commandkit.Outcome { - cause := operationErr - if cause == nil { - cause = deliveryErr - } - outcome := definitionFailure(cause, diagnostics) - outcome.DeliveryError = deliveryErr - outcome.ReportIncomplete = reportStarted - return outcome -} diff --git a/integrations/ckprompt/smoke_command_test.go b/integrations/ckprompt/smoke_command_test.go index 0ae0110..ad14aab 100644 --- a/integrations/ckprompt/smoke_command_test.go +++ b/integrations/ckprompt/smoke_command_test.go @@ -105,16 +105,36 @@ func TestSmokeTestCommandBindsRequestAndRendersFindings(t *testing.T) { func TestSmokeTestCommandDeliversReportWithOperationError(t *testing.T) { sentinel := errors.New("native smoke operation failed") + warning := commandkit.MessageDiagnostic(commandkit.SeverityWarning, "independent smoke warning") app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { - return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication, Completed: true}}, sentinel + return commandkit.Acquired[*promptkit.SmokeTestReport]{ + Value: &promptkit.SmokeTestReport{Kind: promptkit.SmokeTestAuthentication, Completed: true}, + Diagnostics: []commandkit.Diagnostic{warning}, + Explanation: commandkit.Explanation{Details: []commandkit.Detail{func(writer io.Writer) error { + _, err := io.WriteString(writer, "Native authentication context\n") + return err + }}}, + }, sentinel }, })) - var output bytes.Buffer - outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard}) + var output, diagnosticOutput bytes.Buffer + outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnosticOutput}) if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, sentinel) || output.Len() == 0 { t.Fatalf("report-plus-error outcome = (%#v, %q)", outcome, output.String()) } + warningAt := strings.Index(diagnosticOutput.String(), "independent smoke warning") + explanationAt := strings.Index(diagnosticOutput.String(), "Native authentication context") + if warningAt < 0 || explanationAt < 0 || warningAt >= explanationAt || strings.Contains(diagnosticOutput.String(), sentinel.Error()) { + t.Fatalf("smoke warning and native terminal context were not separate and ordered: %q", diagnosticOutput.String()) + } + + output.Reset() + diagnosticOutput.Reset() + outcome = app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--quiet"}, commandkit.Streams{Out: &output, Err: &diagnosticOutput}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, sentinel) || output.Len() != 0 || !strings.Contains(diagnosticOutput.String(), "Native authentication context") || strings.Contains(diagnosticOutput.String(), "independent smoke warning") { + t.Fatalf("quiet report-plus-error = (%#v, out=%q, err=%q)", outcome, output.String(), diagnosticOutput.String()) + } app = newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ SmokeTest: func(context.Context, SmokeRequest, struct{}, struct{}) (commandkit.Acquired[*promptkit.SmokeTestReport], error) { @@ -154,7 +174,7 @@ func TestSmokeTestCommandRetainsShortReportWrite(t *testing.T) { }, })) outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: smokeShortWriter{}, Err: io.Discard}) - if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, io.ErrShortWrite) || !errors.Is(outcome.DeliveryError, io.ErrShortWrite) || !outcome.ReportIncomplete { + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Output.WriteError, io.ErrShortWrite) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete { t.Fatalf("short report delivery = %#v", outcome) } } diff --git a/integrations/ckprompt/smoke_outcome_test.go b/integrations/ckprompt/smoke_outcome_test.go index f12588d..f22f38f 100644 --- a/integrations/ckprompt/smoke_outcome_test.go +++ b/integrations/ckprompt/smoke_outcome_test.go @@ -105,10 +105,9 @@ func TestSmokeTestEncodingAndWriterFailuresRetainOutcomeFields(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { var diagnosticOutput bytes.Buffer - outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput}) + outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--verbose"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput}) warningIndex := strings.Index(diagnosticOutput.String(), "smoke warning") - operationIndex := strings.Index(diagnosticOutput.String(), operationErr.Error()) - if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.DeliveryError, test.want) || !outcome.ReportIncomplete || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || warningIndex < 0 || operationIndex < warningIndex { + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.Presentation.Output.WriteError, test.want) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete || warningIndex < 0 || strings.Contains(diagnosticOutput.String(), operationErr.Error()) { t.Fatalf("report delivery = %#v, diagnostics=%q", outcome, diagnosticOutput.String()) } }) @@ -116,8 +115,8 @@ func TestSmokeTestEncodingAndWriterFailuresRetainOutcomeFields(t *testing.T) { diagnosticErr := errors.New("smoke diagnostic writer failed") var output bytes.Buffer - outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: partialReportWriter{err: diagnosticErr}}) - if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.DeliveryError, diagnosticErr) || outcome.ReportIncomplete || outcome.DiagnosticDelivery != commandkit.DeliveryIncomplete || output.Len() == 0 { + outcome := app.Execute(context.Background(), []string{"promptkit", "test", "authentication", "--verbose"}, commandkit.Streams{Out: &output, Err: partialReportWriter{err: diagnosticErr}}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, operationErr) || !errors.Is(outcome.Presentation.Diagnostics.WriteError, diagnosticErr) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete || output.Len() == 0 { t.Fatalf("diagnostic delivery = (%#v, %q)", outcome, output.String()) } @@ -130,7 +129,7 @@ func TestSmokeTestEncodingAndWriterFailuresRetainOutcomeFields(t *testing.T) { })) output.Reset() outcome = encodeApp.Execute(context.Background(), []string{"promptkit", "test", "authentication"}, commandkit.Streams{Out: &output, Err: io.Discard}) - if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause == nil || !errors.Is(outcome.DeliveryError, outcome.Cause) || outcome.ReportIncomplete || output.Len() != 0 { + if outcome.Kind != commandkit.OutcomeOperational || outcome.Cause != nil || outcome.Presentation.Output.RenderError == nil || output.Len() != 0 { t.Fatalf("encoding failure = (%#v, %q)", outcome, output.String()) } } @@ -151,15 +150,24 @@ func TestSmokeAggregatePreservesMultipleTypedCauses(t *testing.T) { case promptkit.SmokeTestAuthentication: err = secondErr } - return commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}}, err + acquired := commandkit.Acquired[*promptkit.SmokeTestReport]{Value: &promptkit.SmokeTestReport{Kind: request.Request.Kind}} + if err != nil { + acquired.Explanation = commandkit.Explanation{Message: string(request.Request.Kind) + " native context"} + } + return acquired, err }, })) - var output bytes.Buffer - outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: io.Discard}) + var output, diagnosticOutput bytes.Buffer + outcome := app.Execute(context.Background(), []string{"promptkit", "test", "all"}, commandkit.Streams{Out: &output, Err: &diagnosticOutput}) var retained *smokeTypedError if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, firstErr) || !errors.Is(outcome.Cause, secondErr) || !errors.As(outcome.Cause, &retained) || retained != firstErr { t.Fatalf("joined aggregate errors lost identity: %#v, retained=%#v", outcome, retained) } + firstContext := strings.Index(diagnosticOutput.String(), "reachability native context") + secondContext := strings.Index(diagnosticOutput.String(), "authentication native context") + if firstContext < 0 || secondContext < 0 || firstContext >= secondContext { + t.Fatalf("aggregate native explanations were not retained in order: %q", diagnosticOutput.String()) + } var document struct { Tests []struct { Kind string `yaml:"kind"` diff --git a/integrations/ckprompt/validation_command.go b/integrations/ckprompt/validation_command.go index cbc6890..1708a90 100644 --- a/integrations/ckprompt/validation_command.go +++ b/integrations/ckprompt/validation_command.go @@ -2,6 +2,8 @@ package ckprompt import ( "errors" + "fmt" + "io" commandkit "gitea.maximumdirect.net/eric/CommandKit" promptkit "gitea.maximumdirect.net/eric/promptkit" @@ -102,26 +104,26 @@ func newValidationCommand[I, S any](suite Suite[I, S]) commandkit.Command { func runRequestValidation[I, S any](invocation commandkit.Invocation, suite Suite[I, S], input *validationCommandInput[I]) commandkit.Outcome { if err := promptKitContextError(invocation.Context); err != nil { - return definitionFailure(err, nil) + return definitionFailure("PromptKit request validation was canceled.", commandkit.Explanation{}, err) } var state S if suite.NewState != nil { state = suite.NewState() } if err := promptKitContextError(invocation.Context); err != nil { - return definitionFailure(err, nil) + return definitionFailure("PromptKit request validation was canceled.", commandkit.Explanation{}, err) } acquired, err := suite.ValidateRequest(invocation.Context, input.request, input.selected, state) + queuePromptDiagnostics(invocation.Reporter(), acquired.Diagnostics) if err != nil { - return definitionFailure(err, acquired.Diagnostics) + return definitionFailure(fmt.Sprintf("Unable to validate the PromptKit request for prompt %q.", input.promptID), acquired.Explanation, err) } if err := promptKitContextError(invocation.Context); err != nil { - return definitionFailure(err, acquired.Diagnostics) + return definitionFailure(fmt.Sprintf("Validation of the PromptKit request for prompt %q was canceled.", input.promptID), acquired.Explanation, err) } - output, err := encodePromptYAMLBytes(requestValidationReport{PromptID: input.promptID, Valid: true}) - if err != nil { - return definitionFailure(err, acquired.Diagnostics) - } - return writePromptReport(invocation, output, acquired.Diagnostics) + _ = invocation.Reporter().RenderOutput(func(writer io.Writer) error { + return encodePromptYAML(writer, requestValidationReport{PromptID: input.promptID, Valid: true}) + }) + return commandkit.Success() } diff --git a/integrations/ckprompt/validation_command_test.go b/integrations/ckprompt/validation_command_test.go index ae97de9..58031f0 100644 --- a/integrations/ckprompt/validation_command_test.go +++ b/integrations/ckprompt/validation_command_test.go @@ -98,7 +98,7 @@ func TestValidationCommandIsAvailableIndependentlyAndOwnsItsFamily(t *testing.T) }})) var help bytes.Buffer outcome := validationOnly.Execute(context.Background(), []string{"help", "promptkit", "requirements"}, commandkit.Streams{Out: &help, Err: io.Discard}) - if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "validate") || strings.Contains(help.String(), "show") { + if outcome.Kind != commandkit.OutcomeSuccess || !strings.Contains(help.String(), "validate") || strings.Contains(help.String(), "\n show ") { t.Fatalf("validation-only availability = (%#v, %q)", outcome, help.String()) } @@ -359,41 +359,32 @@ func TestValidationCommandUsesNativeOutcomesAndMinimalConfirmation(t *testing.T) } func TestValidationCommandRetainsProviderDiagnosticsAndClassifiesFailures(t *testing.T) { - warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error { - _, err := io.WriteString(writer, "validation warning\n") - return err - }} - withWarning := func(diagnostics []commandkit.Diagnostic) bool { - for _, diagnostic := range diagnostics { - if diagnostic.Severity == commandkit.SeverityWarning && diagnostic.Write != nil { - var output bytes.Buffer - if diagnostic.Write(&output) == nil && output.String() == "validation warning\n" { - return true - } - } - } - return false - } + warning := commandkit.MessageDiagnostic(commandkit.SeverityWarning, "validation warning") var diagnostics bytes.Buffer app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) { return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil }})) - outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics}) - if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || !withWarning(outcome.Diagnostics) || diagnostics.String() != "validation warning\n" { + outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics}) + if outcome.Kind != commandkit.OutcomeSuccess || outcome.Status != 0 || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete || !strings.Contains(diagnostics.String(), "validation warning") { t.Fatalf("warning-only success = %#v, stderr=%q", outcome, diagnostics.String()) } + var quietReport, quietDiagnostics bytes.Buffer + quietOutcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--quiet"}, commandkit.Streams{Out: &quietReport, Err: &quietDiagnostics}) + if quietOutcome.Kind != commandkit.OutcomeSuccess || quietOutcome.Presentation.Output.State != commandkit.DeliverySuppressed || quietOutcome.Presentation.Diagnostics.State != commandkit.DeliverySuppressed || quietReport.Len() != 0 || quietDiagnostics.Len() != 0 { + t.Fatalf("quiet validation = %#v, report=%q diagnostics=%q", quietOutcome, quietReport.String(), quietDiagnostics.String()) + } providerErr := errors.New("validation provider failed") failedApp := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) { return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr }})) diagnostics.Reset() - outcome = failedApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics}) - if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, providerErr) || !withWarning(outcome.Diagnostics) || outcome.DiagnosticDelivery != commandkit.DeliveryComplete { + outcome = failedApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: &diagnostics}) + if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, providerErr) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete { t.Fatalf("provider failure = %#v, stderr=%q", outcome, diagnostics.String()) } - if strings.Count(diagnostics.String(), "validation warning") != 1 || strings.Count(diagnostics.String(), providerErr.Error()) != 1 { - t.Fatalf("provider diagnostics were lost or duplicated: stderr=%q", diagnostics.String()) + if strings.Count(diagnostics.String(), "validation warning") != 1 || strings.Contains(diagnostics.String(), providerErr.Error()) || !strings.Contains(diagnostics.String(), "Unable to validate the PromptKit request") { + t.Fatalf("provider warning or terminal explanation was incorrect: stderr=%q", diagnostics.String()) } builderErr := errors.New("request builder failed") @@ -418,16 +409,13 @@ func TestValidationCommandRetainsProviderDiagnosticsAndClassifiesFailures(t *tes }, nil))) outcome = loadApp.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) var retained *configkit.LoadError - if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, loadErr) || !errors.As(outcome.Cause, &retained) || retained != loadErr || len(outcome.Diagnostics) == 0 { + if outcome.Kind != commandkit.OutcomeOperational || outcome.Status != 1 || !errors.Is(outcome.Cause, loadErr) || !errors.As(outcome.Cause, &retained) || retained != loadErr || outcome.Explanation.Message == "" && len(outcome.Explanation.Details) == 0 { t.Fatalf("loader failure = %#v, want retained ConfigKit error and diagnostics", outcome) } } func TestValidationCommandCancellationPreventsAcquisitionAndReport(t *testing.T) { - warning := commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(writer io.Writer) error { - _, err := io.WriteString(writer, "validation warning\n") - return err - }} + warning := commandkit.MessageDiagnostic(commandkit.SeverityWarning, "validation warning") calls := 0 app := newPromptTestApp(t, Command(Suite[struct{}, struct{}]{ValidateRequest: func(context.Context, ValidationRequest, struct{}, struct{}) (commandkit.Acquired[struct{}], error) { calls++ @@ -448,8 +436,9 @@ func TestValidationCommandCancellationPreventsAcquisitionAndReport(t *testing.T) return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, nil }})) output.Reset() - outcome = canceledApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: &output, Err: io.Discard}) - if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || !withValidationWarning(outcome.Diagnostics) || output.Len() != 0 { + var diagnostics bytes.Buffer + outcome = canceledApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: &output, Err: &diagnostics}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, context.Canceled) || !strings.Contains(diagnostics.String(), "validation warning") || output.Len() != 0 { t.Fatalf("post-acquisition cancellation = %#v, output=%q", outcome, output.String()) } @@ -459,8 +448,9 @@ func TestValidationCommandCancellationPreventsAcquisitionAndReport(t *testing.T) cancel() return commandkit.Acquired[struct{}]{Diagnostics: []commandkit.Diagnostic{warning}}, providerErr }})) - outcome = errorApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: io.Discard}) - if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, providerErr) || !withValidationWarning(outcome.Diagnostics) { + var errorDiagnostics bytes.Buffer + outcome = errorApp.Execute(ctx, []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: &errorDiagnostics}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Cause, providerErr) || !strings.Contains(errorDiagnostics.String(), "validation warning") { t.Fatalf("provider error after cancellation = %#v", outcome) } } @@ -484,28 +474,16 @@ func TestValidationCommandPreservesReportAndDiagnosticDeliveryFailures(t *testin } { t.Run(test.name, func(t *testing.T) { var diagnosticOutput bytes.Buffer - outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput}) - if outcome.Kind != commandkit.OutcomeOperational || !outcome.ReportIncomplete || !errors.Is(outcome.Cause, test.want) || !errors.Is(outcome.DeliveryError, test.want) || outcome.DiagnosticDelivery != commandkit.DeliveryComplete || !strings.Contains(diagnosticOutput.String(), "validation warning") { + outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: test.writer, Err: &diagnosticOutput}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Output.WriteError, test.want) || outcome.Presentation.Output.State != commandkit.DeliveryIncomplete || outcome.Presentation.Diagnostics.State != commandkit.DeliveryComplete || !strings.Contains(diagnosticOutput.String(), "validation warning") { t.Fatalf("report delivery = %#v, diagnostics=%q", outcome, diagnosticOutput.String()) } }) } diagnosticErr := errors.New("validation diagnostic writer failed") - outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing"}, commandkit.Streams{Out: io.Discard, Err: partialReportWriter{err: diagnosticErr}}) - if outcome.Kind != commandkit.OutcomeOperational || outcome.ReportIncomplete || !errors.Is(outcome.DeliveryError, diagnosticErr) || outcome.DiagnosticDelivery != commandkit.DeliveryIncomplete { + outcome := app.Execute(context.Background(), []string{"promptkit", "requirements", "validate", "briefing", "--verbose"}, commandkit.Streams{Out: io.Discard, Err: partialReportWriter{err: diagnosticErr}}) + if outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Diagnostics.WriteError, diagnosticErr) || outcome.Presentation.Diagnostics.State != commandkit.DeliveryIncomplete { t.Fatalf("diagnostic delivery = %#v", outcome) } } - -func withValidationWarning(diagnostics []commandkit.Diagnostic) bool { - for _, diagnostic := range diagnostics { - if diagnostic.Severity == commandkit.SeverityWarning && diagnostic.Write != nil { - var output bytes.Buffer - if diagnostic.Write(&output) == nil && output.String() == "validation warning\n" { - return true - } - } - } - return false -}