From 0b70f372671f9f7853cb723f397438aee0debfb8 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 7 Nov 2026 04:52:58 +0000 Subject: [PATCH] Add detailed validation evidence --- docs/cli.md | 8 ++ docs/roadmap/implementation.md | 7 +- .../ckconfig/detailed_validation_test.go | 136 ++++++++++++++++++ integrations/ckconfig/input.go | 20 +++ integrations/ckconfig/validate.go | 134 ++++++++++++++--- 5 files changed, 286 insertions(+), 19 deletions(-) create mode 100644 integrations/ckconfig/detailed_validation_test.go diff --git a/docs/cli.md b/docs/cli.md index 5cd11e8..4412157 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -335,12 +335,20 @@ with each reached scope and its outcome (`valid`, `failed`, or `canceled`). example config validate example config validate --scope promptkit --format yaml example config validate --scope all +example config validate --scope all --details --severity warning --code schema.version ``` Each successful scope is reported as `SCOPE: valid`. Aggregate validation stops at the first load failure or cancellation, then submits one report containing the reached scopes. Output errors occur after acquisition has stopped. Provider warnings go to the diagnostic stream and do not alone make validation fail. +`--details` includes native findings and the source references needed to +interpret them. Each scope's detail state is `unavailable`, `empty`, `filtered`, +or `present`. Repeat `--severity error|warning` or `--code CODE` to filter +findings: values within a filter match any, and the two filters narrow together. +Filters require `--details` and never change the validation outcome. Detailed +output suppresses ordinary warnings associated with its native findings; other +messages and terminal failure explanations remain visible. `--config` and `--overlay` preserve whether an empty value was explicitly supplied, so a scope callback can map those inputs to its own ConfigKit load policy. Help and invalid syntax do not call scope diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 611ec4d..366e42b 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1,6 +1,6 @@ # Shared reporting and PromptKit v0.20.0 implementation plan -Status: implementation in progress; Stages 1–19 complete. +Status: implementation in progress; Stages 1–20 complete. ## Scope, authority, and execution @@ -709,6 +709,11 @@ add a comparison callback, reload data, or implement list matching. ## Stage 20: Detailed validation and reached-scope aggregation +**Complete.** Detailed validation filters native findings and reports reached +scopes after acquisition stops, preserving independent diagnostics and causes. +Root, ckconfig, and ckprompt workspace suites passed with Go 1.27.0, including +failure, cancellation, source references, quiet mode, and output failure. + Complete Stage 7's validation aggregation using Stage 14's evidence. Add `--details`, repeatable `--severity`, and repeatable `--code` on validation; filters require detailed mode. Parse severity using native known severities; diff --git a/integrations/ckconfig/detailed_validation_test.go b/integrations/ckconfig/detailed_validation_test.go new file mode 100644 index 0000000..462c3db --- /dev/null +++ b/integrations/ckconfig/detailed_validation_test.go @@ -0,0 +1,136 @@ +package ckconfig + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "strings" + "testing" + + commandkit "go.maximumdirect.net/commandkit" + configkit "go.maximumdirect.net/configkit" +) + +func validationDiagnostic(message string) commandkit.Diagnostic { + return commandkit.Diagnostic{Severity: commandkit.SeverityWarning, Write: func(w io.Writer) error { _, err := io.WriteString(w, message+"\n"); return err }} +} + +func TestDetailedValidationFiltersAndRetainsFailureEvidence(t *testing.T) { + cause := errors.New("second scope failed") + calls := []string{} + reader := &sourceInventory{values: []configkit.SourceInfo{{ID: "primary", Label: "main", Path: "main.yml"}, {ID: "related", Label: "included", Path: "include.yml"}, {ID: "unused", Label: "unused"}}} + records := []configkit.Diagnostic{ + {Code: "chosen", Severity: configkit.SeverityWarning, Path: configkit.Keys("a/b"), Message: "selected warning", Primary: &configkit.Location{SourceID: "primary", Line: 2, Column: 3}, Related: []configkit.Location{{SourceID: "related", Line: 7, Column: 8}}}, + {Code: "other", Severity: configkit.SeverityWarning, Path: configkit.Keys("other"), Message: "other warning"}, + } + scopes := []Scope[struct{}, struct{}]{ + {Name: "app", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) { + calls = append(calls, "app") + return commandkit.Acquired[View]{Value: View{Sources: reader, Findings: &Findings{Records: records, Messages: []commandkit.Diagnostic{validationDiagnostic("native warnings")}}}, Diagnostics: []commandkit.Diagnostic{validationDiagnostic("application warning")}}, nil + }}, + {Name: "second", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) { + calls = append(calls, "second") + return commandkit.Acquired[View]{Value: View{Findings: &Findings{Records: []configkit.Diagnostic{{Code: "failed", Severity: configkit.SeverityError, Message: "failed finding"}}}}, Explanation: commandkit.Explanation{Message: "terminal failure"}}, cause + }}, + {Name: "third", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) { + calls = append(calls, "third") + return commandkit.Acquired[View]{}, nil + }}, + } + app := newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: scopes})) + var out, diag bytes.Buffer + result := app.Execute(context.Background(), []string{"config", "validate", "--scope=all", "--details", "--severity=warning", "--code=chosen", "--format=json", "--verbose"}, commandkit.Streams{Out: &out, Err: &diag}) + decoder := json.NewDecoder(&out) + var report validationReport + if err := decoder.Decode(&report); err != nil { + t.Fatal(err) + } + if err := decoder.Decode(new(any)); err != io.EOF { + t.Fatalf("extra JSON: %v", err) + } + if !errors.Is(result.Cause, cause) || result.Kind != commandkit.OutcomeOperational || len(calls) != 2 || calls[0] != "app" || calls[1] != "second" || len(report.Scopes) != 2 || report.Scopes[0].Outcome != "valid" || report.Scopes[1].Outcome != "failed" { + t.Fatalf("result=%#v calls=%v report=%#v", result, calls, report) + } + first, second := report.Scopes[0].Details, report.Scopes[1].Details + if first == nil || first.State != "present" || len(first.Findings) != 1 || first.Findings[0].Path != "/a~1b" || len(first.Sources) != 2 || first.Sources[0].ID != "primary" || first.Sources[1].ID != "related" || second == nil || second.State != "filtered" || len(second.Findings) != 0 { + t.Fatalf("details=%#v %#v", first, second) + } + if strings.Contains(diag.String(), "native warnings") || !strings.Contains(diag.String(), "application warning") || !strings.Contains(diag.String(), "terminal failure") { + t.Fatalf("diagnostics=%q", diag.String()) + } + before := len(calls) + for _, args := range [][]string{{"--severity=warning"}, {"--code=x"}, {"--details", "--severity=unknown"}} { + result = app.Execute(context.Background(), append([]string{"config", "validate"}, args...), commandkit.Streams{Out: io.Discard, Err: io.Discard}) + if result.Kind != commandkit.OutcomeUsage || len(calls) != before { + t.Fatalf("invalid %v: %#v", args, result) + } + } +} + +func TestValidationCancellationAndDeliveryRetainReachedScopes(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + calls := 0 + scopes := []Scope[struct{}, struct{}]{ + {Name: "app", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) { + calls++ + return commandkit.Acquired[View]{Value: View{Findings: &Findings{}}}, nil + }}, + {Name: "second", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) { + calls++ + cancel() + return commandkit.Acquired[View]{}, nil + }}, + {Name: "third", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) { + calls++ + return commandkit.Acquired[View]{}, nil + }}, + } + app := newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: scopes})) + var out bytes.Buffer + result := app.Execute(ctx, []string{"config", "validate", "--scope=all", "--details", "--format=json"}, commandkit.Streams{Out: &out, Err: io.Discard}) + var report validationReport + if err := json.Unmarshal(out.Bytes(), &report); err != nil { + t.Fatal(err) + } + if !errors.Is(result.Cause, context.Canceled) || calls != 2 || len(report.Scopes) != 2 || report.Scopes[0].Details.State != "empty" || report.Scopes[1].Details.State != "unavailable" || report.Scopes[1].Outcome != "canceled" { + t.Fatalf("result=%#v calls=%d report=%#v", result, calls, report) + } + failure := errors.New("load failure") + scope := Scope[struct{}, struct{}]{Name: "app", Operations: []Operation{Validate}, Acquire: func(context.Context, Request, struct{}, struct{}) (commandkit.Acquired[View], error) { + return commandkit.Acquired[View]{}, failure + }} + app = newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: []Scope[struct{}, struct{}]{scope}})) + result = app.Execute(context.Background(), []string{"config", "validate", "--details"}, commandkit.Streams{Out: shortWriter{}, Err: io.Discard}) + if !errors.Is(result.Cause, failure) || result.Presentation.Output.State != commandkit.DeliveryIncomplete { + t.Fatalf("failure and delivery: %#v", result) + } + out.Reset() + result = app.Execute(context.Background(), []string{"config", "validate", "--details", "--quiet"}, commandkit.Streams{Out: &out, Err: io.Discard}) + if !errors.Is(result.Cause, failure) || out.Len() != 0 { + t.Fatalf("quiet failure: %#v %q", result, out.String()) + } +} + +func TestDetailedValidationReportsNativeLoadError(t *testing.T) { + _, failure := configkit.Load[map[string]any](configkit.Bytes("invalid source", []byte("invalid: ["))) + var native *configkit.LoadError + if !errors.As(failure, &native) { + t.Fatalf("fixture: %v", failure) + } + scope := FromConfigKit("app", "", func(context.Context, Selection, struct{}, struct{}) (*configkit.Result[map[string]any], error) { + return nil, failure + }) + app := newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: []Scope[struct{}, struct{}]{scope}})) + var out bytes.Buffer + result := app.Execute(context.Background(), []string{"config", "validate", "--details", "--format=json"}, commandkit.Streams{Out: &out, Err: io.Discard}) + var report validationReport + if err := json.Unmarshal(out.Bytes(), &report); err != nil { + t.Fatal(err) + } + if !errors.Is(result.Cause, failure) || len(report.Scopes) != 1 || report.Scopes[0].Outcome != "failed" || report.Scopes[0].Details.State != "present" || len(report.Scopes[0].Details.Findings) != len(native.Diagnostics()) { + t.Fatalf("native failure: %#v %q", result, out.String()) + } +} diff --git a/integrations/ckconfig/input.go b/integrations/ckconfig/input.go index bbb1eff..4ee12ec 100644 --- a/integrations/ckconfig/input.go +++ b/integrations/ckconfig/input.go @@ -20,6 +20,8 @@ type configCommandInput[I, S any] struct { sourceID commandkit.Supplied[string] details commandkit.Supplied[bool] check commandkit.Supplied[bool] + severities []configkit.Severity + codes []string values commandkit.FlagValues[I] againstValues commandkit.FlagValues[I] plan *configOperationPlan[I, S] @@ -45,6 +47,8 @@ type configOperationPlan[I, S any] struct { sourceID commandkit.Supplied[string] details bool check bool + severities []configkit.Severity + codes []string list bool help bool historical bool @@ -94,6 +98,17 @@ func newConfigOperationCommand[I, S any](prepared *preparedSuite[I, S], operatio }), ) } + if operation == Validate { + flags = append(flags, + commandkit.BoolFlag(commandkit.FlagMeta{Name: "details", Usage: "Show native validation findings."}, false, func(input **configCommandInput[I, S], value commandkit.Supplied[bool]) { (*input).details = value }), + commandkit.RepeatFlag(commandkit.FlagMeta{Name: "severity", Usage: "Filter detailed findings by error or warning."}, []configkit.Severity(nil), parseValidationSeverity, func(input **configCommandInput[I, S], value commandkit.Supplied[[]configkit.Severity]) { + (*input).severities = value.Value + }), + commandkit.RepeatFlag(commandkit.FlagMeta{Name: "code", Usage: "Filter detailed findings by exact native code."}, []string(nil), parseConfigString, func(input **configCommandInput[I, S], value commandkit.Supplied[[]string]) { + (*input).codes = value.Value + }), + ) + } if supportsOverrides(prepared, operation) { flags = append(flags, commandkit.RepeatFlag(commandkit.FlagMeta{Name: "set", Usage: "Override POINTER=JSON (pointer keys cannot contain =)."}, []configkit.Override(nil), overrideParser("--set"), func(input **configCommandInput[I, S], value commandkit.Supplied[[]configkit.Override]) { @@ -179,6 +194,9 @@ func operationDeclaration(operation Operation) (string, string) { } func prepareConfigOperation[I, S any](prepared *preparedSuite[I, S], operation Operation, input *configCommandInput[I, S]) error { + if operation == Validate && !input.details.Value && (len(input.severities) > 0 || len(input.codes) > 0) { + return fmt.Errorf("--severity and --code require --details") + } if operation == Explain { if input.list.Value && input.path.Set { return fmt.Errorf("--list and --path cannot be combined") @@ -256,6 +274,8 @@ func prepareConfigOperation[I, S any](prepared *preparedSuite[I, S], operation O sourceID: input.sourceID, details: input.details.Value, check: input.check.Value, + severities: append([]configkit.Severity(nil), input.severities...), + codes: append([]string(nil), input.codes...), list: input.list.Value, historical: input.historical.Value, under: under, diff --git a/integrations/ckconfig/validate.go b/integrations/ckconfig/validate.go index d27de86..bcea16e 100644 --- a/integrations/ckconfig/validate.go +++ b/integrations/ckconfig/validate.go @@ -1,10 +1,13 @@ package ckconfig import ( + "errors" "fmt" + "slices" "text/template" commandkit "go.maximumdirect.net/commandkit" + configkit "go.maximumdirect.net/configkit" ) type validationReport struct { @@ -12,11 +15,48 @@ type validationReport struct { } type validationScope struct { - Scope string `json:"scope"` - Outcome string `json:"outcome"` + Scope string `json:"scope"` + Outcome string `json:"outcome"` + Details *validationDetails `json:"details,omitempty"` +} + +type validationDetails struct { + State string `json:"state"` + Findings []validationFinding `json:"findings"` + Sources []validationSource `json:"sources"` +} + +type validationFinding struct { + Code string `json:"code"` + Severity string `json:"severity"` + Path string `json:"path"` + Message string `json:"message"` + Primary *locationDTO `json:"primary"` + Related []locationDTO `json:"related"` +} + +type validationSource struct { + ID string `json:"id"` + Label string `json:"label"` + Path string `json:"path"` } var validationTextTemplate = template.Must(commandkit.ParseTextTemplate("validation", "{{range .Scopes}}{{.Scope}}: {{.Outcome}}\n{{end}}", nil)) +var validationDetailsTemplate = template.Must(commandkit.ParseTextTemplate("validation-details", `{{range .Scopes}}{{.Scope}}: {{.Outcome}} ({{.Details.State}}) +{{range .Details.Findings}} {{.Severity}} {{.Code}} {{printf "%q" .Path}}: {{.Message}} +{{if .Primary}} {{.Primary.SourceID}}:{{.Primary.Line}}:{{.Primary.Column}} +{{end}}{{range .Related}} {{.SourceID}}:{{.Line}}:{{.Column}} +{{end}}{{end}}{{range .Details.Sources}} source {{printf "%q" .ID}}: {{.Label}} {{.Path}} +{{end}}{{end}}`, nil)) + +func parseValidationSeverity(raw string) (configkit.Severity, error) { + switch configkit.Severity(raw) { + case configkit.SeverityError, configkit.SeverityWarning: + return configkit.Severity(raw), nil + default: + return "", fmt.Errorf("unknown validation severity %q", raw) + } +} func runConfigValidation[I, S any](invocation commandkit.Invocation, plan *configOperationPlan[I, S], state S) commandkit.Outcome { reporter := invocation.Reporter() @@ -29,30 +69,88 @@ func runConfigValidation[I, S any](invocation commandkit.Invocation, plan *confi } name := selected.scope.scope.Name if selected.scope.scope.Acquire == nil { - report.Scopes = append(report.Scopes, validationScope{Scope: name, Outcome: "failed"}) + entry := validationScope{Scope: name, Outcome: "failed"} + if plan.details { + entry.Details = projectValidationDetails(View{}, plan.severities, plan.codes) + } + report.Scopes = append(report.Scopes, entry) outcome = commandkit.Failure(fmt.Sprintf("Configuration scope %q has no acquisition callback.", name)) break } acquired, err := selected.scope.scope.Acquire(invocation.Context, Request{Operation: Validate, Selection: selected.selection}, selected.input, state) - if err != nil { - status := "failed" - if invocation.Context.Err() != nil { - status = "canceled" + queueConfigDiagnostics(reporter, acquired.Diagnostics) + if !plan.details && acquired.Value.Findings != nil { + queueConfigDiagnostics(reporter, acquired.Value.Findings.Messages) + } + entry := validationScope{Scope: name, Outcome: "valid"} + if plan.details { + entry.Details = projectValidationDetails(acquired.Value, plan.severities, plan.codes) + } + if canceled := invocation.Context.Err(); canceled != nil { + entry.Outcome = "canceled" + if err != nil && !errors.Is(err, canceled) { + err = errors.Join(err, canceled) + } else if err == nil { + err = canceled } - report.Scopes = append(report.Scopes, validationScope{Scope: name, Outcome: status}) - outcome = configAcquisitionFailure(reporter, "validate", name, acquired, err) + } else if err != nil { + entry.Outcome = "failed" + } + report.Scopes = append(report.Scopes, entry) + if err != nil { + message := fmt.Sprintf("Unable to validate configuration scope %q.", name) + if entry.Outcome == "canceled" { + message = "Configuration validation was canceled." + } + outcome = configFailure(message, acquired.Explanation, err) break } - queueConfigAcquisition(reporter, acquired) - if err := invocation.Context.Err(); err != nil { - report.Scopes = append(report.Scopes, validationScope{Scope: name, Outcome: "canceled"}) - outcome = configFailure("Configuration validation was canceled.", acquired.Explanation, err) - break - } - report.Scopes = append(report.Scopes, validationScope{Scope: name, Outcome: "valid"}) } - if len(report.Scopes) > 0 { - _ = reporter.Report(commandkit.Report{Value: report, TextTemplate: validationTextTemplate}) + text := validationTextTemplate + if plan.details { + text = validationDetailsTemplate } + _ = reporter.Report(commandkit.Report{Value: report, TextTemplate: text}) return outcome } + +func projectValidationDetails(view View, severities []configkit.Severity, codes []string) *validationDetails { + details := &validationDetails{State: "unavailable", Findings: []validationFinding{}, Sources: []validationSource{}} + if view.Findings == nil { + return details + } + if len(view.Findings.Records) == 0 { + details.State = "empty" + return details + } + details.State = "filtered" + references := make(map[string]bool) + for _, record := range view.Findings.Records { + if len(severities) > 0 && !slices.Contains(severities, record.Severity) { + continue + } + if len(codes) > 0 && !slices.Contains(codes, string(record.Code)) { + continue + } + finding := validationFinding{Code: string(record.Code), Severity: string(record.Severity), Path: record.Path.String(), Message: record.Message, Related: locationReports(record.Related)} + if record.Primary != nil { + finding.Primary = &locationDTO{SourceID: record.Primary.SourceID, Line: record.Primary.Line, Column: record.Primary.Column} + references[record.Primary.SourceID] = true + } + for _, location := range record.Related { + references[location.SourceID] = true + } + details.Findings = append(details.Findings, finding) + } + if len(details.Findings) > 0 { + details.State = "present" + } + if view.Sources != nil { + for _, source := range view.Sources.Sources() { + if references[source.ID] { + details.Sources = append(details.Sources, validationSource{ID: source.ID, Label: source.Label, Path: source.Path}) + } + } + } + return details +}