Add detailed comparison assertions
This commit is contained in:
13
docs/cli.md
13
docs/cli.md
@@ -511,8 +511,21 @@ example config diff --config before.yml --against-config after.yml
|
||||
example config diff --config base.yml --against-overlay candidate
|
||||
example config diff --config base.yml --overlay current --against-overlay ""
|
||||
example config diff --scope promptkit --config before.yml --against-config after.yml
|
||||
example config diff --config before.yml --against-config after.yml --details --check
|
||||
```
|
||||
|
||||
`--details` replaces the concise report with `{scope, differences}`, containing
|
||||
one entry per native difference. Each entry has `path`, `before`, `after`, and
|
||||
`hidden_changed`; each snapshot has `present`, `redacted`, and `value`. An
|
||||
absent snapshot has `present: false` and a null value, distinct from a present
|
||||
null. Empty comparisons contain `differences: []`.
|
||||
|
||||
`--check` returns status `3` when native differences exist and `0` when they do
|
||||
not. Without it, a completed comparison succeeds even when differences exist.
|
||||
Usage errors retain status `2`, and operational failures retain status `1`.
|
||||
The report is still delivered in check mode; a presentation failure remains
|
||||
separate evidence and does not replace a nonzero difference outcome.
|
||||
|
||||
For ConfigKit-backed scopes, ConfigKit compares frozen effective typed values
|
||||
after defaults and resolution; source or provenance differences alone do not
|
||||
create a change. PromptKit delegates its native comparison to ConfigKit. The
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Shared reporting and PromptKit v0.20.0 implementation plan
|
||||
|
||||
Status: implementation in progress; Stages 1–18 complete.
|
||||
Status: implementation in progress; Stages 1–19 complete.
|
||||
|
||||
## Scope, authority, and execution
|
||||
|
||||
@@ -680,6 +680,10 @@ source files again to recover evidence.
|
||||
|
||||
## Stage 19: Detailed differences and assertion outcomes
|
||||
|
||||
**Complete.** Detailed native difference reports and opt-in assertion status
|
||||
are implemented. Root, ckconfig, and ckprompt workspace suites passed with Go
|
||||
1.27.0, including concealed changes, null versus absence, and delivery failures.
|
||||
|
||||
Reuse existing Scope.Compare and native Difference records. Keep concise
|
||||
additions/changes/deletions as default and add `--details`. The detailed report
|
||||
is `{scope, differences: [...]}`; entries carry native `path`, `before`, `after`,
|
||||
|
||||
104
integrations/ckconfig/detailed_diff_test.go
Normal file
104
integrations/ckconfig/detailed_diff_test.go
Normal file
@@ -0,0 +1,104 @@
|
||||
package ckconfig
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
commandkit "go.maximumdirect.net/commandkit"
|
||||
configkit "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func TestDetailedDiffPreservesNativeEvidenceAndAssertions(t *testing.T) {
|
||||
values := []configkit.Difference{
|
||||
{Path: configkit.Keys("added"), Right: configkit.ValueSnapshot{Present: true, YAML: "null\n"}},
|
||||
{Path: configkit.Keys("removed"), Left: configkit.ValueSnapshot{Present: true, YAML: "false\n"}},
|
||||
{Path: configkit.Keys("secret"), Left: configkit.ValueSnapshot{Present: true, Redacted: true, YAML: "'***'\n"}, Right: configkit.ValueSnapshot{Present: true, Redacted: true, YAML: "'***'\n"}, HiddenChanged: true},
|
||||
{Path: configkit.Keys("list"), Left: configkit.ValueSnapshot{Present: true, YAML: "[one, two]\n"}, Right: configkit.ValueSnapshot{Present: true, YAML: "[three]\n"}},
|
||||
{Path: configkit.Keys("marker"), Left: configkit.ValueSnapshot{Present: true, YAML: "'redacted'\n"}, Right: configkit.ValueSnapshot{Present: true, YAML: "'***'\n"}},
|
||||
}
|
||||
calls := 0
|
||||
scope := Scope[struct{}, struct{}]{Name: "app", Operations: []Operation{Diff}, Compare: func(context.Context, ComparisonRequest[struct{}], struct{}) (commandkit.Acquired[[]configkit.Difference], error) {
|
||||
calls++
|
||||
return commandkit.Acquired[[]configkit.Difference]{Value: values}, nil
|
||||
}}
|
||||
app := newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: []Scope[struct{}, struct{}]{scope}}))
|
||||
for _, format := range []string{"json", "yaml", "text"} {
|
||||
var out bytes.Buffer
|
||||
result := app.Execute(context.Background(), []string{"config", "diff", "--details", "--check", "--format=" + format}, commandkit.Streams{Out: &out, Err: io.Discard})
|
||||
var logical any
|
||||
var err error
|
||||
if format == "json" {
|
||||
err = json.Unmarshal(out.Bytes(), &logical)
|
||||
} else {
|
||||
logical, err = commandkit.DecodeYAMLReport(&out)
|
||||
}
|
||||
if result.Kind != commandkit.OutcomeDomain || result.Status != 3 || err != nil {
|
||||
t.Fatalf("format %s = %#v %v %q", format, result, err, out.String())
|
||||
}
|
||||
report := logical.(map[string]any)
|
||||
entries := report["differences"].([]any)
|
||||
if report["scope"] != "app" || len(entries) != len(values) {
|
||||
t.Fatalf("format %s report %#v", format, report)
|
||||
}
|
||||
added := entries[0].(map[string]any)
|
||||
before, after := added["before"].(map[string]any), added["after"].(map[string]any)
|
||||
if before["present"] != false || before["value"] != nil || after["present"] != true || after["value"] != nil {
|
||||
t.Fatalf("null/absence: %#v", added)
|
||||
}
|
||||
secret := entries[2].(map[string]any)
|
||||
if secret["hidden_changed"] != true || !reflect.DeepEqual(secret["before"].(map[string]any)["value"], secret["after"].(map[string]any)["value"]) {
|
||||
t.Fatalf("hidden change: %#v", secret)
|
||||
}
|
||||
list := entries[3].(map[string]any)["after"].(map[string]any)["value"]
|
||||
if !reflect.DeepEqual(list, []any{"three"}) {
|
||||
t.Fatalf("atomic sequence: %#v", list)
|
||||
}
|
||||
marker := entries[4].(map[string]any)["after"].(map[string]any)
|
||||
if marker["value"] != "***" || marker["redacted"] != false {
|
||||
t.Fatalf("marker string: %#v", marker)
|
||||
}
|
||||
}
|
||||
if calls != 3 {
|
||||
t.Fatalf("provider calls=%d", calls)
|
||||
}
|
||||
result := app.Execute(context.Background(), []string{"config", "diff"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||
if result.Kind != commandkit.OutcomeSuccess || calls != 4 {
|
||||
t.Fatalf("ordinary diff: %#v calls=%d", result, calls)
|
||||
}
|
||||
values = nil
|
||||
var out bytes.Buffer
|
||||
result = app.Execute(context.Background(), []string{"config", "diff", "--check", "--details", "--format=json"}, commandkit.Streams{Out: &out, Err: io.Discard})
|
||||
if result.Kind != commandkit.OutcomeSuccess || !strings.Contains(out.String(), `"differences": []`) {
|
||||
t.Fatalf("equality: %#v %q", result, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffAssertionRetainsDomainOutcomeOnDeliveryFailures(t *testing.T) {
|
||||
values := []configkit.Difference{{Path: configkit.Keys("x"), Right: configkit.ValueSnapshot{Present: true, YAML: "true\n"}}}
|
||||
cause := errors.New("provider failed")
|
||||
var providerErr error
|
||||
scope := Scope[struct{}, struct{}]{Name: "app", Operations: []Operation{Diff}, Compare: func(context.Context, ComparisonRequest[struct{}], struct{}) (commandkit.Acquired[[]configkit.Difference], error) {
|
||||
return commandkit.Acquired[[]configkit.Difference]{Value: values}, providerErr
|
||||
}}
|
||||
app := newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: []Scope[struct{}, struct{}]{scope}}))
|
||||
result := app.Execute(context.Background(), []string{"config", "diff", "--check", "--details"}, commandkit.Streams{Out: shortWriter{}, Err: io.Discard})
|
||||
if result.Kind != commandkit.OutcomeDomain || result.Status != 3 || result.Presentation.Output.State != commandkit.DeliveryIncomplete {
|
||||
t.Fatalf("writer failure: %#v", result)
|
||||
}
|
||||
values[0].Right.YAML = "[invalid"
|
||||
result = app.Execute(context.Background(), []string{"config", "diff", "--check", "--details"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||
if result.Kind != commandkit.OutcomeDomain || result.Status != 3 || result.Presentation.Output.RenderError == nil {
|
||||
t.Fatalf("render failure: %#v", result)
|
||||
}
|
||||
providerErr = cause
|
||||
result = app.Execute(context.Background(), []string{"config", "diff", "--check"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
|
||||
if result.Kind != commandkit.OutcomeOperational || !errors.Is(result.Cause, cause) {
|
||||
t.Fatalf("provider failure: %#v", result)
|
||||
}
|
||||
}
|
||||
@@ -30,7 +30,14 @@ func runConfigDiff[I, S any](invocation commandkit.Invocation, plan *configOpera
|
||||
if err := invocation.Context.Err(); err != nil {
|
||||
return configFailure("Configuration comparison was canceled.", acquired.Explanation, err)
|
||||
}
|
||||
return renderPreparedConfigReport(invocation, acquired.Explanation, func() (any, error) {
|
||||
outcome := renderPreparedConfigReport(invocation, acquired.Explanation, func() (any, error) {
|
||||
if plan.details {
|
||||
return renderDetailedDiff(selected.scope.scope.Name, acquired.Value)
|
||||
}
|
||||
return renderDiffReport(acquired.Value)
|
||||
})
|
||||
if outcome.Kind == commandkit.OutcomeSuccess && plan.check && len(acquired.Value) > 0 {
|
||||
return commandkit.Domain("configuration differences", 3)
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
|
||||
@@ -78,6 +78,52 @@ type diffReport struct {
|
||||
Deletions []string `json:"deletions"`
|
||||
}
|
||||
|
||||
type detailedDiffReport struct {
|
||||
Scope string `json:"scope"`
|
||||
Differences []detailedDifference `json:"differences"`
|
||||
}
|
||||
|
||||
type detailedDifference struct {
|
||||
Path string `json:"path"`
|
||||
Before detailedDiffSnapshot `json:"before"`
|
||||
After detailedDiffSnapshot `json:"after"`
|
||||
HiddenChanged bool `json:"hidden_changed"`
|
||||
}
|
||||
|
||||
type detailedDiffSnapshot struct {
|
||||
Present bool `json:"present"`
|
||||
Redacted bool `json:"redacted"`
|
||||
Value any `json:"value"`
|
||||
}
|
||||
|
||||
func renderDetailedDiff(scope string, differences []configkit.Difference) (any, error) {
|
||||
report := detailedDiffReport{Scope: scope, Differences: make([]detailedDifference, 0, len(differences))}
|
||||
for _, difference := range differences {
|
||||
before, err := detailedSnapshot(difference.Left)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("difference at %q before: %w", difference.Path.String(), err)
|
||||
}
|
||||
after, err := detailedSnapshot(difference.Right)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("difference at %q after: %w", difference.Path.String(), err)
|
||||
}
|
||||
report.Differences = append(report.Differences, detailedDifference{Path: difference.Path.String(), Before: before, After: after, HiddenChanged: difference.HiddenChanged})
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func detailedSnapshot(snapshot configkit.ValueSnapshot) (detailedDiffSnapshot, error) {
|
||||
report := detailedDiffSnapshot{Present: snapshot.Present, Redacted: snapshot.Redacted}
|
||||
if snapshot.Present {
|
||||
value, err := commandkit.DecodeYAMLReport(strings.NewReader(snapshot.YAML))
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.Value = value
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func renderDiffReport(differences []configkit.Difference) (any, error) {
|
||||
report := diffReport{Additions: map[string]any{}, Changes: map[string]any{}, Deletions: []string{}}
|
||||
for _, difference := range differences {
|
||||
|
||||
@@ -18,6 +18,8 @@ type configCommandInput[I, S any] struct {
|
||||
under commandkit.Supplied[string]
|
||||
explicit commandkit.Supplied[bool]
|
||||
sourceID commandkit.Supplied[string]
|
||||
details commandkit.Supplied[bool]
|
||||
check commandkit.Supplied[bool]
|
||||
values commandkit.FlagValues[I]
|
||||
againstValues commandkit.FlagValues[I]
|
||||
plan *configOperationPlan[I, S]
|
||||
@@ -41,6 +43,8 @@ type configOperationPlan[I, S any] struct {
|
||||
path configkit.Path
|
||||
pathSet bool
|
||||
sourceID commandkit.Supplied[string]
|
||||
details bool
|
||||
check bool
|
||||
list bool
|
||||
help bool
|
||||
historical bool
|
||||
@@ -78,6 +82,8 @@ func newConfigOperationCommand[I, S any](prepared *preparedSuite[I, S], operatio
|
||||
}
|
||||
if operation == Diff {
|
||||
flags = append(flags,
|
||||
commandkit.BoolFlag(commandkit.FlagMeta{Name: "details", Usage: "Show native before and after difference evidence."}, false, func(input **configCommandInput[I, S], value commandkit.Supplied[bool]) { (*input).details = value }),
|
||||
commandkit.BoolFlag(commandkit.FlagMeta{Name: "check", Usage: "Return status 3 when differences are found."}, false, func(input **configCommandInput[I, S], value commandkit.Supplied[bool]) { (*input).check = value }),
|
||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "against-config", Usage: "Configuration source for the right side.", Completion: commandkit.CompleteFiles()}, (*string)(nil), parseConfigString,
|
||||
func(input **configCommandInput[I, S], value commandkit.Supplied[string]) {
|
||||
(*input).against.Config = value
|
||||
@@ -248,6 +254,8 @@ func prepareConfigOperation[I, S any](prepared *preparedSuite[I, S], operation O
|
||||
path: path,
|
||||
pathSet: input.path.Set,
|
||||
sourceID: input.sourceID,
|
||||
details: input.details.Value,
|
||||
check: input.check.Value,
|
||||
list: input.list.Value,
|
||||
historical: input.historical.Value,
|
||||
under: under,
|
||||
|
||||
Reference in New Issue
Block a user