Add detailed comparison assertions

This commit is contained in:
2026-11-07 04:49:40 +00:00
parent 65b454b2ab
commit 1986c44f49
6 changed files with 184 additions and 2 deletions

View File

@@ -511,8 +511,21 @@ example config diff --config before.yml --against-config after.yml
example config diff --config base.yml --against-overlay candidate
example config diff --config base.yml --overlay current --against-overlay ""
example config diff --scope promptkit --config before.yml --against-config after.yml
example config diff --config before.yml --against-config after.yml --details --check
```
`--details` replaces the concise report with `{scope, differences}`, containing
one entry per native difference. Each entry has `path`, `before`, `after`, and
`hidden_changed`; each snapshot has `present`, `redacted`, and `value`. An
absent snapshot has `present: false` and a null value, distinct from a present
null. Empty comparisons contain `differences: []`.
`--check` returns status `3` when native differences exist and `0` when they do
not. Without it, a completed comparison succeeds even when differences exist.
Usage errors retain status `2`, and operational failures retain status `1`.
The report is still delivered in check mode; a presentation failure remains
separate evidence and does not replace a nonzero difference outcome.
For ConfigKit-backed scopes, ConfigKit compares frozen effective typed values
after defaults and resolution; source or provenance differences alone do not
create a change. PromptKit delegates its native comparison to ConfigKit. The

View File

@@ -1,6 +1,6 @@
# Shared reporting and PromptKit v0.20.0 implementation plan
Status: implementation in progress; Stages 1–18 complete.
Status: implementation in progress; Stages 1–19 complete.
## Scope, authority, and execution
@@ -680,6 +680,10 @@ source files again to recover evidence.
## Stage 19: Detailed differences and assertion outcomes
**Complete.** Detailed native difference reports and opt-in assertion status
are implemented. Root, ckconfig, and ckprompt workspace suites passed with Go
1.27.0, including concealed changes, null versus absence, and delivery failures.
Reuse existing Scope.Compare and native Difference records. Keep concise
additions/changes/deletions as default and add `--details`. The detailed report
is `{scope, differences: [...]}`; entries carry native `path`, `before`, `after`,

View File

@@ -0,0 +1,104 @@
package ckconfig
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"reflect"
"strings"
"testing"
commandkit "go.maximumdirect.net/commandkit"
configkit "go.maximumdirect.net/configkit"
)
func TestDetailedDiffPreservesNativeEvidenceAndAssertions(t *testing.T) {
values := []configkit.Difference{
{Path: configkit.Keys("added"), Right: configkit.ValueSnapshot{Present: true, YAML: "null\n"}},
{Path: configkit.Keys("removed"), Left: configkit.ValueSnapshot{Present: true, YAML: "false\n"}},
{Path: configkit.Keys("secret"), Left: configkit.ValueSnapshot{Present: true, Redacted: true, YAML: "'***'\n"}, Right: configkit.ValueSnapshot{Present: true, Redacted: true, YAML: "'***'\n"}, HiddenChanged: true},
{Path: configkit.Keys("list"), Left: configkit.ValueSnapshot{Present: true, YAML: "[one, two]\n"}, Right: configkit.ValueSnapshot{Present: true, YAML: "[three]\n"}},
{Path: configkit.Keys("marker"), Left: configkit.ValueSnapshot{Present: true, YAML: "'redacted'\n"}, Right: configkit.ValueSnapshot{Present: true, YAML: "'***'\n"}},
}
calls := 0
scope := Scope[struct{}, struct{}]{Name: "app", Operations: []Operation{Diff}, Compare: func(context.Context, ComparisonRequest[struct{}], struct{}) (commandkit.Acquired[[]configkit.Difference], error) {
calls++
return commandkit.Acquired[[]configkit.Difference]{Value: values}, nil
}}
app := newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: []Scope[struct{}, struct{}]{scope}}))
for _, format := range []string{"json", "yaml", "text"} {
var out bytes.Buffer
result := app.Execute(context.Background(), []string{"config", "diff", "--details", "--check", "--format=" + format}, commandkit.Streams{Out: &out, Err: io.Discard})
var logical any
var err error
if format == "json" {
err = json.Unmarshal(out.Bytes(), &logical)
} else {
logical, err = commandkit.DecodeYAMLReport(&out)
}
if result.Kind != commandkit.OutcomeDomain || result.Status != 3 || err != nil {
t.Fatalf("format %s = %#v %v %q", format, result, err, out.String())
}
report := logical.(map[string]any)
entries := report["differences"].([]any)
if report["scope"] != "app" || len(entries) != len(values) {
t.Fatalf("format %s report %#v", format, report)
}
added := entries[0].(map[string]any)
before, after := added["before"].(map[string]any), added["after"].(map[string]any)
if before["present"] != false || before["value"] != nil || after["present"] != true || after["value"] != nil {
t.Fatalf("null/absence: %#v", added)
}
secret := entries[2].(map[string]any)
if secret["hidden_changed"] != true || !reflect.DeepEqual(secret["before"].(map[string]any)["value"], secret["after"].(map[string]any)["value"]) {
t.Fatalf("hidden change: %#v", secret)
}
list := entries[3].(map[string]any)["after"].(map[string]any)["value"]
if !reflect.DeepEqual(list, []any{"three"}) {
t.Fatalf("atomic sequence: %#v", list)
}
marker := entries[4].(map[string]any)["after"].(map[string]any)
if marker["value"] != "***" || marker["redacted"] != false {
t.Fatalf("marker string: %#v", marker)
}
}
if calls != 3 {
t.Fatalf("provider calls=%d", calls)
}
result := app.Execute(context.Background(), []string{"config", "diff"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if result.Kind != commandkit.OutcomeSuccess || calls != 4 {
t.Fatalf("ordinary diff: %#v calls=%d", result, calls)
}
values = nil
var out bytes.Buffer
result = app.Execute(context.Background(), []string{"config", "diff", "--check", "--details", "--format=json"}, commandkit.Streams{Out: &out, Err: io.Discard})
if result.Kind != commandkit.OutcomeSuccess || !strings.Contains(out.String(), `"differences": []`) {
t.Fatalf("equality: %#v %q", result, out.String())
}
}
func TestDiffAssertionRetainsDomainOutcomeOnDeliveryFailures(t *testing.T) {
values := []configkit.Difference{{Path: configkit.Keys("x"), Right: configkit.ValueSnapshot{Present: true, YAML: "true\n"}}}
cause := errors.New("provider failed")
var providerErr error
scope := Scope[struct{}, struct{}]{Name: "app", Operations: []Operation{Diff}, Compare: func(context.Context, ComparisonRequest[struct{}], struct{}) (commandkit.Acquired[[]configkit.Difference], error) {
return commandkit.Acquired[[]configkit.Difference]{Value: values}, providerErr
}}
app := newConfigTestApp(t, Command(Suite[struct{}, struct{}]{Scopes: []Scope[struct{}, struct{}]{scope}}))
result := app.Execute(context.Background(), []string{"config", "diff", "--check", "--details"}, commandkit.Streams{Out: shortWriter{}, Err: io.Discard})
if result.Kind != commandkit.OutcomeDomain || result.Status != 3 || result.Presentation.Output.State != commandkit.DeliveryIncomplete {
t.Fatalf("writer failure: %#v", result)
}
values[0].Right.YAML = "[invalid"
result = app.Execute(context.Background(), []string{"config", "diff", "--check", "--details"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if result.Kind != commandkit.OutcomeDomain || result.Status != 3 || result.Presentation.Output.RenderError == nil {
t.Fatalf("render failure: %#v", result)
}
providerErr = cause
result = app.Execute(context.Background(), []string{"config", "diff", "--check"}, commandkit.Streams{Out: io.Discard, Err: io.Discard})
if result.Kind != commandkit.OutcomeOperational || !errors.Is(result.Cause, cause) {
t.Fatalf("provider failure: %#v", result)
}
}

View File

@@ -30,7 +30,14 @@ func runConfigDiff[I, S any](invocation commandkit.Invocation, plan *configOpera
if err := invocation.Context.Err(); err != nil {
return configFailure("Configuration comparison was canceled.", acquired.Explanation, err)
}
return renderPreparedConfigReport(invocation, acquired.Explanation, func() (any, error) {
outcome := renderPreparedConfigReport(invocation, acquired.Explanation, func() (any, error) {
if plan.details {
return renderDetailedDiff(selected.scope.scope.Name, acquired.Value)
}
return renderDiffReport(acquired.Value)
})
if outcome.Kind == commandkit.OutcomeSuccess && plan.check && len(acquired.Value) > 0 {
return commandkit.Domain("configuration differences", 3)
}
return outcome
}

View File

@@ -78,6 +78,52 @@ type diffReport struct {
Deletions []string `json:"deletions"`
}
type detailedDiffReport struct {
Scope string `json:"scope"`
Differences []detailedDifference `json:"differences"`
}
type detailedDifference struct {
Path string `json:"path"`
Before detailedDiffSnapshot `json:"before"`
After detailedDiffSnapshot `json:"after"`
HiddenChanged bool `json:"hidden_changed"`
}
type detailedDiffSnapshot struct {
Present bool `json:"present"`
Redacted bool `json:"redacted"`
Value any `json:"value"`
}
func renderDetailedDiff(scope string, differences []configkit.Difference) (any, error) {
report := detailedDiffReport{Scope: scope, Differences: make([]detailedDifference, 0, len(differences))}
for _, difference := range differences {
before, err := detailedSnapshot(difference.Left)
if err != nil {
return nil, fmt.Errorf("difference at %q before: %w", difference.Path.String(), err)
}
after, err := detailedSnapshot(difference.Right)
if err != nil {
return nil, fmt.Errorf("difference at %q after: %w", difference.Path.String(), err)
}
report.Differences = append(report.Differences, detailedDifference{Path: difference.Path.String(), Before: before, After: after, HiddenChanged: difference.HiddenChanged})
}
return report, nil
}
func detailedSnapshot(snapshot configkit.ValueSnapshot) (detailedDiffSnapshot, error) {
report := detailedDiffSnapshot{Present: snapshot.Present, Redacted: snapshot.Redacted}
if snapshot.Present {
value, err := commandkit.DecodeYAMLReport(strings.NewReader(snapshot.YAML))
if err != nil {
return report, err
}
report.Value = value
}
return report, nil
}
func renderDiffReport(differences []configkit.Difference) (any, error) {
report := diffReport{Additions: map[string]any{}, Changes: map[string]any{}, Deletions: []string{}}
for _, difference := range differences {

View File

@@ -18,6 +18,8 @@ type configCommandInput[I, S any] struct {
under commandkit.Supplied[string]
explicit commandkit.Supplied[bool]
sourceID commandkit.Supplied[string]
details commandkit.Supplied[bool]
check commandkit.Supplied[bool]
values commandkit.FlagValues[I]
againstValues commandkit.FlagValues[I]
plan *configOperationPlan[I, S]
@@ -41,6 +43,8 @@ type configOperationPlan[I, S any] struct {
path configkit.Path
pathSet bool
sourceID commandkit.Supplied[string]
details bool
check bool
list bool
help bool
historical bool
@@ -78,6 +82,8 @@ func newConfigOperationCommand[I, S any](prepared *preparedSuite[I, S], operatio
}
if operation == Diff {
flags = append(flags,
commandkit.BoolFlag(commandkit.FlagMeta{Name: "details", Usage: "Show native before and after difference evidence."}, false, func(input **configCommandInput[I, S], value commandkit.Supplied[bool]) { (*input).details = value }),
commandkit.BoolFlag(commandkit.FlagMeta{Name: "check", Usage: "Return status 3 when differences are found."}, false, func(input **configCommandInput[I, S], value commandkit.Supplied[bool]) { (*input).check = value }),
commandkit.ValueFlag(commandkit.FlagMeta{Name: "against-config", Usage: "Configuration source for the right side.", Completion: commandkit.CompleteFiles()}, (*string)(nil), parseConfigString,
func(input **configCommandInput[I, S], value commandkit.Supplied[string]) {
(*input).against.Config = value
@@ -248,6 +254,8 @@ func prepareConfigOperation[I, S any](prepared *preparedSuite[I, S], operation O
path: path,
pathSet: input.path.Set,
sourceID: input.sourceID,
details: input.details.Value,
check: input.check.Value,
list: input.list.Value,
historical: input.historical.Value,
under: under,