eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Verify configuration diff disclosure semantics

Browse Source
This commit is contained in:
Eric Rakestraw
2026-10-09 15:59:58 +00:00
parent fd82524977
commit 870e4ca4e6
2 changed files with 161 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
configuration_diff_contract_test.go
View File

@@ -68,13 +68,147 @@ func TestConfigurationInspectionDiffRejectsZeroViews(t *testing.T) {
}
}
func TestConfigurationInspectionDiffIgnoresSourceHistory(t *testing.T) {
left := comparisonInspectionFrom(t, "earlier source", "same-model", nil)
right := comparisonInspectionFrom(t, "later source", "same-model", nil)
if !hasSourceLabel(left, "earlier source") || !hasSourceLabel(right, "later source") {
t.Fatal("comparison fixtures did not retain their distinct source labels")
}
differences, err := left.Diff(right)
if err != nil || len(differences) != 0 {
t.Fatalf("compare equal effective values with different sources = (%+v, %v), want no differences", differences, err)
}
}
func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
"sequence": []any{"earlier", "unchanged"},
})
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
"nullable": nil,
"sequence": []any{"later", "unchanged"},
})
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare presence and sequence values: %v", err)
}
if len(differences) != 2 {
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
}
byPath := differencesByPath(differences)
nullable, ok := byPath["/backends/local/extra_params/nullable"]
if !ok || nullable.Left.Present || !nullable.Right.Present || strings.TrimSpace(nullable.Right.YAML) != "null" {
t.Fatalf("absent-to-null difference = (%+v, %t)", nullable, ok)
}
sequence, ok := byPath["/backends/local/extra_params/sequence"]
if !ok || !sequence.Left.Present || !sequence.Right.Present ||
!strings.Contains(sequence.Left.YAML, "earlier") || !strings.Contains(sequence.Right.YAML, "later") {
t.Fatalf("atomic sequence difference = (%+v, %t)", sequence, ok)
}
}
func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t *testing.T) {
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
"token": "left-secret-sentinel",
"visible": "before-value",
})
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
"token": "right-secret-sentinel",
"visible": "after-value",
})
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare sensitive values: %v", err)
}
if len(differences) != 2 {
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
}
byPath := differencesByPath(differences)
secret, ok := byPath["/backends/local/extra_params/token"]
if !ok || !secret.Left.Redacted || !secret.Right.Redacted || !secret.HiddenChanged {
t.Fatalf("sensitive difference = (%+v, %t)", secret, ok)
}
if strings.Contains(secret.Left.YAML, "left-secret-sentinel") || strings.Contains(secret.Left.YAML, "right-secret-sentinel") ||
strings.Contains(secret.Right.YAML, "left-secret-sentinel") || strings.Contains(secret.Right.YAML, "right-secret-sentinel") {
t.Fatalf("sensitive snapshots disclosed a value: %+v", secret)
}
visible, ok := byPath["/backends/local/extra_params/visible"]
if !ok || visible.Left.Redacted || visible.Right.Redacted || visible.HiddenChanged ||
!strings.Contains(visible.Left.YAML, "before-value") || !strings.Contains(visible.Right.YAML, "after-value") {
t.Fatalf("public sibling difference = (%+v, %t)", visible, ok)
}
differences[0].Path = configkit.Keys("caller", "mutation")
differences[0].Left.YAML = "caller mutation"
differences[0].Right.YAML = "caller mutation"
again, err := left.Diff(right)
if err != nil {
t.Fatalf("repeat comparison after mutating returned records: %v", err)
}
if len(again) != 2 {
t.Fatalf("repeat difference count = %d, want 2: %+v", len(again), again)
}
againByPath := differencesByPath(again)
if _, exists := againByPath["/caller/mutation"]; exists {
t.Fatal("mutating returned path reached retained inspection")
}
if _, exists := againByPath["/backends/local/extra_params/token"]; !exists {
t.Fatalf("original sensitive path was lost after mutation: %+v", again)
}
if _, exists := againByPath["/backends/local/extra_params/visible"]; !exists {
t.Fatalf("original public path was lost after mutation: %+v", again)
}
for _, difference := range again {
if difference.Left.YAML == "caller mutation" || difference.Right.YAML == "caller mutation" {
t.Fatalf("mutating returned snapshots reached retained inspection: %+v", difference)
}
}
}
func TestConfigurationInspectionDiffMasksSensitiveShapeReplacement(t *testing.T) {
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
"details": map[string]any{
"token": "nested-secret-sentinel",
"label": "public detail",
},
})
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
"details": "replacement",
})
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare sensitive shape replacement: %v", err)
}
if len(differences) != 1 || differences[0].Path.String() != "/backends/local/extra_params/details" {
t.Fatalf("shape replacement differences = %+v", differences)
}
difference := differences[0]
if !difference.Left.Redacted || !difference.Right.Redacted || !difference.HiddenChanged {
t.Fatalf("shape replacement masking = %+v", difference)
}
if strings.Contains(difference.Left.YAML, "nested-secret-sentinel") || strings.Contains(difference.Right.YAML, "nested-secret-sentinel") {
t.Fatalf("shape replacement disclosed a sensitive value: %+v", difference)
}
}
func comparisonInspection(t *testing.T, model string) promptkit.ConfigurationInspection {
t.Helper()
return comparisonInspectionFrom(t, "comparison fixture", model, nil)
}
func comparisonInspectionFrom(t *testing.T, source, model string, extraParams map[string]any) promptkit.ConfigurationInspection {
t.Helper()
backend := map[string]any{"endpoint": "https://example.invalid/v1"}
if extraParams != nil {
backend["extra_params"] = extraParams
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping("comparison fixture", map[string]any{
"backends": map[string]any{"local": map[string]any{
"endpoint": "https://example.invalid/v1",
}},
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping(source, map[string]any{
"backends": map[string]any{"local": backend},
"profiles": map[string]any{"offline": map[string]any{
"backend": "local",
"model": model,
@@ -90,3 +224,20 @@ func comparisonInspection(t *testing.T, model string) promptkit.ConfigurationIns
}
return inspection
}
func differencesByPath(differences []configkit.Difference) map[string]configkit.Difference {
byPath := make(map[string]configkit.Difference, len(differences))
for _, difference := range differences {
byPath[difference.Path.String()] = difference
}
return byPath
}
func hasSourceLabel(view promptkit.ConfigurationInspection, label string) bool {
for _, source := range view.Sources() {
if source.Label == label {
return true
}
}
return false
}

6
docs/roadmap/implementation.md
View File

@@ -121,6 +121,12 @@ still behave as before. No runner or domain-package integration is needed.
## Stage 3: Verify Result Semantics And Sensitive Changes
**Status: Complete.** Facade tests cover distinct source labels with equal
effective values, absent versus null, atomic sequence changes, paired secret
masking, sensitive shape replacement, and mutation of returned records.
`GOTOOLCHAIN=local go test . ./internal/configuration` and
`git diff --check` passed.
Scope: focused facade integration tests, not a second comparator test suite.
1. Extend the comparison tests with two valid engine configurations whose