244 lines
9.1 KiB
Go
244 lines
9.1 KiB
Go
package promptkit_test
|
|
|
|
import (
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
|
"gitea.maximumdirect.net/eric/promptkit"
|
|
)
|
|
|
|
func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) {
|
|
left := comparisonInspection(t, "earlier-model")
|
|
right := comparisonInspection(t, "later-model")
|
|
|
|
differences, err := left.Diff(right)
|
|
if err != nil {
|
|
t.Fatalf("compare captured configurations: %v", err)
|
|
}
|
|
if len(differences) != 1 {
|
|
t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences)
|
|
}
|
|
difference := differences[0]
|
|
if got, want := difference.Path.String(), "/profiles/offline/model"; got != want {
|
|
t.Fatalf("difference path = %q, want %q", got, want)
|
|
}
|
|
if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") {
|
|
t.Fatalf("difference snapshots = (%q, %q)", difference.Left.YAML, difference.Right.YAML)
|
|
}
|
|
|
|
reversed, err := right.Diff(left)
|
|
if err != nil {
|
|
t.Fatalf("compare reversed configurations: %v", err)
|
|
}
|
|
if len(reversed) != 1 || reversed[0].Path.String() != difference.Path.String() ||
|
|
!strings.Contains(reversed[0].Left.YAML, "later-model") ||
|
|
!strings.Contains(reversed[0].Right.YAML, "earlier-model") {
|
|
t.Fatalf("reversed difference = %+v", reversed)
|
|
}
|
|
|
|
equal := comparisonInspection(t, "earlier-model")
|
|
if equal, err := left.Diff(equal); err != nil || len(equal) != 0 {
|
|
t.Fatalf("compare equal configurations = (%+v, %v), want no differences", equal, err)
|
|
}
|
|
if equal, err := left.Diff(left); err != nil || len(equal) != 0 {
|
|
t.Fatalf("compare view with itself = (%+v, %v), want no differences", equal, err)
|
|
}
|
|
}
|
|
|
|
func TestConfigurationInspectionDiffRejectsZeroViews(t *testing.T) {
|
|
valid := comparisonInspection(t, "current-model")
|
|
var zero promptkit.ConfigurationInspection
|
|
|
|
for _, comparison := range []struct {
|
|
name string
|
|
left, right promptkit.ConfigurationInspection
|
|
}{
|
|
{name: "zero left", left: zero, right: valid},
|
|
{name: "zero right", left: valid, right: zero},
|
|
{name: "both zero", left: zero, right: zero},
|
|
} {
|
|
t.Run(comparison.name, func(t *testing.T) {
|
|
differences, err := comparison.left.Diff(comparison.right)
|
|
if !errors.Is(err, promptkit.ErrInvalidConfig) || len(differences) != 0 {
|
|
t.Fatalf("Diff() = (%+v, %v), want ErrInvalidConfig and no records", differences, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestConfigurationInspectionDiffIgnoresSourceHistory(t *testing.T) {
|
|
left := comparisonInspectionFrom(t, "earlier source", "same-model", nil)
|
|
right := comparisonInspectionFrom(t, "later source", "same-model", nil)
|
|
if !hasSourceLabel(left, "earlier source") || !hasSourceLabel(right, "later source") {
|
|
t.Fatal("comparison fixtures did not retain their distinct source labels")
|
|
}
|
|
|
|
differences, err := left.Diff(right)
|
|
if err != nil || len(differences) != 0 {
|
|
t.Fatalf("compare equal effective values with different sources = (%+v, %v), want no differences", differences, err)
|
|
}
|
|
}
|
|
|
|
func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
|
|
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
|
"sequence": []any{"earlier", "unchanged"},
|
|
})
|
|
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
|
"nullable": nil,
|
|
"sequence": []any{"later", "unchanged"},
|
|
})
|
|
|
|
differences, err := left.Diff(right)
|
|
if err != nil {
|
|
t.Fatalf("compare presence and sequence values: %v", err)
|
|
}
|
|
if len(differences) != 2 {
|
|
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
|
|
}
|
|
byPath := differencesByPath(differences)
|
|
nullable, ok := byPath["/backends/local/extra_params/nullable"]
|
|
if !ok || nullable.Left.Present || !nullable.Right.Present || strings.TrimSpace(nullable.Right.YAML) != "null" {
|
|
t.Fatalf("absent-to-null difference = (%+v, %t)", nullable, ok)
|
|
}
|
|
sequence, ok := byPath["/backends/local/extra_params/sequence"]
|
|
if !ok || !sequence.Left.Present || !sequence.Right.Present ||
|
|
!strings.Contains(sequence.Left.YAML, "earlier") || !strings.Contains(sequence.Right.YAML, "later") {
|
|
t.Fatalf("atomic sequence difference = (%+v, %t)", sequence, ok)
|
|
}
|
|
}
|
|
|
|
func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t *testing.T) {
|
|
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
|
"token": "left-secret-sentinel",
|
|
"visible": "before-value",
|
|
})
|
|
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
|
"token": "right-secret-sentinel",
|
|
"visible": "after-value",
|
|
})
|
|
|
|
differences, err := left.Diff(right)
|
|
if err != nil {
|
|
t.Fatalf("compare sensitive values: %v", err)
|
|
}
|
|
if len(differences) != 2 {
|
|
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
|
|
}
|
|
byPath := differencesByPath(differences)
|
|
secret, ok := byPath["/backends/local/extra_params/token"]
|
|
if !ok || !secret.Left.Redacted || !secret.Right.Redacted || !secret.HiddenChanged {
|
|
t.Fatalf("sensitive difference = (%+v, %t)", secret, ok)
|
|
}
|
|
if strings.Contains(secret.Left.YAML, "left-secret-sentinel") || strings.Contains(secret.Left.YAML, "right-secret-sentinel") ||
|
|
strings.Contains(secret.Right.YAML, "left-secret-sentinel") || strings.Contains(secret.Right.YAML, "right-secret-sentinel") {
|
|
t.Fatalf("sensitive snapshots disclosed a value: %+v", secret)
|
|
}
|
|
visible, ok := byPath["/backends/local/extra_params/visible"]
|
|
if !ok || visible.Left.Redacted || visible.Right.Redacted || visible.HiddenChanged ||
|
|
!strings.Contains(visible.Left.YAML, "before-value") || !strings.Contains(visible.Right.YAML, "after-value") {
|
|
t.Fatalf("public sibling difference = (%+v, %t)", visible, ok)
|
|
}
|
|
|
|
differences[0].Path = configkit.Keys("caller", "mutation")
|
|
differences[0].Left.YAML = "caller mutation"
|
|
differences[0].Right.YAML = "caller mutation"
|
|
again, err := left.Diff(right)
|
|
if err != nil {
|
|
t.Fatalf("repeat comparison after mutating returned records: %v", err)
|
|
}
|
|
if len(again) != 2 {
|
|
t.Fatalf("repeat difference count = %d, want 2: %+v", len(again), again)
|
|
}
|
|
againByPath := differencesByPath(again)
|
|
if _, exists := againByPath["/caller/mutation"]; exists {
|
|
t.Fatal("mutating returned path reached retained inspection")
|
|
}
|
|
if _, exists := againByPath["/backends/local/extra_params/token"]; !exists {
|
|
t.Fatalf("original sensitive path was lost after mutation: %+v", again)
|
|
}
|
|
if _, exists := againByPath["/backends/local/extra_params/visible"]; !exists {
|
|
t.Fatalf("original public path was lost after mutation: %+v", again)
|
|
}
|
|
for _, difference := range again {
|
|
if difference.Left.YAML == "caller mutation" || difference.Right.YAML == "caller mutation" {
|
|
t.Fatalf("mutating returned snapshots reached retained inspection: %+v", difference)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestConfigurationInspectionDiffMasksSensitiveShapeReplacement(t *testing.T) {
|
|
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
|
"details": map[string]any{
|
|
"token": "nested-secret-sentinel",
|
|
"label": "public detail",
|
|
},
|
|
})
|
|
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
|
"details": "replacement",
|
|
})
|
|
|
|
differences, err := left.Diff(right)
|
|
if err != nil {
|
|
t.Fatalf("compare sensitive shape replacement: %v", err)
|
|
}
|
|
if len(differences) != 1 || differences[0].Path.String() != "/backends/local/extra_params/details" {
|
|
t.Fatalf("shape replacement differences = %+v", differences)
|
|
}
|
|
difference := differences[0]
|
|
if !difference.Left.Redacted || !difference.Right.Redacted || !difference.HiddenChanged {
|
|
t.Fatalf("shape replacement masking = %+v", difference)
|
|
}
|
|
if strings.Contains(difference.Left.YAML, "nested-secret-sentinel") || strings.Contains(difference.Right.YAML, "nested-secret-sentinel") {
|
|
t.Fatalf("shape replacement disclosed a sensitive value: %+v", difference)
|
|
}
|
|
}
|
|
|
|
func comparisonInspection(t *testing.T, model string) promptkit.ConfigurationInspection {
|
|
t.Helper()
|
|
return comparisonInspectionFrom(t, "comparison fixture", model, nil)
|
|
}
|
|
|
|
func comparisonInspectionFrom(t *testing.T, source, model string, extraParams map[string]any) promptkit.ConfigurationInspection {
|
|
t.Helper()
|
|
backend := map[string]any{"endpoint": "https://example.invalid/v1"}
|
|
if extraParams != nil {
|
|
backend["extra_params"] = extraParams
|
|
}
|
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping(source, map[string]any{
|
|
"backends": map[string]any{"local": backend},
|
|
"profiles": map[string]any{"offline": map[string]any{
|
|
"backend": "local",
|
|
"model": model,
|
|
}},
|
|
})}}}},
|
|
}})
|
|
if err != nil {
|
|
t.Fatalf("construct comparison engine: %v", err)
|
|
}
|
|
inspection, err := engine.InspectConfiguration()
|
|
if err != nil {
|
|
t.Fatalf("inspect comparison engine: %v", err)
|
|
}
|
|
return inspection
|
|
}
|
|
|
|
func differencesByPath(differences []configkit.Difference) map[string]configkit.Difference {
|
|
byPath := make(map[string]configkit.Difference, len(differences))
|
|
for _, difference := range differences {
|
|
byPath[difference.Path.String()] = difference
|
|
}
|
|
return byPath
|
|
}
|
|
|
|
func hasSourceLabel(view promptkit.ConfigurationInspection, label string) bool {
|
|
for _, source := range view.Sources() {
|
|
if source.Label == label {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|