Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 143c6e9b3a | |||
| 48e4784fcd | |||
| 78af5069c1 | |||
| 18963266d7 | |||
| d039a019cb | |||
| d1c0281053 | |||
| 870e4ca4e6 | |||
| fd82524977 | |||
| 3f76c75919 | |||
| fe764f72d8 | |||
| 26ab566f38 |
243
configuration_diff_contract_test.go
Normal file
243
configuration_diff_contract_test.go
Normal file
@@ -0,0 +1,243 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) {
|
||||||
|
left := comparisonInspection(t, "earlier-model")
|
||||||
|
right := comparisonInspection(t, "later-model")
|
||||||
|
|
||||||
|
differences, err := left.Diff(right)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compare captured configurations: %v", err)
|
||||||
|
}
|
||||||
|
if len(differences) != 1 {
|
||||||
|
t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences)
|
||||||
|
}
|
||||||
|
difference := differences[0]
|
||||||
|
if got, want := difference.Path.String(), "/profiles/offline/model"; got != want {
|
||||||
|
t.Fatalf("difference path = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") {
|
||||||
|
t.Fatalf("difference snapshots = (%q, %q)", difference.Left.YAML, difference.Right.YAML)
|
||||||
|
}
|
||||||
|
|
||||||
|
reversed, err := right.Diff(left)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compare reversed configurations: %v", err)
|
||||||
|
}
|
||||||
|
if len(reversed) != 1 || reversed[0].Path.String() != difference.Path.String() ||
|
||||||
|
!strings.Contains(reversed[0].Left.YAML, "later-model") ||
|
||||||
|
!strings.Contains(reversed[0].Right.YAML, "earlier-model") {
|
||||||
|
t.Fatalf("reversed difference = %+v", reversed)
|
||||||
|
}
|
||||||
|
|
||||||
|
equal := comparisonInspection(t, "earlier-model")
|
||||||
|
if equal, err := left.Diff(equal); err != nil || len(equal) != 0 {
|
||||||
|
t.Fatalf("compare equal configurations = (%+v, %v), want no differences", equal, err)
|
||||||
|
}
|
||||||
|
if equal, err := left.Diff(left); err != nil || len(equal) != 0 {
|
||||||
|
t.Fatalf("compare view with itself = (%+v, %v), want no differences", equal, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffRejectsZeroViews(t *testing.T) {
|
||||||
|
valid := comparisonInspection(t, "current-model")
|
||||||
|
var zero promptkit.ConfigurationInspection
|
||||||
|
|
||||||
|
for _, comparison := range []struct {
|
||||||
|
name string
|
||||||
|
left, right promptkit.ConfigurationInspection
|
||||||
|
}{
|
||||||
|
{name: "zero left", left: zero, right: valid},
|
||||||
|
{name: "zero right", left: valid, right: zero},
|
||||||
|
{name: "both zero", left: zero, right: zero},
|
||||||
|
} {
|
||||||
|
t.Run(comparison.name, func(t *testing.T) {
|
||||||
|
differences, err := comparison.left.Diff(comparison.right)
|
||||||
|
if !errors.Is(err, promptkit.ErrInvalidConfig) || len(differences) != 0 {
|
||||||
|
t.Fatalf("Diff() = (%+v, %v), want ErrInvalidConfig and no records", differences, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffIgnoresSourceHistory(t *testing.T) {
|
||||||
|
left := comparisonInspectionFrom(t, "earlier source", "same-model", nil)
|
||||||
|
right := comparisonInspectionFrom(t, "later source", "same-model", nil)
|
||||||
|
if !hasSourceLabel(left, "earlier source") || !hasSourceLabel(right, "later source") {
|
||||||
|
t.Fatal("comparison fixtures did not retain their distinct source labels")
|
||||||
|
}
|
||||||
|
|
||||||
|
differences, err := left.Diff(right)
|
||||||
|
if err != nil || len(differences) != 0 {
|
||||||
|
t.Fatalf("compare equal effective values with different sources = (%+v, %v), want no differences", differences, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
|
||||||
|
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||||
|
"sequence": []any{"earlier", "unchanged"},
|
||||||
|
})
|
||||||
|
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||||
|
"nullable": nil,
|
||||||
|
"sequence": []any{"later", "unchanged"},
|
||||||
|
})
|
||||||
|
|
||||||
|
differences, err := left.Diff(right)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compare presence and sequence values: %v", err)
|
||||||
|
}
|
||||||
|
if len(differences) != 2 {
|
||||||
|
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
|
||||||
|
}
|
||||||
|
byPath := differencesByPath(differences)
|
||||||
|
nullable, ok := byPath["/backends/local/extra_params/nullable"]
|
||||||
|
if !ok || nullable.Left.Present || !nullable.Right.Present || strings.TrimSpace(nullable.Right.YAML) != "null" {
|
||||||
|
t.Fatalf("absent-to-null difference = (%+v, %t)", nullable, ok)
|
||||||
|
}
|
||||||
|
sequence, ok := byPath["/backends/local/extra_params/sequence"]
|
||||||
|
if !ok || !sequence.Left.Present || !sequence.Right.Present ||
|
||||||
|
!strings.Contains(sequence.Left.YAML, "earlier") || !strings.Contains(sequence.Right.YAML, "later") {
|
||||||
|
t.Fatalf("atomic sequence difference = (%+v, %t)", sequence, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t *testing.T) {
|
||||||
|
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||||
|
"token": "left-secret-sentinel",
|
||||||
|
"visible": "before-value",
|
||||||
|
})
|
||||||
|
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||||
|
"token": "right-secret-sentinel",
|
||||||
|
"visible": "after-value",
|
||||||
|
})
|
||||||
|
|
||||||
|
differences, err := left.Diff(right)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compare sensitive values: %v", err)
|
||||||
|
}
|
||||||
|
if len(differences) != 2 {
|
||||||
|
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
|
||||||
|
}
|
||||||
|
byPath := differencesByPath(differences)
|
||||||
|
secret, ok := byPath["/backends/local/extra_params/token"]
|
||||||
|
if !ok || !secret.Left.Redacted || !secret.Right.Redacted || !secret.HiddenChanged {
|
||||||
|
t.Fatalf("sensitive difference = (%+v, %t)", secret, ok)
|
||||||
|
}
|
||||||
|
if strings.Contains(secret.Left.YAML, "left-secret-sentinel") || strings.Contains(secret.Left.YAML, "right-secret-sentinel") ||
|
||||||
|
strings.Contains(secret.Right.YAML, "left-secret-sentinel") || strings.Contains(secret.Right.YAML, "right-secret-sentinel") {
|
||||||
|
t.Fatalf("sensitive snapshots disclosed a value: %+v", secret)
|
||||||
|
}
|
||||||
|
visible, ok := byPath["/backends/local/extra_params/visible"]
|
||||||
|
if !ok || visible.Left.Redacted || visible.Right.Redacted || visible.HiddenChanged ||
|
||||||
|
!strings.Contains(visible.Left.YAML, "before-value") || !strings.Contains(visible.Right.YAML, "after-value") {
|
||||||
|
t.Fatalf("public sibling difference = (%+v, %t)", visible, ok)
|
||||||
|
}
|
||||||
|
|
||||||
|
differences[0].Path = configkit.Keys("caller", "mutation")
|
||||||
|
differences[0].Left.YAML = "caller mutation"
|
||||||
|
differences[0].Right.YAML = "caller mutation"
|
||||||
|
again, err := left.Diff(right)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("repeat comparison after mutating returned records: %v", err)
|
||||||
|
}
|
||||||
|
if len(again) != 2 {
|
||||||
|
t.Fatalf("repeat difference count = %d, want 2: %+v", len(again), again)
|
||||||
|
}
|
||||||
|
againByPath := differencesByPath(again)
|
||||||
|
if _, exists := againByPath["/caller/mutation"]; exists {
|
||||||
|
t.Fatal("mutating returned path reached retained inspection")
|
||||||
|
}
|
||||||
|
if _, exists := againByPath["/backends/local/extra_params/token"]; !exists {
|
||||||
|
t.Fatalf("original sensitive path was lost after mutation: %+v", again)
|
||||||
|
}
|
||||||
|
if _, exists := againByPath["/backends/local/extra_params/visible"]; !exists {
|
||||||
|
t.Fatalf("original public path was lost after mutation: %+v", again)
|
||||||
|
}
|
||||||
|
for _, difference := range again {
|
||||||
|
if difference.Left.YAML == "caller mutation" || difference.Right.YAML == "caller mutation" {
|
||||||
|
t.Fatalf("mutating returned snapshots reached retained inspection: %+v", difference)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffMasksSensitiveShapeReplacement(t *testing.T) {
|
||||||
|
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||||
|
"details": map[string]any{
|
||||||
|
"token": "nested-secret-sentinel",
|
||||||
|
"label": "public detail",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||||
|
"details": "replacement",
|
||||||
|
})
|
||||||
|
|
||||||
|
differences, err := left.Diff(right)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compare sensitive shape replacement: %v", err)
|
||||||
|
}
|
||||||
|
if len(differences) != 1 || differences[0].Path.String() != "/backends/local/extra_params/details" {
|
||||||
|
t.Fatalf("shape replacement differences = %+v", differences)
|
||||||
|
}
|
||||||
|
difference := differences[0]
|
||||||
|
if !difference.Left.Redacted || !difference.Right.Redacted || !difference.HiddenChanged {
|
||||||
|
t.Fatalf("shape replacement masking = %+v", difference)
|
||||||
|
}
|
||||||
|
if strings.Contains(difference.Left.YAML, "nested-secret-sentinel") || strings.Contains(difference.Right.YAML, "nested-secret-sentinel") {
|
||||||
|
t.Fatalf("shape replacement disclosed a sensitive value: %+v", difference)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonInspection(t *testing.T, model string) promptkit.ConfigurationInspection {
|
||||||
|
t.Helper()
|
||||||
|
return comparisonInspectionFrom(t, "comparison fixture", model, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonInspectionFrom(t *testing.T, source, model string, extraParams map[string]any) promptkit.ConfigurationInspection {
|
||||||
|
t.Helper()
|
||||||
|
backend := map[string]any{"endpoint": "https://example.invalid/v1"}
|
||||||
|
if extraParams != nil {
|
||||||
|
backend["extra_params"] = extraParams
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping(source, map[string]any{
|
||||||
|
"backends": map[string]any{"local": backend},
|
||||||
|
"profiles": map[string]any{"offline": map[string]any{
|
||||||
|
"backend": "local",
|
||||||
|
"model": model,
|
||||||
|
}},
|
||||||
|
})}}}},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct comparison engine: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect comparison engine: %v", err)
|
||||||
|
}
|
||||||
|
return inspection
|
||||||
|
}
|
||||||
|
|
||||||
|
func differencesByPath(differences []configkit.Difference) map[string]configkit.Difference {
|
||||||
|
byPath := make(map[string]configkit.Difference, len(differences))
|
||||||
|
for _, difference := range differences {
|
||||||
|
byPath[difference.Path.String()] = difference
|
||||||
|
}
|
||||||
|
return byPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasSourceLabel(view promptkit.ConfigurationInspection, label string) bool {
|
||||||
|
for _, source := range view.Sources() {
|
||||||
|
if source.Label == label {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -27,6 +27,23 @@ func (e *Engine) InspectConfiguration() (ConfigurationInspection, error) {
|
|||||||
return ConfigurationInspection{source: e.internal.InspectConfiguration()}, nil
|
return ConfigurationInspection{source: e.internal.InspectConfiguration()}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Diff compares this view's captured effective configuration with other. This
|
||||||
|
// view is the earlier (left) value and other is the later (right) value. The
|
||||||
|
// comparison delegates to ConfigKit's frozen effective-value semantics, so
|
||||||
|
// source or resolution history alone does not create differences. It does not
|
||||||
|
// reread external sources or compare referenced file contents, resolved
|
||||||
|
// credential contents, later execution preparation, or request-specific
|
||||||
|
// overrides. Either zero view returns ErrInvalidConfig and no records; other
|
||||||
|
// ConfigKit errors are returned unchanged. An empty diff is not evidence of
|
||||||
|
// equivalent engine behavior or execution. Records can contain ordinary
|
||||||
|
// application data, so callers choose where and to whom to disclose them.
|
||||||
|
func (v ConfigurationInspection) Diff(other ConfigurationInspection) ([]configkit.Difference, error) {
|
||||||
|
if v.source == nil || other.source == nil {
|
||||||
|
return nil, ErrInvalidConfig
|
||||||
|
}
|
||||||
|
return configuration.Diff(v.source, other.source)
|
||||||
|
}
|
||||||
|
|
||||||
// Metadata returns detached effective presence and history records.
|
// Metadata returns detached effective presence and history records.
|
||||||
func (v ConfigurationInspection) Metadata() configkit.Metadata {
|
func (v ConfigurationInspection) Metadata() configkit.Metadata {
|
||||||
if v.source == nil {
|
if v.source == nil {
|
||||||
|
|||||||
@@ -144,9 +144,17 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("select inputs: %v", err)
|
t.Fatalf("select inputs: %v", err)
|
||||||
}
|
}
|
||||||
|
otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
||||||
|
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||||
|
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
||||||
|
}}
|
||||||
|
otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("select comparison inputs: %v", err)
|
||||||
|
}
|
||||||
var modelCalls atomic.Int64
|
var modelCalls atomic.Int64
|
||||||
client := countingLLMClient{calls: &modelCalls}
|
client := countingLLMClient{calls: &modelCalls}
|
||||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
|
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
|
||||||
"offline": {Endpoint: &endpoint, Model: &model},
|
"offline": {Endpoint: &endpoint, Model: &model},
|
||||||
@@ -155,11 +163,42 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine: %v", err)
|
t.Fatalf("construct engine: %v", err)
|
||||||
}
|
}
|
||||||
|
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("comparison profile", map[string]promptkit.ProfileConfig{
|
||||||
|
"offline": {Endpoint: &endpoint, Model: &otherModel},
|
||||||
|
})}}}},
|
||||||
|
}}, promptkit.WithLLMClient(client))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct comparison engine: %v", err)
|
||||||
|
}
|
||||||
source.closed.Store(true)
|
source.closed.Store(true)
|
||||||
|
otherSource.closed.Store(true)
|
||||||
view, err := engine.InspectConfiguration()
|
view, err := engine.InspectConfiguration()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect configuration: %v", err)
|
t.Fatalf("inspect configuration: %v", err)
|
||||||
}
|
}
|
||||||
|
otherView, err := otherEngine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect comparison configuration: %v", err)
|
||||||
|
}
|
||||||
|
var expected configkit.Difference
|
||||||
|
for index := 0; index < 3; index++ {
|
||||||
|
differences, err := view.Diff(otherView)
|
||||||
|
if err != nil || len(differences) != 1 {
|
||||||
|
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
|
||||||
|
}
|
||||||
|
if differences[0].Path.String() != "/profiles/offline/model" ||
|
||||||
|
!strings.Contains(differences[0].Left.YAML, model) ||
|
||||||
|
!strings.Contains(differences[0].Right.YAML, otherModel) {
|
||||||
|
t.Fatalf("captured difference = %+v", differences[0])
|
||||||
|
}
|
||||||
|
if index == 0 {
|
||||||
|
expected = differences[0]
|
||||||
|
} else if differences[0].Path.String() != expected.Path.String() ||
|
||||||
|
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML {
|
||||||
|
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
if _, found := view.Explain(configkit.Keys("prompts", "captured", "1", "messages").Index(0).Key("content_file")); !found {
|
if _, found := view.Explain(configkit.Keys("prompts", "captured", "1", "messages").Index(0).Key("content_file")); !found {
|
||||||
t.Fatal("captured prompt content path was not inspectable")
|
t.Fatal("captured prompt content path was not inspectable")
|
||||||
}
|
}
|
||||||
@@ -167,24 +206,32 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") {
|
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") {
|
||||||
t.Fatalf("write captured report = (%q, %v)", report.String(), err)
|
t.Fatalf("write captured report = (%q, %v)", report.String(), err)
|
||||||
}
|
}
|
||||||
if source.reads.Load() != 0 || modelCalls.Load() != 0 {
|
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != 0 {
|
||||||
t.Fatalf("inspection touched source or provider: reads=%d requests=%d", source.reads.Load(), modelCalls.Load())
|
t.Fatalf("inspection or comparison touched source or provider: reads=(%d,%d) requests=%d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
|
||||||
}
|
}
|
||||||
|
|
||||||
const calls = 16
|
const calls = 24
|
||||||
var wait sync.WaitGroup
|
var wait sync.WaitGroup
|
||||||
errs := make(chan error, calls)
|
errs := make(chan error, calls)
|
||||||
for index := 0; index < calls; index++ {
|
for index := 0; index < calls; index++ {
|
||||||
wait.Add(1)
|
wait.Add(1)
|
||||||
go func(index int) {
|
go func(index int) {
|
||||||
defer wait.Done()
|
defer wait.Done()
|
||||||
if index%2 == 0 {
|
switch index % 3 {
|
||||||
|
case 0:
|
||||||
_, err := engine.InspectConfiguration()
|
_, err := engine.InspectConfiguration()
|
||||||
errs <- err
|
errs <- err
|
||||||
return
|
case 1:
|
||||||
|
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
|
||||||
|
errs <- err
|
||||||
|
case 2:
|
||||||
|
differences, err := view.Diff(otherView)
|
||||||
|
if err == nil && (len(differences) != 1 || differences[0].Path.String() != expected.Path.String() ||
|
||||||
|
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML) {
|
||||||
|
err = errors.New("concurrent comparison returned an unexpected difference")
|
||||||
|
}
|
||||||
|
errs <- err
|
||||||
}
|
}
|
||||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
|
|
||||||
errs <- err
|
|
||||||
}(index)
|
}(index)
|
||||||
}
|
}
|
||||||
wait.Wait()
|
wait.Wait()
|
||||||
@@ -194,7 +241,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if source.reads.Load() != 0 || modelCalls.Load() != calls/2 {
|
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != calls/3 {
|
||||||
t.Fatalf("concurrent inspection/execution = source reads %d, model calls %d", source.reads.Load(), modelCalls.Load())
|
t.Fatalf("concurrent inspection/comparison/execution = source reads (%d,%d), model calls %d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -181,6 +181,41 @@ shows backend-completed execution settings for one profile, and `Prepare` or
|
|||||||
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
|
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
|
||||||
and validation resources. None of those views is a substitute for the others.
|
and validation resources. None of those views is a substitute for the others.
|
||||||
|
|
||||||
|
To review changes between two independently captured configurations, compare
|
||||||
|
their inspection views:
|
||||||
|
|
||||||
|
```go
|
||||||
|
earlier, err := earlierEngine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
later, err := candidateEngine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
differences, err := earlier.Diff(later)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, difference := range differences {
|
||||||
|
if difference.HiddenChanged {
|
||||||
|
// Report a protected change without inferring its value.
|
||||||
|
}
|
||||||
|
if difference.Left.Redacted || difference.Right.Redacted {
|
||||||
|
// Apply the application's disclosure policy before presenting snapshots.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
|
||||||
|
and ConfigKit's [v0.7.0 effective-configuration diff guidance](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.7.0/docs/config.md#effective-configuration-diffs)
|
||||||
|
for the exact comparison contract. The operation compares only frozen effective
|
||||||
|
configuration values: it does not reread external files or compare their
|
||||||
|
contents, resolved credential contents, later execution preparation, or
|
||||||
|
request-specific overrides. An empty diff is not proof of equivalent execution.
|
||||||
|
Records can contain ordinary application data, so use the application's
|
||||||
|
disclosure policy when presenting them.
|
||||||
|
|
||||||
## Inspect A Prompt Before Preparation
|
## Inspect A Prompt Before Preparation
|
||||||
|
|
||||||
Use [`Engine.InspectPrompt`](../../engine.go) to check one configured prompt's
|
Use [`Engine.InspectPrompt`](../../engine.go) to check one configured prompt's
|
||||||
|
|||||||
@@ -21,6 +21,16 @@ profile inheritance. It retains ConfigKit's result privately for explicit
|
|||||||
inspection, not for per-run resolution. The runtime/domain packages do not
|
inspection, not for per-run resolution. The runtime/domain packages do not
|
||||||
depend on ConfigKit.
|
depend on ConfigKit.
|
||||||
|
|
||||||
|
The `ConfigurationInspection.Diff` facade method compares two retained results
|
||||||
|
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
|
||||||
|
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
|
||||||
|
once to ConfigKit v0.7.0 and returns its public difference records. Comparison
|
||||||
|
uses only frozen effective values and does not reopen sources, resolve runtime
|
||||||
|
credentials, or enter the execution path. ConfigKit applies sensitivity from
|
||||||
|
both results; reports may still contain ordinary application data. See the
|
||||||
|
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||||
|
and ConfigKit's [v0.7.0 comparison contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.7.0/docs/config.md#effective-configuration-diffs).
|
||||||
|
|
||||||
After prompt materialization and effective-profile snapshot construction, the
|
After prompt materialization and effective-profile snapshot construction, the
|
||||||
adapter captures lexically ordered, identity-only inventories from the final
|
adapter captures lexically ordered, identity-only inventories from the final
|
||||||
prompt and indexed-profile maps. Its listing methods return caller-owned copies
|
prompt and indexed-profile maps. Its listing methods return caller-owned copies
|
||||||
@@ -49,9 +59,9 @@ strings, including empty strings. It closes content-file handles before
|
|||||||
returning and retains no source capability for public inspection.
|
returning and retains no source capability for public inspection.
|
||||||
|
|
||||||
Role and cache-control normalization is applied to execution-owned message
|
Role and cache-control normalization is applied to execution-owned message
|
||||||
copies. The retained ConfigKit configuration keeps effective input message
|
copies. ConfigKit v0.7.0 retains effective input message metadata and its
|
||||||
metadata and its resource origins: recording a sequence-wide normalization in
|
resource origins; recording a sequence-wide normalization in the retained
|
||||||
ConfigKit v0.6.0 would replace origins of unchanged content-file fields.
|
configuration would replace origins of unchanged content-file fields.
|
||||||
|
|
||||||
The runner selects full definitions from this index for preparation and uses
|
The runner selects full definitions from this index for preparation and uses
|
||||||
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
|
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
|
||||||
|
|||||||
@@ -26,17 +26,17 @@ License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
|||||||
|
|
||||||
## ConfigKit
|
## ConfigKit
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.6.0`.
|
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.7.1`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
SPDX-License-Identifier: LGPL-3.0-or-later
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
License and notices: [LGPL 3.0](../../LICENSE) |
|
License and notices: [LGPL 3.0](../../LICENSE) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/1a885934c28733d9a5dca261fd6da0e2a4279285/COPYRIGHT) |
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/1a885934c28733d9a5dca261fd6da0e2a4279285/LICENSE).
|
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/1a885934c28733d9a5dca261fd6da0e2a4279285).
|
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77).
|
||||||
|
|
||||||
## Promptkit OpenRouter Catalog v3
|
## Promptkit OpenRouter Catalog v3
|
||||||
|
|
||||||
|
|||||||
@@ -74,7 +74,7 @@ For an optional local diagnostic of a committed candidate, run:
|
|||||||
```
|
```
|
||||||
|
|
||||||
This is not a required step before publication. The upstream
|
This is not a required step before publication. The upstream
|
||||||
[candidate-check contract](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.3.0/docs/consumers/check.md)
|
[candidate-check contract](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/consumers/check.md)
|
||||||
describes its requirements.
|
describes its requirements.
|
||||||
|
|
||||||
Run the single publication command with the chosen version:
|
Run the single publication command with the chosen version:
|
||||||
@@ -114,7 +114,7 @@ resolved. ReleaseKit reports the attempted version, tag object, and recovery
|
|||||||
state, including the captured publication endpoint when available. Use that
|
state, including the captured publication endpoint when available. Use that
|
||||||
endpoint—not merely the current `origin` URL—to query the exact tag; follow the
|
endpoint—not merely the current `origin` URL—to query the exact tag; follow the
|
||||||
upstream
|
upstream
|
||||||
[failure-recovery procedure](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.3.0/docs/release.md#resolve-failures):
|
[failure-recovery procedure](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/release.md#resolve-failures):
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
releasekit_endpoint='COPY_CAPTURED_ENDPOINT'
|
releasekit_endpoint='COPY_CAPTURED_ENDPOINT'
|
||||||
|
|||||||
48
docs/releases/v0.14.0.md
Normal file
48
docs/releases/v0.14.0.md
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
# Promptkit v0.14.0
|
||||||
|
|
||||||
|
This release adds comparison of captured configuration inspections and updates
|
||||||
|
ConfigKit and the maintainer release tool. The minimum supported Go version
|
||||||
|
remains 1.26.0.
|
||||||
|
|
||||||
|
## Compare Captured Configurations
|
||||||
|
|
||||||
|
`ConfigurationInspection.Diff` compares the effective configuration captured by
|
||||||
|
two engines. Consumers can use it to review a candidate configuration, inspect
|
||||||
|
an overlay's effect, or present configuration changes to operators.
|
||||||
|
|
||||||
|
Comparison delegates to ConfigKit and returns its difference records, including
|
||||||
|
presence information, paired redaction, and protected-change indicators. It
|
||||||
|
does not reload sources or contact providers. Source-history differences alone
|
||||||
|
do not produce value differences.
|
||||||
|
|
||||||
|
An empty diff is not proof of equivalent execution: referenced file contents,
|
||||||
|
resolved credentials, later execution preparation, and request-specific
|
||||||
|
overrides are outside the comparison. Records can contain ordinary application
|
||||||
|
data; callers retain responsibility for their disclosure.
|
||||||
|
|
||||||
|
See the [configuration comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||||
|
and [GoDoc](../../configuration_inspection.go) for usage and the exact contract.
|
||||||
|
|
||||||
|
## Dependency And Tooling Updates
|
||||||
|
|
||||||
|
- Upgrade ConfigKit from v0.6.0 to v0.7.1, providing the upstream comparison API.
|
||||||
|
v0.7.1 also updates ConfigKit's maintainer tooling without changing its runtime
|
||||||
|
behavior from v0.7.0.
|
||||||
|
- Upgrade the module-selected ReleaseKit tool from v0.3.0 to v0.4.1. Promptkit
|
||||||
|
retains its existing root-module profile, publication entry point, and release
|
||||||
|
gates. See the [release procedure](../release.md) for maintainer commands.
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
No application-code, configuration, or Go-toolchain migration is required from
|
||||||
|
v0.13.0. Configuration comparison is an optional API addition; existing
|
||||||
|
construction, inspection, preparation, and execution workflows remain available.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.14.0
|
||||||
|
```
|
||||||
|
|
||||||
|
See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility) for the
|
||||||
|
distinction between the consumer minimum and maintainer toolchain.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/docs/oss/attribution.md)
|
||||||
4
go.mod
4
go.mod
@@ -3,14 +3,14 @@ module gitea.maximumdirect.net/eric/promptkit
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.6.0
|
gitea.maximumdirect.net/eric/ConfigKit v0.7.1
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/ReleaseKit v0.3.0 // indirect
|
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 // indirect
|
||||||
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||||
golang.org/x/mod v0.40.0 // indirect
|
golang.org/x/mod v0.40.0 // indirect
|
||||||
|
|||||||
8
go.sum
8
go.sum
@@ -1,7 +1,7 @@
|
|||||||
gitea.maximumdirect.net/eric/ConfigKit v0.6.0 h1:KdsCVO5kKctdAUQJ6X7dJ/GemlVjCYCcn/xZrMAE/aw=
|
gitea.maximumdirect.net/eric/ConfigKit v0.7.1 h1:q3HrX4WWCPt1WfMydqFEaAmCf9s6Dq+vknJj0C/mdcI=
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.6.0/go.mod h1:xN4dRdDGIW4yh/e9GghDF/hSIzc8/77blRhwyf90KSA=
|
gitea.maximumdirect.net/eric/ConfigKit v0.7.1/go.mod h1:C82sbyKHYMV1OwaD2GJanFrwTfcZfjlW2JsGze5b0kA=
|
||||||
gitea.maximumdirect.net/eric/ReleaseKit v0.3.0 h1:aTMn+50MEpHpHI2RoCCmJYsy7Ohrpa+RSva63Kq0/tU=
|
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 h1:Qn+mdebYOzCVCEXgCbuGSO/WHmJtItwaQ8e3XwY3qtI=
|
||||||
gitea.maximumdirect.net/eric/ReleaseKit v0.3.0/go.mod h1:LVodqNQ3OIKv9NBq+fFeyMByd1lHY25YOmmpah/NaNI=
|
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1/go.mod h1:LVodqNQ3OIKv9NBq+fFeyMByd1lHY25YOmmpah/NaNI=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
||||||
|
|||||||
@@ -16,6 +16,15 @@ type Inspection interface {
|
|||||||
Explain(configkit.Path) (configkit.Explanation, bool)
|
Explain(configkit.Path) (configkit.Explanation, bool)
|
||||||
WriteEffective(io.Writer) error
|
WriteEffective(io.Writer) error
|
||||||
WriteExplanation(io.Writer, configkit.Path) error
|
WriteExplanation(io.Writer, configkit.Path) error
|
||||||
|
capturedResult() *configkit.Result[document]
|
||||||
|
}
|
||||||
|
|
||||||
|
type inspectionResult struct {
|
||||||
|
*configkit.Result[document]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (inspection inspectionResult) capturedResult() *configkit.Result[document] {
|
||||||
|
return inspection.Result
|
||||||
}
|
}
|
||||||
|
|
||||||
// Inspection returns only the retained result's inspection operations.
|
// Inspection returns only the retained result's inspection operations.
|
||||||
@@ -23,7 +32,19 @@ func (state *loaded) Inspection() Inspection {
|
|||||||
if state == nil {
|
if state == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return state.result
|
return inspectionResult{Result: state.result}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Diff compares two captured ConfigKit results without reconstructing them.
|
||||||
|
func Diff(left, right Inspection) ([]configkit.Difference, error) {
|
||||||
|
var leftResult, rightResult *configkit.Result[document]
|
||||||
|
if left != nil {
|
||||||
|
leftResult = left.capturedResult()
|
||||||
|
}
|
||||||
|
if right != nil {
|
||||||
|
rightResult = right.capturedResult()
|
||||||
|
}
|
||||||
|
return configkit.Diff(leftResult, rightResult)
|
||||||
}
|
}
|
||||||
|
|
||||||
func secretProviderPath(path configkit.Path) bool {
|
func secretProviderPath(path configkit.Path) bool {
|
||||||
|
|||||||
Reference in New Issue
Block a user