Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 635674c066 | |||
| 50bc35516a | |||
| 18309f50db | |||
| 60784e7875 | |||
| f3148bfe52 | |||
| f315620bf7 | |||
| 97357d62b9 | |||
| 0c3651fceb | |||
| 5a5d47d36e | |||
| 143c6e9b3a | |||
| 48e4784fcd | |||
| 78af5069c1 | |||
| 18963266d7 | |||
| d039a019cb | |||
| d1c0281053 | |||
| 870e4ca4e6 | |||
| fd82524977 | |||
| 3f76c75919 | |||
| fe764f72d8 | |||
| 26ab566f38 |
@@ -37,6 +37,9 @@ boundary and constraints that framework work must preserve.
|
|||||||
|
|
||||||
## Release Guidance
|
## Release Guidance
|
||||||
|
|
||||||
|
The [v0.15.0 release notes](docs/releases/v0.15.0.md) cover error-only
|
||||||
|
validation of concrete requests with `Engine.ValidateRequest`.
|
||||||
|
|
||||||
Consumers upgrading to `v0.12.0` should read the
|
Consumers upgrading to `v0.12.0` should read the
|
||||||
[v0.12.0 release notes](docs/releases/v0.12.0.md) for the intentional ConfigKit
|
[v0.12.0 release notes](docs/releases/v0.12.0.md) for the intentional ConfigKit
|
||||||
configuration and public construction API migration.
|
configuration and public construction API migration.
|
||||||
|
|||||||
243
configuration_diff_contract_test.go
Normal file
243
configuration_diff_contract_test.go
Normal file
@@ -0,0 +1,243 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/ConfigKit"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) {
|
||||||
|
left := comparisonInspection(t, "earlier-model")
|
||||||
|
right := comparisonInspection(t, "later-model")
|
||||||
|
|
||||||
|
differences, err := left.Diff(right)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compare captured configurations: %v", err)
|
||||||
|
}
|
||||||
|
if len(differences) != 1 {
|
||||||
|
t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences)
|
||||||
|
}
|
||||||
|
difference := differences[0]
|
||||||
|
if got, want := difference.Path.String(), "/profiles/offline/model"; got != want {
|
||||||
|
t.Fatalf("difference path = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") {
|
||||||
|
t.Fatalf("difference snapshots = (%q, %q)", difference.Left.YAML, difference.Right.YAML)
|
||||||
|
}
|
||||||
|
|
||||||
|
reversed, err := right.Diff(left)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compare reversed configurations: %v", err)
|
||||||
|
}
|
||||||
|
if len(reversed) != 1 || reversed[0].Path.String() != difference.Path.String() ||
|
||||||
|
!strings.Contains(reversed[0].Left.YAML, "later-model") ||
|
||||||
|
!strings.Contains(reversed[0].Right.YAML, "earlier-model") {
|
||||||
|
t.Fatalf("reversed difference = %+v", reversed)
|
||||||
|
}
|
||||||
|
|
||||||
|
equal := comparisonInspection(t, "earlier-model")
|
||||||
|
if equal, err := left.Diff(equal); err != nil || len(equal) != 0 {
|
||||||
|
t.Fatalf("compare equal configurations = (%+v, %v), want no differences", equal, err)
|
||||||
|
}
|
||||||
|
if equal, err := left.Diff(left); err != nil || len(equal) != 0 {
|
||||||
|
t.Fatalf("compare view with itself = (%+v, %v), want no differences", equal, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffRejectsZeroViews(t *testing.T) {
|
||||||
|
valid := comparisonInspection(t, "current-model")
|
||||||
|
var zero promptkit.ConfigurationInspection
|
||||||
|
|
||||||
|
for _, comparison := range []struct {
|
||||||
|
name string
|
||||||
|
left, right promptkit.ConfigurationInspection
|
||||||
|
}{
|
||||||
|
{name: "zero left", left: zero, right: valid},
|
||||||
|
{name: "zero right", left: valid, right: zero},
|
||||||
|
{name: "both zero", left: zero, right: zero},
|
||||||
|
} {
|
||||||
|
t.Run(comparison.name, func(t *testing.T) {
|
||||||
|
differences, err := comparison.left.Diff(comparison.right)
|
||||||
|
if !errors.Is(err, promptkit.ErrInvalidConfig) || len(differences) != 0 {
|
||||||
|
t.Fatalf("Diff() = (%+v, %v), want ErrInvalidConfig and no records", differences, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffIgnoresSourceHistory(t *testing.T) {
|
||||||
|
left := comparisonInspectionFrom(t, "earlier source", "same-model", nil)
|
||||||
|
right := comparisonInspectionFrom(t, "later source", "same-model", nil)
|
||||||
|
if !hasSourceLabel(left, "earlier source") || !hasSourceLabel(right, "later source") {
|
||||||
|
t.Fatal("comparison fixtures did not retain their distinct source labels")
|
||||||
|
}
|
||||||
|
|
||||||
|
differences, err := left.Diff(right)
|
||||||
|
if err != nil || len(differences) != 0 {
|
||||||
|
t.Fatalf("compare equal effective values with different sources = (%+v, %v), want no differences", differences, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
|
||||||
|
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||||
|
"sequence": []any{"earlier", "unchanged"},
|
||||||
|
})
|
||||||
|
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||||
|
"nullable": nil,
|
||||||
|
"sequence": []any{"later", "unchanged"},
|
||||||
|
})
|
||||||
|
|
||||||
|
differences, err := left.Diff(right)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compare presence and sequence values: %v", err)
|
||||||
|
}
|
||||||
|
if len(differences) != 2 {
|
||||||
|
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
|
||||||
|
}
|
||||||
|
byPath := differencesByPath(differences)
|
||||||
|
nullable, ok := byPath["/backends/local/extra_params/nullable"]
|
||||||
|
if !ok || nullable.Left.Present || !nullable.Right.Present || strings.TrimSpace(nullable.Right.YAML) != "null" {
|
||||||
|
t.Fatalf("absent-to-null difference = (%+v, %t)", nullable, ok)
|
||||||
|
}
|
||||||
|
sequence, ok := byPath["/backends/local/extra_params/sequence"]
|
||||||
|
if !ok || !sequence.Left.Present || !sequence.Right.Present ||
|
||||||
|
!strings.Contains(sequence.Left.YAML, "earlier") || !strings.Contains(sequence.Right.YAML, "later") {
|
||||||
|
t.Fatalf("atomic sequence difference = (%+v, %t)", sequence, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t *testing.T) {
|
||||||
|
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||||
|
"token": "left-secret-sentinel",
|
||||||
|
"visible": "before-value",
|
||||||
|
})
|
||||||
|
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||||
|
"token": "right-secret-sentinel",
|
||||||
|
"visible": "after-value",
|
||||||
|
})
|
||||||
|
|
||||||
|
differences, err := left.Diff(right)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compare sensitive values: %v", err)
|
||||||
|
}
|
||||||
|
if len(differences) != 2 {
|
||||||
|
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
|
||||||
|
}
|
||||||
|
byPath := differencesByPath(differences)
|
||||||
|
secret, ok := byPath["/backends/local/extra_params/token"]
|
||||||
|
if !ok || !secret.Left.Redacted || !secret.Right.Redacted || !secret.HiddenChanged {
|
||||||
|
t.Fatalf("sensitive difference = (%+v, %t)", secret, ok)
|
||||||
|
}
|
||||||
|
if strings.Contains(secret.Left.YAML, "left-secret-sentinel") || strings.Contains(secret.Left.YAML, "right-secret-sentinel") ||
|
||||||
|
strings.Contains(secret.Right.YAML, "left-secret-sentinel") || strings.Contains(secret.Right.YAML, "right-secret-sentinel") {
|
||||||
|
t.Fatalf("sensitive snapshots disclosed a value: %+v", secret)
|
||||||
|
}
|
||||||
|
visible, ok := byPath["/backends/local/extra_params/visible"]
|
||||||
|
if !ok || visible.Left.Redacted || visible.Right.Redacted || visible.HiddenChanged ||
|
||||||
|
!strings.Contains(visible.Left.YAML, "before-value") || !strings.Contains(visible.Right.YAML, "after-value") {
|
||||||
|
t.Fatalf("public sibling difference = (%+v, %t)", visible, ok)
|
||||||
|
}
|
||||||
|
|
||||||
|
differences[0].Path = configkit.Keys("caller", "mutation")
|
||||||
|
differences[0].Left.YAML = "caller mutation"
|
||||||
|
differences[0].Right.YAML = "caller mutation"
|
||||||
|
again, err := left.Diff(right)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("repeat comparison after mutating returned records: %v", err)
|
||||||
|
}
|
||||||
|
if len(again) != 2 {
|
||||||
|
t.Fatalf("repeat difference count = %d, want 2: %+v", len(again), again)
|
||||||
|
}
|
||||||
|
againByPath := differencesByPath(again)
|
||||||
|
if _, exists := againByPath["/caller/mutation"]; exists {
|
||||||
|
t.Fatal("mutating returned path reached retained inspection")
|
||||||
|
}
|
||||||
|
if _, exists := againByPath["/backends/local/extra_params/token"]; !exists {
|
||||||
|
t.Fatalf("original sensitive path was lost after mutation: %+v", again)
|
||||||
|
}
|
||||||
|
if _, exists := againByPath["/backends/local/extra_params/visible"]; !exists {
|
||||||
|
t.Fatalf("original public path was lost after mutation: %+v", again)
|
||||||
|
}
|
||||||
|
for _, difference := range again {
|
||||||
|
if difference.Left.YAML == "caller mutation" || difference.Right.YAML == "caller mutation" {
|
||||||
|
t.Fatalf("mutating returned snapshots reached retained inspection: %+v", difference)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigurationInspectionDiffMasksSensitiveShapeReplacement(t *testing.T) {
|
||||||
|
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
|
||||||
|
"details": map[string]any{
|
||||||
|
"token": "nested-secret-sentinel",
|
||||||
|
"label": "public detail",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
|
||||||
|
"details": "replacement",
|
||||||
|
})
|
||||||
|
|
||||||
|
differences, err := left.Diff(right)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("compare sensitive shape replacement: %v", err)
|
||||||
|
}
|
||||||
|
if len(differences) != 1 || differences[0].Path.String() != "/backends/local/extra_params/details" {
|
||||||
|
t.Fatalf("shape replacement differences = %+v", differences)
|
||||||
|
}
|
||||||
|
difference := differences[0]
|
||||||
|
if !difference.Left.Redacted || !difference.Right.Redacted || !difference.HiddenChanged {
|
||||||
|
t.Fatalf("shape replacement masking = %+v", difference)
|
||||||
|
}
|
||||||
|
if strings.Contains(difference.Left.YAML, "nested-secret-sentinel") || strings.Contains(difference.Right.YAML, "nested-secret-sentinel") {
|
||||||
|
t.Fatalf("shape replacement disclosed a sensitive value: %+v", difference)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonInspection(t *testing.T, model string) promptkit.ConfigurationInspection {
|
||||||
|
t.Helper()
|
||||||
|
return comparisonInspectionFrom(t, "comparison fixture", model, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func comparisonInspectionFrom(t *testing.T, source, model string, extraParams map[string]any) promptkit.ConfigurationInspection {
|
||||||
|
t.Helper()
|
||||||
|
backend := map[string]any{"endpoint": "https://example.invalid/v1"}
|
||||||
|
if extraParams != nil {
|
||||||
|
backend["extra_params"] = extraParams
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping(source, map[string]any{
|
||||||
|
"backends": map[string]any{"local": backend},
|
||||||
|
"profiles": map[string]any{"offline": map[string]any{
|
||||||
|
"backend": "local",
|
||||||
|
"model": model,
|
||||||
|
}},
|
||||||
|
})}}}},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct comparison engine: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect comparison engine: %v", err)
|
||||||
|
}
|
||||||
|
return inspection
|
||||||
|
}
|
||||||
|
|
||||||
|
func differencesByPath(differences []configkit.Difference) map[string]configkit.Difference {
|
||||||
|
byPath := make(map[string]configkit.Difference, len(differences))
|
||||||
|
for _, difference := range differences {
|
||||||
|
byPath[difference.Path.String()] = difference
|
||||||
|
}
|
||||||
|
return byPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasSourceLabel(view promptkit.ConfigurationInspection, label string) bool {
|
||||||
|
for _, source := range view.Sources() {
|
||||||
|
if source.Label == label {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -27,6 +27,23 @@ func (e *Engine) InspectConfiguration() (ConfigurationInspection, error) {
|
|||||||
return ConfigurationInspection{source: e.internal.InspectConfiguration()}, nil
|
return ConfigurationInspection{source: e.internal.InspectConfiguration()}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Diff compares this view's captured effective configuration with other. This
|
||||||
|
// view is the earlier (left) value and other is the later (right) value. The
|
||||||
|
// comparison delegates to ConfigKit's frozen effective-value semantics, so
|
||||||
|
// source or resolution history alone does not create differences. It does not
|
||||||
|
// reread external sources or compare referenced file contents, resolved
|
||||||
|
// credential contents, later execution preparation, or request-specific
|
||||||
|
// overrides. Either zero view returns ErrInvalidConfig and no records; other
|
||||||
|
// ConfigKit errors are returned unchanged. An empty diff is not evidence of
|
||||||
|
// equivalent engine behavior or execution. Records can contain ordinary
|
||||||
|
// application data, so callers choose where and to whom to disclose them.
|
||||||
|
func (v ConfigurationInspection) Diff(other ConfigurationInspection) ([]configkit.Difference, error) {
|
||||||
|
if v.source == nil || other.source == nil {
|
||||||
|
return nil, ErrInvalidConfig
|
||||||
|
}
|
||||||
|
return configuration.Diff(v.source, other.source)
|
||||||
|
}
|
||||||
|
|
||||||
// Metadata returns detached effective presence and history records.
|
// Metadata returns detached effective presence and history records.
|
||||||
func (v ConfigurationInspection) Metadata() configkit.Metadata {
|
func (v ConfigurationInspection) Metadata() configkit.Metadata {
|
||||||
if v.source == nil {
|
if v.source == nil {
|
||||||
|
|||||||
@@ -144,9 +144,17 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("select inputs: %v", err)
|
t.Fatalf("select inputs: %v", err)
|
||||||
}
|
}
|
||||||
|
otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{
|
||||||
|
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
|
||||||
|
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
|
||||||
|
}}
|
||||||
|
otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("select comparison inputs: %v", err)
|
||||||
|
}
|
||||||
var modelCalls atomic.Int64
|
var modelCalls atomic.Int64
|
||||||
client := countingLLMClient{calls: &modelCalls}
|
client := countingLLMClient{calls: &modelCalls}
|
||||||
endpoint, model := "https://example.invalid/v1", "offline-model"
|
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
|
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
|
||||||
"offline": {Endpoint: &endpoint, Model: &model},
|
"offline": {Endpoint: &endpoint, Model: &model},
|
||||||
@@ -155,11 +163,42 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine: %v", err)
|
t.Fatalf("construct engine: %v", err)
|
||||||
}
|
}
|
||||||
|
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
|
||||||
|
Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("comparison profile", map[string]promptkit.ProfileConfig{
|
||||||
|
"offline": {Endpoint: &endpoint, Model: &otherModel},
|
||||||
|
})}}}},
|
||||||
|
}}, promptkit.WithLLMClient(client))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct comparison engine: %v", err)
|
||||||
|
}
|
||||||
source.closed.Store(true)
|
source.closed.Store(true)
|
||||||
|
otherSource.closed.Store(true)
|
||||||
view, err := engine.InspectConfiguration()
|
view, err := engine.InspectConfiguration()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("inspect configuration: %v", err)
|
t.Fatalf("inspect configuration: %v", err)
|
||||||
}
|
}
|
||||||
|
otherView, err := otherEngine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect comparison configuration: %v", err)
|
||||||
|
}
|
||||||
|
var expected configkit.Difference
|
||||||
|
for index := 0; index < 3; index++ {
|
||||||
|
differences, err := view.Diff(otherView)
|
||||||
|
if err != nil || len(differences) != 1 {
|
||||||
|
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
|
||||||
|
}
|
||||||
|
if differences[0].Path.String() != "/profiles/offline/model" ||
|
||||||
|
!strings.Contains(differences[0].Left.YAML, model) ||
|
||||||
|
!strings.Contains(differences[0].Right.YAML, otherModel) {
|
||||||
|
t.Fatalf("captured difference = %+v", differences[0])
|
||||||
|
}
|
||||||
|
if index == 0 {
|
||||||
|
expected = differences[0]
|
||||||
|
} else if differences[0].Path.String() != expected.Path.String() ||
|
||||||
|
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML {
|
||||||
|
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
if _, found := view.Explain(configkit.Keys("prompts", "captured", "1", "messages").Index(0).Key("content_file")); !found {
|
if _, found := view.Explain(configkit.Keys("prompts", "captured", "1", "messages").Index(0).Key("content_file")); !found {
|
||||||
t.Fatal("captured prompt content path was not inspectable")
|
t.Fatal("captured prompt content path was not inspectable")
|
||||||
}
|
}
|
||||||
@@ -167,24 +206,32 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") {
|
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") {
|
||||||
t.Fatalf("write captured report = (%q, %v)", report.String(), err)
|
t.Fatalf("write captured report = (%q, %v)", report.String(), err)
|
||||||
}
|
}
|
||||||
if source.reads.Load() != 0 || modelCalls.Load() != 0 {
|
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != 0 {
|
||||||
t.Fatalf("inspection touched source or provider: reads=%d requests=%d", source.reads.Load(), modelCalls.Load())
|
t.Fatalf("inspection or comparison touched source or provider: reads=(%d,%d) requests=%d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
|
||||||
}
|
}
|
||||||
|
|
||||||
const calls = 16
|
const calls = 24
|
||||||
var wait sync.WaitGroup
|
var wait sync.WaitGroup
|
||||||
errs := make(chan error, calls)
|
errs := make(chan error, calls)
|
||||||
for index := 0; index < calls; index++ {
|
for index := 0; index < calls; index++ {
|
||||||
wait.Add(1)
|
wait.Add(1)
|
||||||
go func(index int) {
|
go func(index int) {
|
||||||
defer wait.Done()
|
defer wait.Done()
|
||||||
if index%2 == 0 {
|
switch index % 3 {
|
||||||
|
case 0:
|
||||||
_, err := engine.InspectConfiguration()
|
_, err := engine.InspectConfiguration()
|
||||||
errs <- err
|
errs <- err
|
||||||
return
|
case 1:
|
||||||
|
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
|
||||||
|
errs <- err
|
||||||
|
case 2:
|
||||||
|
differences, err := view.Diff(otherView)
|
||||||
|
if err == nil && (len(differences) != 1 || differences[0].Path.String() != expected.Path.String() ||
|
||||||
|
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML) {
|
||||||
|
err = errors.New("concurrent comparison returned an unexpected difference")
|
||||||
|
}
|
||||||
|
errs <- err
|
||||||
}
|
}
|
||||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
|
|
||||||
errs <- err
|
|
||||||
}(index)
|
}(index)
|
||||||
}
|
}
|
||||||
wait.Wait()
|
wait.Wait()
|
||||||
@@ -194,7 +241,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if source.reads.Load() != 0 || modelCalls.Load() != calls/2 {
|
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != calls/3 {
|
||||||
t.Fatalf("concurrent inspection/execution = source reads %d, model calls %d", source.reads.Load(), modelCalls.Load())
|
t.Fatalf("concurrent inspection/comparison/execution = source reads (%d,%d), model calls %d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -181,6 +181,41 @@ shows backend-completed execution settings for one profile, and `Prepare` or
|
|||||||
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
|
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
|
||||||
and validation resources. None of those views is a substitute for the others.
|
and validation resources. None of those views is a substitute for the others.
|
||||||
|
|
||||||
|
To review changes between two independently captured configurations, compare
|
||||||
|
their inspection views:
|
||||||
|
|
||||||
|
```go
|
||||||
|
earlier, err := earlierEngine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
later, err := candidateEngine.InspectConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
differences, err := earlier.Diff(later)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, difference := range differences {
|
||||||
|
if difference.HiddenChanged {
|
||||||
|
// Report a protected change without inferring its value.
|
||||||
|
}
|
||||||
|
if difference.Left.Redacted || difference.Right.Redacted {
|
||||||
|
// Apply the application's disclosure policy before presenting snapshots.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
|
||||||
|
and ConfigKit's [v0.7.0 effective-configuration diff guidance](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.7.0/docs/config.md#effective-configuration-diffs)
|
||||||
|
for the exact comparison contract. The operation compares only frozen effective
|
||||||
|
configuration values: it does not reread external files or compare their
|
||||||
|
contents, resolved credential contents, later execution preparation, or
|
||||||
|
request-specific overrides. An empty diff is not proof of equivalent execution.
|
||||||
|
Records can contain ordinary application data, so use the application's
|
||||||
|
disclosure policy when presenting them.
|
||||||
|
|
||||||
## Inspect A Prompt Before Preparation
|
## Inspect A Prompt Before Preparation
|
||||||
|
|
||||||
Use [`Engine.InspectPrompt`](../../engine.go) to check one configured prompt's
|
Use [`Engine.InspectPrompt`](../../engine.go) to check one configured prompt's
|
||||||
@@ -249,6 +284,41 @@ conditional entries. Do not treat an empty partial inventory as proof that no
|
|||||||
input bindings are needed. See the method's [GoDoc](../../requirements.go) for
|
input bindings are needed. See the method's [GoDoc](../../requirements.go) for
|
||||||
the exact classifications, supported syntax, ownership, and error contract.
|
the exact classifications, supported syntax, ownership, and error contract.
|
||||||
|
|
||||||
|
## Validate A Concrete Request
|
||||||
|
|
||||||
|
Use [`Engine.ValidateRequest`](../../engine.go) when the application has a
|
||||||
|
complete `RunRequest` and needs to know whether that concrete invocation passes
|
||||||
|
preparation:
|
||||||
|
|
||||||
|
```go
|
||||||
|
request := promptkit.RunRequest{
|
||||||
|
PromptID: "meeting.summary",
|
||||||
|
Vars: map[string]string{"audience": "project leads"},
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"note": promptkit.Inline("Synthetic meeting notes"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if err := engine.ValidateRequest(ctx, request); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Unlike `InspectRequirements`, this checks supplied values, the selected template
|
||||||
|
branches, and request bindings through the existing preparation path. It checks
|
||||||
|
only this invocation, does not aggregate errors, and loads every supplied input
|
||||||
|
according to ordinary preparation behavior, even if a branch does not use it.
|
||||||
|
It makes no model call and returns no prepared snapshot, but artifact and schema
|
||||||
|
resource I/O and its time and memory costs remain. Success does not certify
|
||||||
|
provider acceptance or later execution success. See the method's GoDoc for its
|
||||||
|
exact error and context behavior.
|
||||||
|
|
||||||
|
Scriptorium can map the returned error to its own diagnostics and exit status.
|
||||||
|
Its `validate` and `run` commands should construct equivalent requests for
|
||||||
|
equivalent caller inputs so their preparation behavior agrees. Separate
|
||||||
|
subprocess calls prepare independently; an opaque `PrepareExecution` handle
|
||||||
|
cannot be transferred from `validate` to `run`. Use `PrepareExecution` when one
|
||||||
|
process and engine must execute the exact prepared snapshot.
|
||||||
|
|
||||||
## Prepare Without Model Execution
|
## Prepare Without Model Execution
|
||||||
|
|
||||||
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
|
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
|
||||||
@@ -269,8 +339,11 @@ prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
|
|||||||
The maintained
|
The maintained
|
||||||
[offline preparation example](../../examples/go-library/prepare/main.go)
|
[offline preparation example](../../examples/go-library/prepare/main.go)
|
||||||
shows a complete runnable setup with a prompt file, in-memory profile, and
|
shows a complete runnable setup with a prompt file, in-memory profile, and
|
||||||
inline input. Exact request requirements and prepared-result fields belong to
|
inline input. It demonstrates validation and preparation with the same request
|
||||||
the [`RunRequest` and `PreparedRun` GoDoc](../../types.go).
|
to make the two operations concrete; this performs preparation twice and is
|
||||||
|
illustrative, not a requirement to validate before `Prepare` or `Run`. Exact
|
||||||
|
request requirements and prepared-result fields belong to the
|
||||||
|
[`RunRequest` and `PreparedRun` GoDoc](../../types.go).
|
||||||
|
|
||||||
## Prepare Now And Execute The Same Snapshot Later
|
## Prepare Now And Execute The Same Snapshot Later
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ contributor workflow and validation.
|
|||||||
|
|
||||||
| Component | Implemented responsibility | References |
|
| Component | Implemented responsibility | References |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, prompt/profile/requirement inspection and identity listing, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
|
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, prompt/profile/requirement inspection and identity listing, concrete-request validation, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
|
||||||
| `examples/go-library/prepare` | Demonstrates offline preparation and explicit configuration inspection with a prompt file, programmatic profile, and inline input. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
| `examples/go-library/prepare` | Demonstrates offline preparation and explicit configuration inspection with a prompt file, programmatic profile, and inline input. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
||||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||||
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
|
||||||
|
|||||||
@@ -111,6 +111,12 @@ into its private payload, while deep-copying mutable provider extras and
|
|||||||
creating an independent details snapshot. Claim completion or discard clears
|
creating an independent details snapshot. Claim completion or discard clears
|
||||||
the execution payload without removing the details snapshot.
|
the execution payload without removing the details snapshot.
|
||||||
|
|
||||||
|
The root facade's `Engine.ValidateRequest` uses the same complete preparation
|
||||||
|
path as `Prepare` and discards the resulting internal prepared value without
|
||||||
|
converting it into a public snapshot. It does not add a runner operation or
|
||||||
|
validator component; request conversion and public error mapping remain at the
|
||||||
|
facade boundary.
|
||||||
|
|
||||||
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
||||||
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
||||||
the selected backend identity. Non-empty extra-parameter maps replace whole
|
the selected backend identity. Non-empty extra-parameter maps replace whole
|
||||||
|
|||||||
@@ -21,6 +21,16 @@ profile inheritance. It retains ConfigKit's result privately for explicit
|
|||||||
inspection, not for per-run resolution. The runtime/domain packages do not
|
inspection, not for per-run resolution. The runtime/domain packages do not
|
||||||
depend on ConfigKit.
|
depend on ConfigKit.
|
||||||
|
|
||||||
|
The `ConfigurationInspection.Diff` facade method compares two retained results
|
||||||
|
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
|
||||||
|
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
|
||||||
|
once to ConfigKit v0.7.0 and returns its public difference records. Comparison
|
||||||
|
uses only frozen effective values and does not reopen sources, resolve runtime
|
||||||
|
credentials, or enter the execution path. ConfigKit applies sensitivity from
|
||||||
|
both results; reports may still contain ordinary application data. See the
|
||||||
|
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||||
|
and ConfigKit's [v0.7.0 comparison contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.7.0/docs/config.md#effective-configuration-diffs).
|
||||||
|
|
||||||
After prompt materialization and effective-profile snapshot construction, the
|
After prompt materialization and effective-profile snapshot construction, the
|
||||||
adapter captures lexically ordered, identity-only inventories from the final
|
adapter captures lexically ordered, identity-only inventories from the final
|
||||||
prompt and indexed-profile maps. Its listing methods return caller-owned copies
|
prompt and indexed-profile maps. Its listing methods return caller-owned copies
|
||||||
@@ -49,9 +59,9 @@ strings, including empty strings. It closes content-file handles before
|
|||||||
returning and retains no source capability for public inspection.
|
returning and retains no source capability for public inspection.
|
||||||
|
|
||||||
Role and cache-control normalization is applied to execution-owned message
|
Role and cache-control normalization is applied to execution-owned message
|
||||||
copies. The retained ConfigKit configuration keeps effective input message
|
copies. ConfigKit v0.7.0 retains effective input message metadata and its
|
||||||
metadata and its resource origins: recording a sequence-wide normalization in
|
resource origins; recording a sequence-wide normalization in the retained
|
||||||
ConfigKit v0.6.0 would replace origins of unchanged content-file fields.
|
configuration would replace origins of unchanged content-file fields.
|
||||||
|
|
||||||
The runner selects full definitions from this index for preparation and uses
|
The runner selects full definitions from this index for preparation and uses
|
||||||
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
|
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
|
||||||
|
|||||||
@@ -26,17 +26,17 @@ License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
|
|||||||
|
|
||||||
## ConfigKit
|
## ConfigKit
|
||||||
|
|
||||||
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.6.0`.
|
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.7.1`.
|
||||||
|
|
||||||
Copyright (C) 2026 Eric Rakestraw
|
Copyright (C) 2026 Eric Rakestraw
|
||||||
|
|
||||||
SPDX-License-Identifier: LGPL-3.0-or-later
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
||||||
|
|
||||||
License and notices: [LGPL 3.0](../../LICENSE) |
|
License and notices: [LGPL 3.0](../../LICENSE) |
|
||||||
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/1a885934c28733d9a5dca261fd6da0e2a4279285/COPYRIGHT) |
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77/COPYRIGHT) |
|
||||||
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/1a885934c28733d9a5dca261fd6da0e2a4279285/LICENSE).
|
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77/LICENSE).
|
||||||
|
|
||||||
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/1a885934c28733d9a5dca261fd6da0e2a4279285).
|
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77).
|
||||||
|
|
||||||
## Promptkit OpenRouter Catalog v3
|
## Promptkit OpenRouter Catalog v3
|
||||||
|
|
||||||
|
|||||||
@@ -74,7 +74,7 @@ For an optional local diagnostic of a committed candidate, run:
|
|||||||
```
|
```
|
||||||
|
|
||||||
This is not a required step before publication. The upstream
|
This is not a required step before publication. The upstream
|
||||||
[candidate-check contract](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.3.0/docs/consumers/check.md)
|
[candidate-check contract](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/consumers/check.md)
|
||||||
describes its requirements.
|
describes its requirements.
|
||||||
|
|
||||||
Run the single publication command with the chosen version:
|
Run the single publication command with the chosen version:
|
||||||
@@ -114,7 +114,7 @@ resolved. ReleaseKit reports the attempted version, tag object, and recovery
|
|||||||
state, including the captured publication endpoint when available. Use that
|
state, including the captured publication endpoint when available. Use that
|
||||||
endpoint—not merely the current `origin` URL—to query the exact tag; follow the
|
endpoint—not merely the current `origin` URL—to query the exact tag; follow the
|
||||||
upstream
|
upstream
|
||||||
[failure-recovery procedure](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.3.0/docs/release.md#resolve-failures):
|
[failure-recovery procedure](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/release.md#resolve-failures):
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
releasekit_endpoint='COPY_CAPTURED_ENDPOINT'
|
releasekit_endpoint='COPY_CAPTURED_ENDPOINT'
|
||||||
|
|||||||
48
docs/releases/v0.14.0.md
Normal file
48
docs/releases/v0.14.0.md
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
# Promptkit v0.14.0
|
||||||
|
|
||||||
|
This release adds comparison of captured configuration inspections and updates
|
||||||
|
ConfigKit and the maintainer release tool. The minimum supported Go version
|
||||||
|
remains 1.26.0.
|
||||||
|
|
||||||
|
## Compare Captured Configurations
|
||||||
|
|
||||||
|
`ConfigurationInspection.Diff` compares the effective configuration captured by
|
||||||
|
two engines. Consumers can use it to review a candidate configuration, inspect
|
||||||
|
an overlay's effect, or present configuration changes to operators.
|
||||||
|
|
||||||
|
Comparison delegates to ConfigKit and returns its difference records, including
|
||||||
|
presence information, paired redaction, and protected-change indicators. It
|
||||||
|
does not reload sources or contact providers. Source-history differences alone
|
||||||
|
do not produce value differences.
|
||||||
|
|
||||||
|
An empty diff is not proof of equivalent execution: referenced file contents,
|
||||||
|
resolved credentials, later execution preparation, and request-specific
|
||||||
|
overrides are outside the comparison. Records can contain ordinary application
|
||||||
|
data; callers retain responsibility for their disclosure.
|
||||||
|
|
||||||
|
See the [configuration comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
|
||||||
|
and [GoDoc](../../configuration_inspection.go) for usage and the exact contract.
|
||||||
|
|
||||||
|
## Dependency And Tooling Updates
|
||||||
|
|
||||||
|
- Upgrade ConfigKit from v0.6.0 to v0.7.1, providing the upstream comparison API.
|
||||||
|
v0.7.1 also updates ConfigKit's maintainer tooling without changing its runtime
|
||||||
|
behavior from v0.7.0.
|
||||||
|
- Upgrade the module-selected ReleaseKit tool from v0.3.0 to v0.4.1. Promptkit
|
||||||
|
retains its existing root-module profile, publication entry point, and release
|
||||||
|
gates. See the [release procedure](../release.md) for maintainer commands.
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
No application-code, configuration, or Go-toolchain migration is required from
|
||||||
|
v0.13.0. Configuration comparison is an optional API addition; existing
|
||||||
|
construction, inspection, preparation, and execution workflows remain available.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.14.0
|
||||||
|
```
|
||||||
|
|
||||||
|
See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility) for the
|
||||||
|
distinction between the consumer minimum and maintainer toolchain.
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/docs/oss/attribution.md)
|
||||||
34
docs/releases/v0.15.0.md
Normal file
34
docs/releases/v0.15.0.md
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
# Promptkit v0.15.0
|
||||||
|
|
||||||
|
This release adds error-only validation of a concrete prompt request without
|
||||||
|
calling a model. The minimum supported Go version remains 1.26.0.
|
||||||
|
|
||||||
|
## Validate A Concrete Request
|
||||||
|
|
||||||
|
`Engine.ValidateRequest` accepts an ordinary `RunRequest` and returns nil when
|
||||||
|
it passes preparation, or the existing preparation error. Consumers can use it
|
||||||
|
for fail-fast checks, including downstream CLI validation commands, without
|
||||||
|
receiving a prepared snapshot.
|
||||||
|
|
||||||
|
The method shares `Prepare`'s request conversion and preparation path. Unlike
|
||||||
|
requirements inspection, it checks the actual supplied values and bindings.
|
||||||
|
Artifact and schema resource I/O still occurs; success does not certify provider
|
||||||
|
acceptance or later execution success. A subsequent run prepares independently.
|
||||||
|
Validation is optional, not a required step before preparation or execution.
|
||||||
|
|
||||||
|
See the [request validation guide](../consumers/pkg-promptkit.md#validate-a-concrete-request)
|
||||||
|
and [GoDoc](../../engine.go) for usage and the exact contract. Consumers that
|
||||||
|
need execution tied to a checked snapshot can use the existing
|
||||||
|
[prepared execution workflow](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later).
|
||||||
|
|
||||||
|
## Compatibility And Upgrade
|
||||||
|
|
||||||
|
No application-code, configuration, or Go-toolchain migration is required from
|
||||||
|
v0.14.0. This is an optional API addition; existing inspection, preparation, and
|
||||||
|
execution workflows remain available.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.15.0
|
||||||
|
```
|
||||||
|
|
||||||
|
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/docs/oss/attribution.md)
|
||||||
28
engine.go
28
engine.go
@@ -346,6 +346,32 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
|
|||||||
// relationship to ctx.Err is promised. Prepare returns no partial result on
|
// relationship to ctx.Err is promised. Prepare returns no partial result on
|
||||||
// error. It validates RunRequest.TransientRetries but makes no model attempts.
|
// error. It validates RunRequest.TransientRetries but makes no model attempts.
|
||||||
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
||||||
|
prepared, err := e.prepareRequest(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return fromDomainPreparedRun(prepared), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateRequest reports whether req passes the same preparation path used by
|
||||||
|
// [Engine.Prepare] at the time of this call. A nil error means this concrete
|
||||||
|
// request passed preparation. It checks only this invocation and returns errors
|
||||||
|
// as encountered rather than aggregating them. All supplied inputs follow the
|
||||||
|
// same loading behavior as [Engine.Prepare], including inputs unused by the
|
||||||
|
// selected template branch. The method does not call an LLM or retain a
|
||||||
|
// prepared snapshot or executable handle, but preparation may still perform
|
||||||
|
// resource I/O and incur its associated time and memory costs.
|
||||||
|
//
|
||||||
|
// Its errors and context handling follow [Engine.Prepare], including an error
|
||||||
|
// matching ErrInvalidConfig when the Engine is nil. Success does not guarantee
|
||||||
|
// provider acceptance or later execution success. Use [Engine.PrepareExecution]
|
||||||
|
// when execution must use the snapshot produced by preparation.
|
||||||
|
func (e *Engine) ValidateRequest(ctx context.Context, req RunRequest) error {
|
||||||
|
_, err := e.prepareRequest(ctx, req)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) prepareRequest(ctx context.Context, req RunRequest) (*domain.PreparedRun, error) {
|
||||||
ctx = observation.WithoutExecutionState(ctx)
|
ctx = observation.WithoutExecutionState(ctx)
|
||||||
if e == nil || e.internal == nil {
|
if e == nil || e.internal == nil {
|
||||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
@@ -360,7 +386,7 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, mapPublicError(err)
|
return nil, mapPublicError(err)
|
||||||
}
|
}
|
||||||
return fromDomainPreparedRun(prepared), nil
|
return prepared, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrepareExecution completely prepares a prompt request without calling the
|
// PrepareExecution completely prepares a prompt request without calling the
|
||||||
|
|||||||
@@ -48,12 +48,16 @@ func main() {
|
|||||||
exit(fmt.Errorf("required note input was not discovered before preparation"))
|
exit(fmt.Errorf("required note input was not discovered before preparation"))
|
||||||
}
|
}
|
||||||
|
|
||||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
request := promptkit.RunRequest{
|
||||||
PromptID: "example.prepare",
|
PromptID: "example.prepare",
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
"note": promptkit.Inline("Ada finished the migration review."),
|
"note": promptkit.Inline("Ada finished the migration review."),
|
||||||
},
|
},
|
||||||
})
|
}
|
||||||
|
if err := engine.ValidateRequest(context.Background(), request); err != nil {
|
||||||
|
exit(err)
|
||||||
|
}
|
||||||
|
prepared, err := engine.Prepare(context.Background(), request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
exit(err)
|
exit(err)
|
||||||
}
|
}
|
||||||
|
|||||||
4
go.mod
4
go.mod
@@ -3,14 +3,14 @@ module gitea.maximumdirect.net/eric/promptkit
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.6.0
|
gitea.maximumdirect.net/eric/ConfigKit v0.7.1
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
gitea.maximumdirect.net/eric/ReleaseKit v0.3.0 // indirect
|
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 // indirect
|
||||||
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
github.com/yuin/goldmark/v2 v2.1.6 // indirect
|
||||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
|
||||||
golang.org/x/mod v0.40.0 // indirect
|
golang.org/x/mod v0.40.0 // indirect
|
||||||
|
|||||||
8
go.sum
8
go.sum
@@ -1,7 +1,7 @@
|
|||||||
gitea.maximumdirect.net/eric/ConfigKit v0.6.0 h1:KdsCVO5kKctdAUQJ6X7dJ/GemlVjCYCcn/xZrMAE/aw=
|
gitea.maximumdirect.net/eric/ConfigKit v0.7.1 h1:q3HrX4WWCPt1WfMydqFEaAmCf9s6Dq+vknJj0C/mdcI=
|
||||||
gitea.maximumdirect.net/eric/ConfigKit v0.6.0/go.mod h1:xN4dRdDGIW4yh/e9GghDF/hSIzc8/77blRhwyf90KSA=
|
gitea.maximumdirect.net/eric/ConfigKit v0.7.1/go.mod h1:C82sbyKHYMV1OwaD2GJanFrwTfcZfjlW2JsGze5b0kA=
|
||||||
gitea.maximumdirect.net/eric/ReleaseKit v0.3.0 h1:aTMn+50MEpHpHI2RoCCmJYsy7Ohrpa+RSva63Kq0/tU=
|
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 h1:Qn+mdebYOzCVCEXgCbuGSO/WHmJtItwaQ8e3XwY3qtI=
|
||||||
gitea.maximumdirect.net/eric/ReleaseKit v0.3.0/go.mod h1:LVodqNQ3OIKv9NBq+fFeyMByd1lHY25YOmmpah/NaNI=
|
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1/go.mod h1:LVodqNQ3OIKv9NBq+fFeyMByd1lHY25YOmmpah/NaNI=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
|
||||||
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
|
||||||
|
|||||||
@@ -16,6 +16,15 @@ type Inspection interface {
|
|||||||
Explain(configkit.Path) (configkit.Explanation, bool)
|
Explain(configkit.Path) (configkit.Explanation, bool)
|
||||||
WriteEffective(io.Writer) error
|
WriteEffective(io.Writer) error
|
||||||
WriteExplanation(io.Writer, configkit.Path) error
|
WriteExplanation(io.Writer, configkit.Path) error
|
||||||
|
capturedResult() *configkit.Result[document]
|
||||||
|
}
|
||||||
|
|
||||||
|
type inspectionResult struct {
|
||||||
|
*configkit.Result[document]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (inspection inspectionResult) capturedResult() *configkit.Result[document] {
|
||||||
|
return inspection.Result
|
||||||
}
|
}
|
||||||
|
|
||||||
// Inspection returns only the retained result's inspection operations.
|
// Inspection returns only the retained result's inspection operations.
|
||||||
@@ -23,7 +32,19 @@ func (state *loaded) Inspection() Inspection {
|
|||||||
if state == nil {
|
if state == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return state.result
|
return inspectionResult{Result: state.result}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Diff compares two captured ConfigKit results without reconstructing them.
|
||||||
|
func Diff(left, right Inspection) ([]configkit.Difference, error) {
|
||||||
|
var leftResult, rightResult *configkit.Result[document]
|
||||||
|
if left != nil {
|
||||||
|
leftResult = left.capturedResult()
|
||||||
|
}
|
||||||
|
if right != nil {
|
||||||
|
rightResult = right.capturedResult()
|
||||||
|
}
|
||||||
|
return configkit.Diff(leftResult, rightResult)
|
||||||
}
|
}
|
||||||
|
|
||||||
func secretProviderPath(path configkit.Path) bool {
|
func secretProviderPath(path configkit.Path) bool {
|
||||||
|
|||||||
218
validate_request_contract_test.go
Normal file
218
validate_request_contract_test.go
Normal file
@@ -0,0 +1,218 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"path/filepath"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestValidateRequestMatchesPrepareContract(t *testing.T) {
|
||||||
|
var modelCalls atomic.Int64
|
||||||
|
profiles := map[string]any{
|
||||||
|
"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
|
||||||
|
}
|
||||||
|
newEngine := func(t *testing.T, definition map[string]any) *promptkit.Engine {
|
||||||
|
t.Helper()
|
||||||
|
return requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), profiles,
|
||||||
|
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("successful request", func(t *testing.T) {
|
||||||
|
engine := newEngine(t, contractPrompt("offline", "hello", nil, "", noValidationOutput))
|
||||||
|
assertValidateRequestParity(t, engine, promptkit.RunRequest{PromptID: "synthetic"}, nil)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("nil engine", func(t *testing.T) {
|
||||||
|
var engine *promptkit.Engine
|
||||||
|
request := promptkit.RunRequest{PromptID: "synthetic"}
|
||||||
|
assertValidateRequestParity(t, engine, request, promptkit.ErrInvalidConfig)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("invalid appended message", func(t *testing.T) {
|
||||||
|
engine := newEngine(t, contractPrompt("offline", "hello", nil, "", noValidationOutput))
|
||||||
|
request := promptkit.RunRequest{
|
||||||
|
PromptID: "synthetic",
|
||||||
|
AppendedMessages: []promptkit.RenderedMessage{{
|
||||||
|
Role: "unsupported-role", Content: "not sent to a model",
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
assertValidateRequestParity(t, engine, request, promptkit.ErrInvalidRequest)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("missing required input", func(t *testing.T) {
|
||||||
|
definition := contractPrompt("offline", `{{input "required"}}`, []any{
|
||||||
|
map[string]any{"name": "required", "required": true},
|
||||||
|
}, "", noValidationOutput)
|
||||||
|
engine := newEngine(t, definition)
|
||||||
|
assertValidateRequestParity(t, engine, promptkit.RunRequest{PromptID: "synthetic"}, promptkit.ErrPromptRender)
|
||||||
|
})
|
||||||
|
|
||||||
|
if got := modelCalls.Load(); got != 0 {
|
||||||
|
t.Fatalf("preparation operations made %d model calls, want none", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertValidateRequestParity(t *testing.T, engine *promptkit.Engine, request promptkit.RunRequest, want error) *promptkit.PreparedRun {
|
||||||
|
t.Helper()
|
||||||
|
prepared, prepareErr := engine.Prepare(context.Background(), request)
|
||||||
|
validateErr := engine.ValidateRequest(context.Background(), request)
|
||||||
|
if want == nil {
|
||||||
|
if prepareErr != nil || prepared == nil {
|
||||||
|
t.Fatalf("Prepare() = (%+v, %v), want a prepared result", prepared, prepareErr)
|
||||||
|
}
|
||||||
|
if validateErr != nil {
|
||||||
|
t.Fatalf("ValidateRequest() error = %v, want nil", validateErr)
|
||||||
|
}
|
||||||
|
return prepared
|
||||||
|
}
|
||||||
|
if prepared != nil || !errors.Is(prepareErr, want) {
|
||||||
|
t.Fatalf("Prepare() = (%+v, %v), want error matching %v", prepared, prepareErr, want)
|
||||||
|
}
|
||||||
|
if !errors.Is(validateErr, want) {
|
||||||
|
t.Fatalf("ValidateRequest() error = %v, want error matching %v", validateErr, want)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRequestUsesSelectedTemplateBranchesAndVariablePresence(t *testing.T) {
|
||||||
|
var modelCalls atomic.Int64
|
||||||
|
profiles := map[string]any{
|
||||||
|
"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
|
||||||
|
}
|
||||||
|
branchContent := `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`
|
||||||
|
optionalDefinition := contractPrompt("offline", branchContent, []any{
|
||||||
|
map[string]any{"name": "details", "required": false},
|
||||||
|
}, "", noValidationOutput)
|
||||||
|
requiredDefinition := contractPrompt("offline", branchContent, []any{
|
||||||
|
map[string]any{"name": "details", "required": true},
|
||||||
|
}, "", noValidationOutput)
|
||||||
|
engine := requirementsEngine(t, map[string]any{
|
||||||
|
"optional": oneVersionPrompt("optional", "v1", optionalDefinition)["optional"],
|
||||||
|
"required": oneVersionPrompt("required", "v1", requiredDefinition)["required"],
|
||||||
|
"direct": oneVersionPrompt("direct", "v1", contractPrompt("offline", `{{.audience}}`, nil, "", noValidationOutput))["direct"],
|
||||||
|
"index": oneVersionPrompt("index", "v1", contractPrompt("offline", `{{index . "audience"}}`, nil, "", noValidationOutput))["index"],
|
||||||
|
}, profiles, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
request promptkit.RunRequest
|
||||||
|
want error
|
||||||
|
}{
|
||||||
|
{name: "brief branch omits optional input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "brief"}}},
|
||||||
|
{name: "detailed branch needs optional input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "detailed"}}, want: promptkit.ErrPromptRender},
|
||||||
|
{name: "detailed branch accepts inline input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "detailed"}, Inputs: map[string]promptkit.ArtifactRef{"details": promptkit.Inline("Long form details")}}},
|
||||||
|
{name: "declared required input is needed on brief branch", request: promptkit.RunRequest{PromptID: "required", Vars: map[string]string{"mode": "brief"}}, want: promptkit.ErrPromptRender},
|
||||||
|
{name: "evaluated variable needs a supplied key", request: promptkit.RunRequest{PromptID: "direct"}, want: promptkit.ErrPromptRender},
|
||||||
|
{name: "empty value and unrelated variable are accepted", request: promptkit.RunRequest{PromptID: "direct", Vars: map[string]string{"audience": "", "unrelated": "accepted"}}},
|
||||||
|
{name: "missing index lookup remains optional", request: promptkit.RunRequest{PromptID: "index"}},
|
||||||
|
}
|
||||||
|
for _, test := range cases {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
assertValidateRequestParity(t, engine, test.request, test.want)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if got := modelCalls.Load(); got != 0 {
|
||||||
|
t.Fatalf("preparation operations made %d model calls, want none", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type canceledValidationArtifactReader struct{}
|
||||||
|
|
||||||
|
func (canceledValidationArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||||
|
return nil, context.Canceled
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRequestPreservesResourceAndSelectionFailures(t *testing.T) {
|
||||||
|
var modelCalls atomic.Int64
|
||||||
|
profiles := map[string]any{
|
||||||
|
"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
|
||||||
|
}
|
||||||
|
branch := `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`
|
||||||
|
definition := contractPrompt("offline", branch, []any{
|
||||||
|
map[string]any{"name": "details", "required": false},
|
||||||
|
}, "", noValidationOutput)
|
||||||
|
profilePrompt := contractPrompt("offline", "hello", nil, "", noValidationOutput)
|
||||||
|
schemaPrompt := contractPrompt("offline", "{}", nil, "", map[string]any{
|
||||||
|
"format": "json", "validation_mode": "json_schema", "schema_path": "missing-schema.json",
|
||||||
|
})
|
||||||
|
prompts := map[string]any{
|
||||||
|
"branch": oneVersionPrompt("branch", "v1", definition)["branch"],
|
||||||
|
"profile": oneVersionPrompt("profile", "v1", profilePrompt)["profile"],
|
||||||
|
"schema": oneVersionPrompt("schema", "v1", schemaPrompt)["schema"],
|
||||||
|
}
|
||||||
|
engine := requirementsEngine(t, prompts, profiles,
|
||||||
|
promptkit.WithSchemaFS(fstest.MapFS{}, "."),
|
||||||
|
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}),
|
||||||
|
)
|
||||||
|
|
||||||
|
t.Run("unselected supplied input is still loaded", func(t *testing.T) {
|
||||||
|
missingPath := filepath.Join(t.TempDir(), "missing-input.txt")
|
||||||
|
request := promptkit.RunRequest{
|
||||||
|
PromptID: "branch",
|
||||||
|
Vars: map[string]string{"mode": "brief"},
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{"details": promptkit.File(missingPath)},
|
||||||
|
}
|
||||||
|
assertValidateRequestParity(t, engine, request, promptkit.ErrArtifactLoad)
|
||||||
|
})
|
||||||
|
t.Run("missing selected profile", func(t *testing.T) {
|
||||||
|
request := promptkit.RunRequest{PromptID: "profile", ProfileID: "not-configured"}
|
||||||
|
assertValidateRequestParity(t, engine, request, promptkit.ErrProfileNotFound)
|
||||||
|
})
|
||||||
|
t.Run("unavailable selected schema", func(t *testing.T) {
|
||||||
|
request := promptkit.RunRequest{PromptID: "schema"}
|
||||||
|
assertValidateRequestParity(t, engine, request, promptkit.ErrValidation)
|
||||||
|
})
|
||||||
|
if got := modelCalls.Load(); got != 0 {
|
||||||
|
t.Fatalf("preparation operations made %d model calls, want none", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRequestPreservesCanceledLoaderCategory(t *testing.T) {
|
||||||
|
var modelCalls atomic.Int64
|
||||||
|
definition := contractPrompt("offline", `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`, []any{
|
||||||
|
map[string]any{"name": "details", "required": false},
|
||||||
|
}, "", noValidationOutput)
|
||||||
|
engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{
|
||||||
|
"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
|
||||||
|
}, promptkit.WithArtifactReader(canceledValidationArtifactReader{}),
|
||||||
|
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
|
||||||
|
request := promptkit.RunRequest{
|
||||||
|
PromptID: "synthetic",
|
||||||
|
Vars: map[string]string{"mode": "brief"},
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{"details": promptkit.File("controlled-canceled-loader")},
|
||||||
|
}
|
||||||
|
assertValidateRequestParity(t, engine, request, promptkit.ErrArtifactLoad)
|
||||||
|
if got := modelCalls.Load(); got != 0 {
|
||||||
|
t.Fatalf("preparation operations made %d model calls, want none", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRequestForwardsProfileOverridesAndRequestSettings(t *testing.T) {
|
||||||
|
var modelCalls atomic.Int64
|
||||||
|
definition := contractPrompt("partial", "base message", nil, "", noValidationOutput)
|
||||||
|
engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{
|
||||||
|
"partial": map[string]any{"model": "partial-model"},
|
||||||
|
}, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
|
||||||
|
request := promptkit.RunRequest{
|
||||||
|
PromptID: "synthetic",
|
||||||
|
SessionID: "request-session",
|
||||||
|
Execution: &promptkit.ExecutionTargetOverride{Endpoint: "http://override.example/v1"},
|
||||||
|
AppendedMessages: []promptkit.RenderedMessage{{
|
||||||
|
Role: promptkit.RoleAssistant, Content: "caller suffix",
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
prepared := assertValidateRequestParity(t, engine, request, nil)
|
||||||
|
if prepared.SelectedProfileID != "partial" || prepared.EffectiveModelParams.Model != "partial-model" ||
|
||||||
|
prepared.EffectiveModelParams.Endpoint != "http://override.example/v1" || prepared.SessionID != "request-session" ||
|
||||||
|
len(prepared.Messages) != 2 || prepared.Messages[1].Content != "caller suffix" {
|
||||||
|
t.Fatalf("preparation fixture did not exercise profile and request settings: %+v", prepared)
|
||||||
|
}
|
||||||
|
if got := modelCalls.Load(); got != 0 {
|
||||||
|
t.Fatalf("preparation operations made %d model calls, want none", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user