20 Commits

Author SHA1 Message Date
635674c066 Prepare v0.15.0 release documentation 2026-10-12 20:17:52 +00:00
50bc35516a Consolidate request validation parity assertions 2026-10-12 19:31:54 +00:00
18309f50db Record request validation review finding 2026-10-12 19:05:38 +00:00
60784e7875 Document request validation workflow 2026-10-12 17:47:04 +00:00
f3148bfe52 Verify request validation parity 2026-10-12 17:45:18 +00:00
f315620bf7 Add request validation API 2026-10-12 17:43:11 +00:00
97357d62b9 Plan request validation in focused stages 2026-10-12 17:35:25 +00:00
0c3651fceb Clarify request validation roadmap and CLI guidance 2026-10-12 17:26:50 +00:00
5a5d47d36e Define request preflight validation roadmap 2026-10-12 16:56:33 +00:00
143c6e9b3a Prepare v0.14.0 release notes 2026-10-11 01:19:17 +00:00
48e4784fcd Retire completed configuration comparison roadmaps 2026-10-11 01:18:30 +00:00
78af5069c1 Update ConfigKit to v0.7.1 2026-10-11 01:17:10 +00:00
18963266d7 Upgrade ReleaseKit to v0.4.1 2026-10-11 01:11:19 +00:00
d039a019cb Document captured configuration comparison 2026-10-09 16:06:09 +00:00
d1c0281053 Verify captured configuration diff concurrency 2026-10-09 16:01:24 +00:00
870e4ca4e6 Verify configuration diff disclosure semantics 2026-10-09 15:59:58 +00:00
fd82524977 Add captured configuration comparison 2026-10-09 15:58:04 +00:00
3f76c75919 Adopt ConfigKit v0.7.0 2026-10-09 15:56:10 +00:00
fe764f72d8 Plan captured configuration comparison in focused stages 2026-10-09 15:46:41 +00:00
26ab566f38 Define captured configuration comparison roadmap 2026-10-09 15:41:53 +00:00
18 changed files with 782 additions and 32 deletions

View File

@@ -37,6 +37,9 @@ boundary and constraints that framework work must preserve.
## Release Guidance
The [v0.15.0 release notes](docs/releases/v0.15.0.md) cover error-only
validation of concrete requests with `Engine.ValidateRequest`.
Consumers upgrading to `v0.12.0` should read the
[v0.12.0 release notes](docs/releases/v0.12.0.md) for the intentional ConfigKit
configuration and public construction API migration.

View File

@@ -0,0 +1,243 @@
package promptkit_test
import (
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) {
left := comparisonInspection(t, "earlier-model")
right := comparisonInspection(t, "later-model")
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare captured configurations: %v", err)
}
if len(differences) != 1 {
t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences)
}
difference := differences[0]
if got, want := difference.Path.String(), "/profiles/offline/model"; got != want {
t.Fatalf("difference path = %q, want %q", got, want)
}
if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") {
t.Fatalf("difference snapshots = (%q, %q)", difference.Left.YAML, difference.Right.YAML)
}
reversed, err := right.Diff(left)
if err != nil {
t.Fatalf("compare reversed configurations: %v", err)
}
if len(reversed) != 1 || reversed[0].Path.String() != difference.Path.String() ||
!strings.Contains(reversed[0].Left.YAML, "later-model") ||
!strings.Contains(reversed[0].Right.YAML, "earlier-model") {
t.Fatalf("reversed difference = %+v", reversed)
}
equal := comparisonInspection(t, "earlier-model")
if equal, err := left.Diff(equal); err != nil || len(equal) != 0 {
t.Fatalf("compare equal configurations = (%+v, %v), want no differences", equal, err)
}
if equal, err := left.Diff(left); err != nil || len(equal) != 0 {
t.Fatalf("compare view with itself = (%+v, %v), want no differences", equal, err)
}
}
func TestConfigurationInspectionDiffRejectsZeroViews(t *testing.T) {
valid := comparisonInspection(t, "current-model")
var zero promptkit.ConfigurationInspection
for _, comparison := range []struct {
name string
left, right promptkit.ConfigurationInspection
}{
{name: "zero left", left: zero, right: valid},
{name: "zero right", left: valid, right: zero},
{name: "both zero", left: zero, right: zero},
} {
t.Run(comparison.name, func(t *testing.T) {
differences, err := comparison.left.Diff(comparison.right)
if !errors.Is(err, promptkit.ErrInvalidConfig) || len(differences) != 0 {
t.Fatalf("Diff() = (%+v, %v), want ErrInvalidConfig and no records", differences, err)
}
})
}
}
func TestConfigurationInspectionDiffIgnoresSourceHistory(t *testing.T) {
left := comparisonInspectionFrom(t, "earlier source", "same-model", nil)
right := comparisonInspectionFrom(t, "later source", "same-model", nil)
if !hasSourceLabel(left, "earlier source") || !hasSourceLabel(right, "later source") {
t.Fatal("comparison fixtures did not retain their distinct source labels")
}
differences, err := left.Diff(right)
if err != nil || len(differences) != 0 {
t.Fatalf("compare equal effective values with different sources = (%+v, %v), want no differences", differences, err)
}
}
func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
"sequence": []any{"earlier", "unchanged"},
})
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
"nullable": nil,
"sequence": []any{"later", "unchanged"},
})
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare presence and sequence values: %v", err)
}
if len(differences) != 2 {
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
}
byPath := differencesByPath(differences)
nullable, ok := byPath["/backends/local/extra_params/nullable"]
if !ok || nullable.Left.Present || !nullable.Right.Present || strings.TrimSpace(nullable.Right.YAML) != "null" {
t.Fatalf("absent-to-null difference = (%+v, %t)", nullable, ok)
}
sequence, ok := byPath["/backends/local/extra_params/sequence"]
if !ok || !sequence.Left.Present || !sequence.Right.Present ||
!strings.Contains(sequence.Left.YAML, "earlier") || !strings.Contains(sequence.Right.YAML, "later") {
t.Fatalf("atomic sequence difference = (%+v, %t)", sequence, ok)
}
}
func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t *testing.T) {
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
"token": "left-secret-sentinel",
"visible": "before-value",
})
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
"token": "right-secret-sentinel",
"visible": "after-value",
})
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare sensitive values: %v", err)
}
if len(differences) != 2 {
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
}
byPath := differencesByPath(differences)
secret, ok := byPath["/backends/local/extra_params/token"]
if !ok || !secret.Left.Redacted || !secret.Right.Redacted || !secret.HiddenChanged {
t.Fatalf("sensitive difference = (%+v, %t)", secret, ok)
}
if strings.Contains(secret.Left.YAML, "left-secret-sentinel") || strings.Contains(secret.Left.YAML, "right-secret-sentinel") ||
strings.Contains(secret.Right.YAML, "left-secret-sentinel") || strings.Contains(secret.Right.YAML, "right-secret-sentinel") {
t.Fatalf("sensitive snapshots disclosed a value: %+v", secret)
}
visible, ok := byPath["/backends/local/extra_params/visible"]
if !ok || visible.Left.Redacted || visible.Right.Redacted || visible.HiddenChanged ||
!strings.Contains(visible.Left.YAML, "before-value") || !strings.Contains(visible.Right.YAML, "after-value") {
t.Fatalf("public sibling difference = (%+v, %t)", visible, ok)
}
differences[0].Path = configkit.Keys("caller", "mutation")
differences[0].Left.YAML = "caller mutation"
differences[0].Right.YAML = "caller mutation"
again, err := left.Diff(right)
if err != nil {
t.Fatalf("repeat comparison after mutating returned records: %v", err)
}
if len(again) != 2 {
t.Fatalf("repeat difference count = %d, want 2: %+v", len(again), again)
}
againByPath := differencesByPath(again)
if _, exists := againByPath["/caller/mutation"]; exists {
t.Fatal("mutating returned path reached retained inspection")
}
if _, exists := againByPath["/backends/local/extra_params/token"]; !exists {
t.Fatalf("original sensitive path was lost after mutation: %+v", again)
}
if _, exists := againByPath["/backends/local/extra_params/visible"]; !exists {
t.Fatalf("original public path was lost after mutation: %+v", again)
}
for _, difference := range again {
if difference.Left.YAML == "caller mutation" || difference.Right.YAML == "caller mutation" {
t.Fatalf("mutating returned snapshots reached retained inspection: %+v", difference)
}
}
}
func TestConfigurationInspectionDiffMasksSensitiveShapeReplacement(t *testing.T) {
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
"details": map[string]any{
"token": "nested-secret-sentinel",
"label": "public detail",
},
})
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
"details": "replacement",
})
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare sensitive shape replacement: %v", err)
}
if len(differences) != 1 || differences[0].Path.String() != "/backends/local/extra_params/details" {
t.Fatalf("shape replacement differences = %+v", differences)
}
difference := differences[0]
if !difference.Left.Redacted || !difference.Right.Redacted || !difference.HiddenChanged {
t.Fatalf("shape replacement masking = %+v", difference)
}
if strings.Contains(difference.Left.YAML, "nested-secret-sentinel") || strings.Contains(difference.Right.YAML, "nested-secret-sentinel") {
t.Fatalf("shape replacement disclosed a sensitive value: %+v", difference)
}
}
func comparisonInspection(t *testing.T, model string) promptkit.ConfigurationInspection {
t.Helper()
return comparisonInspectionFrom(t, "comparison fixture", model, nil)
}
func comparisonInspectionFrom(t *testing.T, source, model string, extraParams map[string]any) promptkit.ConfigurationInspection {
t.Helper()
backend := map[string]any{"endpoint": "https://example.invalid/v1"}
if extraParams != nil {
backend["extra_params"] = extraParams
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping(source, map[string]any{
"backends": map[string]any{"local": backend},
"profiles": map[string]any{"offline": map[string]any{
"backend": "local",
"model": model,
}},
})}}}},
}})
if err != nil {
t.Fatalf("construct comparison engine: %v", err)
}
inspection, err := engine.InspectConfiguration()
if err != nil {
t.Fatalf("inspect comparison engine: %v", err)
}
return inspection
}
func differencesByPath(differences []configkit.Difference) map[string]configkit.Difference {
byPath := make(map[string]configkit.Difference, len(differences))
for _, difference := range differences {
byPath[difference.Path.String()] = difference
}
return byPath
}
func hasSourceLabel(view promptkit.ConfigurationInspection, label string) bool {
for _, source := range view.Sources() {
if source.Label == label {
return true
}
}
return false
}

View File

@@ -27,6 +27,23 @@ func (e *Engine) InspectConfiguration() (ConfigurationInspection, error) {
return ConfigurationInspection{source: e.internal.InspectConfiguration()}, nil
}
// Diff compares this view's captured effective configuration with other. This
// view is the earlier (left) value and other is the later (right) value. The
// comparison delegates to ConfigKit's frozen effective-value semantics, so
// source or resolution history alone does not create differences. It does not
// reread external sources or compare referenced file contents, resolved
// credential contents, later execution preparation, or request-specific
// overrides. Either zero view returns ErrInvalidConfig and no records; other
// ConfigKit errors are returned unchanged. An empty diff is not evidence of
// equivalent engine behavior or execution. Records can contain ordinary
// application data, so callers choose where and to whom to disclose them.
func (v ConfigurationInspection) Diff(other ConfigurationInspection) ([]configkit.Difference, error) {
if v.source == nil || other.source == nil {
return nil, ErrInvalidConfig
}
return configuration.Diff(v.source, other.source)
}
// Metadata returns detached effective presence and history records.
func (v ConfigurationInspection) Metadata() configkit.Metadata {
if v.source == nil {

View File

@@ -144,9 +144,17 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
if err != nil {
t.Fatalf("select inputs: %v", err)
}
otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
}}
otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config")
if err != nil {
t.Fatalf("select comparison inputs: %v", err)
}
var modelCalls atomic.Int64
client := countingLLMClient{calls: &modelCalls}
endpoint, model := "https://example.invalid/v1", "offline-model"
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
"offline": {Endpoint: &endpoint, Model: &model},
@@ -155,11 +163,42 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
if err != nil {
t.Fatalf("construct engine: %v", err)
}
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("comparison profile", map[string]promptkit.ProfileConfig{
"offline": {Endpoint: &endpoint, Model: &otherModel},
})}}}},
}}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatalf("construct comparison engine: %v", err)
}
source.closed.Store(true)
otherSource.closed.Store(true)
view, err := engine.InspectConfiguration()
if err != nil {
t.Fatalf("inspect configuration: %v", err)
}
otherView, err := otherEngine.InspectConfiguration()
if err != nil {
t.Fatalf("inspect comparison configuration: %v", err)
}
var expected configkit.Difference
for index := 0; index < 3; index++ {
differences, err := view.Diff(otherView)
if err != nil || len(differences) != 1 {
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
}
if differences[0].Path.String() != "/profiles/offline/model" ||
!strings.Contains(differences[0].Left.YAML, model) ||
!strings.Contains(differences[0].Right.YAML, otherModel) {
t.Fatalf("captured difference = %+v", differences[0])
}
if index == 0 {
expected = differences[0]
} else if differences[0].Path.String() != expected.Path.String() ||
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML {
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
}
}
if _, found := view.Explain(configkit.Keys("prompts", "captured", "1", "messages").Index(0).Key("content_file")); !found {
t.Fatal("captured prompt content path was not inspectable")
}
@@ -167,24 +206,32 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") {
t.Fatalf("write captured report = (%q, %v)", report.String(), err)
}
if source.reads.Load() != 0 || modelCalls.Load() != 0 {
t.Fatalf("inspection touched source or provider: reads=%d requests=%d", source.reads.Load(), modelCalls.Load())
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != 0 {
t.Fatalf("inspection or comparison touched source or provider: reads=(%d,%d) requests=%d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
}
const calls = 16
const calls = 24
var wait sync.WaitGroup
errs := make(chan error, calls)
for index := 0; index < calls; index++ {
wait.Add(1)
go func(index int) {
defer wait.Done()
if index%2 == 0 {
switch index % 3 {
case 0:
_, err := engine.InspectConfiguration()
errs <- err
return
case 1:
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
errs <- err
case 2:
differences, err := view.Diff(otherView)
if err == nil && (len(differences) != 1 || differences[0].Path.String() != expected.Path.String() ||
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML) {
err = errors.New("concurrent comparison returned an unexpected difference")
}
errs <- err
}
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
errs <- err
}(index)
}
wait.Wait()
@@ -194,7 +241,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
t.Fatal(err)
}
}
if source.reads.Load() != 0 || modelCalls.Load() != calls/2 {
t.Fatalf("concurrent inspection/execution = source reads %d, model calls %d", source.reads.Load(), modelCalls.Load())
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != calls/3 {
t.Fatalf("concurrent inspection/comparison/execution = source reads (%d,%d), model calls %d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
}
}

View File

@@ -181,6 +181,41 @@ shows backend-completed execution settings for one profile, and `Prepare` or
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
and validation resources. None of those views is a substitute for the others.
To review changes between two independently captured configurations, compare
their inspection views:
```go
earlier, err := earlierEngine.InspectConfiguration()
if err != nil {
return err
}
later, err := candidateEngine.InspectConfiguration()
if err != nil {
return err
}
differences, err := earlier.Diff(later)
if err != nil {
return err
}
for _, difference := range differences {
if difference.HiddenChanged {
// Report a protected change without inferring its value.
}
if difference.Left.Redacted || difference.Right.Redacted {
// Apply the application's disclosure policy before presenting snapshots.
}
}
```
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
and ConfigKit's [v0.7.0 effective-configuration diff guidance](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.7.0/docs/config.md#effective-configuration-diffs)
for the exact comparison contract. The operation compares only frozen effective
configuration values: it does not reread external files or compare their
contents, resolved credential contents, later execution preparation, or
request-specific overrides. An empty diff is not proof of equivalent execution.
Records can contain ordinary application data, so use the application's
disclosure policy when presenting them.
## Inspect A Prompt Before Preparation
Use [`Engine.InspectPrompt`](../../engine.go) to check one configured prompt's
@@ -249,6 +284,41 @@ conditional entries. Do not treat an empty partial inventory as proof that no
input bindings are needed. See the method's [GoDoc](../../requirements.go) for
the exact classifications, supported syntax, ownership, and error contract.
## Validate A Concrete Request
Use [`Engine.ValidateRequest`](../../engine.go) when the application has a
complete `RunRequest` and needs to know whether that concrete invocation passes
preparation:
```go
request := promptkit.RunRequest{
PromptID: "meeting.summary",
Vars: map[string]string{"audience": "project leads"},
Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Synthetic meeting notes"),
},
}
if err := engine.ValidateRequest(ctx, request); err != nil {
return err
}
```
Unlike `InspectRequirements`, this checks supplied values, the selected template
branches, and request bindings through the existing preparation path. It checks
only this invocation, does not aggregate errors, and loads every supplied input
according to ordinary preparation behavior, even if a branch does not use it.
It makes no model call and returns no prepared snapshot, but artifact and schema
resource I/O and its time and memory costs remain. Success does not certify
provider acceptance or later execution success. See the method's GoDoc for its
exact error and context behavior.
Scriptorium can map the returned error to its own diagnostics and exit status.
Its `validate` and `run` commands should construct equivalent requests for
equivalent caller inputs so their preparation behavior agrees. Separate
subprocess calls prepare independently; an opaque `PrepareExecution` handle
cannot be transferred from `validate` to `run`. Use `PrepareExecution` when one
process and engine must execute the exact prepared snapshot.
## Prepare Without Model Execution
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
@@ -269,8 +339,11 @@ prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
The maintained
[offline preparation example](../../examples/go-library/prepare/main.go)
shows a complete runnable setup with a prompt file, in-memory profile, and
inline input. Exact request requirements and prepared-result fields belong to
the [`RunRequest` and `PreparedRun` GoDoc](../../types.go).
inline input. It demonstrates validation and preparation with the same request
to make the two operations concrete; this performs preparation twice and is
illustrative, not a requirement to validate before `Prepare` or `Run`. Exact
request requirements and prepared-result fields belong to the
[`RunRequest` and `PreparedRun` GoDoc](../../types.go).
## Prepare Now And Execute The Same Snapshot Later

View File

@@ -11,7 +11,7 @@ contributor workflow and validation.
| Component | Implemented responsibility | References |
| --- | --- | --- |
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, prompt/profile/requirement inspection and identity listing, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, prompt/profile/requirement inspection and identity listing, concrete-request validation, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
| `examples/go-library/prepare` | Demonstrates offline preparation and explicit configuration inspection with a prompt file, programmatic profile, and inline input. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |

View File

@@ -111,6 +111,12 @@ into its private payload, while deep-copying mutable provider extras and
creating an independent details snapshot. Claim completion or discard clears
the execution payload without removing the details snapshot.
The root facade's `Engine.ValidateRequest` uses the same complete preparation
path as `Prepare` and discards the resulting internal prepared value without
converting it into a public snapshot. It does not add a runner operation or
validator component; request conversion and public error mapping remain at the
facade boundary.
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
out-of-range values fail as invalid requests. Endpoint overrides do not change
the selected backend identity. Non-empty extra-parameter maps replace whole

View File

@@ -21,6 +21,16 @@ profile inheritance. It retains ConfigKit's result privately for explicit
inspection, not for per-run resolution. The runtime/domain packages do not
depend on ConfigKit.
The `ConfigurationInspection.Diff` facade method compares two retained results
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
once to ConfigKit v0.7.0 and returns its public difference records. Comparison
uses only frozen effective values and does not reopen sources, resolve runtime
credentials, or enter the execution path. ConfigKit applies sensitivity from
both results; reports may still contain ordinary application data. See the
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
and ConfigKit's [v0.7.0 comparison contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.7.0/docs/config.md#effective-configuration-diffs).
After prompt materialization and effective-profile snapshot construction, the
adapter captures lexically ordered, identity-only inventories from the final
prompt and indexed-profile maps. Its listing methods return caller-owned copies
@@ -49,9 +59,9 @@ strings, including empty strings. It closes content-file handles before
returning and retains no source capability for public inspection.
Role and cache-control normalization is applied to execution-owned message
copies. The retained ConfigKit configuration keeps effective input message
metadata and its resource origins: recording a sequence-wide normalization in
ConfigKit v0.6.0 would replace origins of unchanged content-file fields.
copies. ConfigKit v0.7.0 retains effective input message metadata and its
resource origins; recording a sequence-wide normalization in the retained
configuration would replace origins of unchanged content-file fields.
The runner selects full definitions from this index for preparation and uses
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the

View File

@@ -26,17 +26,17 @@ License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
## ConfigKit
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.6.0`.
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.7.1`.
Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: LGPL-3.0-or-later
License and notices: [LGPL 3.0](../../LICENSE) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/1a885934c28733d9a5dca261fd6da0e2a4279285/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/1a885934c28733d9a5dca261fd6da0e2a4279285/LICENSE).
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77/LICENSE).
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/1a885934c28733d9a5dca261fd6da0e2a4279285).
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77).
## Promptkit OpenRouter Catalog v3

View File

@@ -74,7 +74,7 @@ For an optional local diagnostic of a committed candidate, run:
```
This is not a required step before publication. The upstream
[candidate-check contract](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.3.0/docs/consumers/check.md)
[candidate-check contract](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/consumers/check.md)
describes its requirements.
Run the single publication command with the chosen version:
@@ -114,7 +114,7 @@ resolved. ReleaseKit reports the attempted version, tag object, and recovery
state, including the captured publication endpoint when available. Use that
endpoint—not merely the current `origin` URL—to query the exact tag; follow the
upstream
[failure-recovery procedure](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.3.0/docs/release.md#resolve-failures):
[failure-recovery procedure](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/release.md#resolve-failures):
```sh
releasekit_endpoint='COPY_CAPTURED_ENDPOINT'

48
docs/releases/v0.14.0.md Normal file
View File

@@ -0,0 +1,48 @@
# Promptkit v0.14.0
This release adds comparison of captured configuration inspections and updates
ConfigKit and the maintainer release tool. The minimum supported Go version
remains 1.26.0.
## Compare Captured Configurations
`ConfigurationInspection.Diff` compares the effective configuration captured by
two engines. Consumers can use it to review a candidate configuration, inspect
an overlay's effect, or present configuration changes to operators.
Comparison delegates to ConfigKit and returns its difference records, including
presence information, paired redaction, and protected-change indicators. It
does not reload sources or contact providers. Source-history differences alone
do not produce value differences.
An empty diff is not proof of equivalent execution: referenced file contents,
resolved credentials, later execution preparation, and request-specific
overrides are outside the comparison. Records can contain ordinary application
data; callers retain responsibility for their disclosure.
See the [configuration comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
and [GoDoc](../../configuration_inspection.go) for usage and the exact contract.
## Dependency And Tooling Updates
- Upgrade ConfigKit from v0.6.0 to v0.7.1, providing the upstream comparison API.
v0.7.1 also updates ConfigKit's maintainer tooling without changing its runtime
behavior from v0.7.0.
- Upgrade the module-selected ReleaseKit tool from v0.3.0 to v0.4.1. Promptkit
retains its existing root-module profile, publication entry point, and release
gates. See the [release procedure](../release.md) for maintainer commands.
## Compatibility And Upgrade
No application-code, configuration, or Go-toolchain migration is required from
v0.13.0. Configuration comparison is an optional API addition; existing
construction, inspection, preparation, and execution workflows remain available.
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.14.0
```
See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility) for the
distinction between the consumer minimum and maintainer toolchain.
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/docs/oss/attribution.md)

34
docs/releases/v0.15.0.md Normal file
View File

@@ -0,0 +1,34 @@
# Promptkit v0.15.0
This release adds error-only validation of a concrete prompt request without
calling a model. The minimum supported Go version remains 1.26.0.
## Validate A Concrete Request
`Engine.ValidateRequest` accepts an ordinary `RunRequest` and returns nil when
it passes preparation, or the existing preparation error. Consumers can use it
for fail-fast checks, including downstream CLI validation commands, without
receiving a prepared snapshot.
The method shares `Prepare`'s request conversion and preparation path. Unlike
requirements inspection, it checks the actual supplied values and bindings.
Artifact and schema resource I/O still occurs; success does not certify provider
acceptance or later execution success. A subsequent run prepares independently.
Validation is optional, not a required step before preparation or execution.
See the [request validation guide](../consumers/pkg-promptkit.md#validate-a-concrete-request)
and [GoDoc](../../engine.go) for usage and the exact contract. Consumers that
need execution tied to a checked snapshot can use the existing
[prepared execution workflow](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later).
## Compatibility And Upgrade
No application-code, configuration, or Go-toolchain migration is required from
v0.14.0. This is an optional API addition; existing inspection, preparation, and
execution workflows remain available.
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.15.0
```
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.15.0/docs/oss/attribution.md)

View File

@@ -346,6 +346,32 @@ func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*Profile
// relationship to ctx.Err is promised. Prepare returns no partial result on
// error. It validates RunRequest.TransientRetries but makes no model attempts.
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
prepared, err := e.prepareRequest(ctx, req)
if err != nil {
return nil, err
}
return fromDomainPreparedRun(prepared), nil
}
// ValidateRequest reports whether req passes the same preparation path used by
// [Engine.Prepare] at the time of this call. A nil error means this concrete
// request passed preparation. It checks only this invocation and returns errors
// as encountered rather than aggregating them. All supplied inputs follow the
// same loading behavior as [Engine.Prepare], including inputs unused by the
// selected template branch. The method does not call an LLM or retain a
// prepared snapshot or executable handle, but preparation may still perform
// resource I/O and incur its associated time and memory costs.
//
// Its errors and context handling follow [Engine.Prepare], including an error
// matching ErrInvalidConfig when the Engine is nil. Success does not guarantee
// provider acceptance or later execution success. Use [Engine.PrepareExecution]
// when execution must use the snapshot produced by preparation.
func (e *Engine) ValidateRequest(ctx context.Context, req RunRequest) error {
_, err := e.prepareRequest(ctx, req)
return err
}
func (e *Engine) prepareRequest(ctx context.Context, req RunRequest) (*domain.PreparedRun, error) {
ctx = observation.WithoutExecutionState(ctx)
if e == nil || e.internal == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
@@ -360,7 +386,7 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
if err != nil {
return nil, mapPublicError(err)
}
return fromDomainPreparedRun(prepared), nil
return prepared, nil
}
// PrepareExecution completely prepares a prompt request without calling the

View File

@@ -48,12 +48,16 @@ func main() {
exit(fmt.Errorf("required note input was not discovered before preparation"))
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
request := promptkit.RunRequest{
PromptID: "example.prepare",
Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Ada finished the migration review."),
},
})
}
if err := engine.ValidateRequest(context.Background(), request); err != nil {
exit(err)
}
prepared, err := engine.Prepare(context.Background(), request)
if err != nil {
exit(err)
}

4
go.mod
View File

@@ -3,14 +3,14 @@ module gitea.maximumdirect.net/eric/promptkit
go 1.26.0
require (
gitea.maximumdirect.net/eric/ConfigKit v0.6.0
gitea.maximumdirect.net/eric/ConfigKit v0.7.1
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
)
require (
gitea.maximumdirect.net/eric/ReleaseKit v0.3.0 // indirect
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 // indirect
github.com/yuin/goldmark/v2 v2.1.6 // indirect
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
golang.org/x/mod v0.40.0 // indirect

8
go.sum
View File

@@ -1,7 +1,7 @@
gitea.maximumdirect.net/eric/ConfigKit v0.6.0 h1:KdsCVO5kKctdAUQJ6X7dJ/GemlVjCYCcn/xZrMAE/aw=
gitea.maximumdirect.net/eric/ConfigKit v0.6.0/go.mod h1:xN4dRdDGIW4yh/e9GghDF/hSIzc8/77blRhwyf90KSA=
gitea.maximumdirect.net/eric/ReleaseKit v0.3.0 h1:aTMn+50MEpHpHI2RoCCmJYsy7Ohrpa+RSva63Kq0/tU=
gitea.maximumdirect.net/eric/ReleaseKit v0.3.0/go.mod h1:LVodqNQ3OIKv9NBq+fFeyMByd1lHY25YOmmpah/NaNI=
gitea.maximumdirect.net/eric/ConfigKit v0.7.1 h1:q3HrX4WWCPt1WfMydqFEaAmCf9s6Dq+vknJj0C/mdcI=
gitea.maximumdirect.net/eric/ConfigKit v0.7.1/go.mod h1:C82sbyKHYMV1OwaD2GJanFrwTfcZfjlW2JsGze5b0kA=
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 h1:Qn+mdebYOzCVCEXgCbuGSO/WHmJtItwaQ8e3XwY3qtI=
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1/go.mod h1:LVodqNQ3OIKv9NBq+fFeyMByd1lHY25YOmmpah/NaNI=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=

View File

@@ -16,6 +16,15 @@ type Inspection interface {
Explain(configkit.Path) (configkit.Explanation, bool)
WriteEffective(io.Writer) error
WriteExplanation(io.Writer, configkit.Path) error
capturedResult() *configkit.Result[document]
}
type inspectionResult struct {
*configkit.Result[document]
}
func (inspection inspectionResult) capturedResult() *configkit.Result[document] {
return inspection.Result
}
// Inspection returns only the retained result's inspection operations.
@@ -23,7 +32,19 @@ func (state *loaded) Inspection() Inspection {
if state == nil {
return nil
}
return state.result
return inspectionResult{Result: state.result}
}
// Diff compares two captured ConfigKit results without reconstructing them.
func Diff(left, right Inspection) ([]configkit.Difference, error) {
var leftResult, rightResult *configkit.Result[document]
if left != nil {
leftResult = left.capturedResult()
}
if right != nil {
rightResult = right.capturedResult()
}
return configkit.Diff(leftResult, rightResult)
}
func secretProviderPath(path configkit.Path) bool {

View File

@@ -0,0 +1,218 @@
package promptkit_test
import (
"context"
"errors"
"path/filepath"
"sync/atomic"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestValidateRequestMatchesPrepareContract(t *testing.T) {
var modelCalls atomic.Int64
profiles := map[string]any{
"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
}
newEngine := func(t *testing.T, definition map[string]any) *promptkit.Engine {
t.Helper()
return requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), profiles,
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
}
t.Run("successful request", func(t *testing.T) {
engine := newEngine(t, contractPrompt("offline", "hello", nil, "", noValidationOutput))
assertValidateRequestParity(t, engine, promptkit.RunRequest{PromptID: "synthetic"}, nil)
})
t.Run("nil engine", func(t *testing.T) {
var engine *promptkit.Engine
request := promptkit.RunRequest{PromptID: "synthetic"}
assertValidateRequestParity(t, engine, request, promptkit.ErrInvalidConfig)
})
t.Run("invalid appended message", func(t *testing.T) {
engine := newEngine(t, contractPrompt("offline", "hello", nil, "", noValidationOutput))
request := promptkit.RunRequest{
PromptID: "synthetic",
AppendedMessages: []promptkit.RenderedMessage{{
Role: "unsupported-role", Content: "not sent to a model",
}},
}
assertValidateRequestParity(t, engine, request, promptkit.ErrInvalidRequest)
})
t.Run("missing required input", func(t *testing.T) {
definition := contractPrompt("offline", `{{input "required"}}`, []any{
map[string]any{"name": "required", "required": true},
}, "", noValidationOutput)
engine := newEngine(t, definition)
assertValidateRequestParity(t, engine, promptkit.RunRequest{PromptID: "synthetic"}, promptkit.ErrPromptRender)
})
if got := modelCalls.Load(); got != 0 {
t.Fatalf("preparation operations made %d model calls, want none", got)
}
}
func assertValidateRequestParity(t *testing.T, engine *promptkit.Engine, request promptkit.RunRequest, want error) *promptkit.PreparedRun {
t.Helper()
prepared, prepareErr := engine.Prepare(context.Background(), request)
validateErr := engine.ValidateRequest(context.Background(), request)
if want == nil {
if prepareErr != nil || prepared == nil {
t.Fatalf("Prepare() = (%+v, %v), want a prepared result", prepared, prepareErr)
}
if validateErr != nil {
t.Fatalf("ValidateRequest() error = %v, want nil", validateErr)
}
return prepared
}
if prepared != nil || !errors.Is(prepareErr, want) {
t.Fatalf("Prepare() = (%+v, %v), want error matching %v", prepared, prepareErr, want)
}
if !errors.Is(validateErr, want) {
t.Fatalf("ValidateRequest() error = %v, want error matching %v", validateErr, want)
}
return nil
}
func TestValidateRequestUsesSelectedTemplateBranchesAndVariablePresence(t *testing.T) {
var modelCalls atomic.Int64
profiles := map[string]any{
"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
}
branchContent := `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`
optionalDefinition := contractPrompt("offline", branchContent, []any{
map[string]any{"name": "details", "required": false},
}, "", noValidationOutput)
requiredDefinition := contractPrompt("offline", branchContent, []any{
map[string]any{"name": "details", "required": true},
}, "", noValidationOutput)
engine := requirementsEngine(t, map[string]any{
"optional": oneVersionPrompt("optional", "v1", optionalDefinition)["optional"],
"required": oneVersionPrompt("required", "v1", requiredDefinition)["required"],
"direct": oneVersionPrompt("direct", "v1", contractPrompt("offline", `{{.audience}}`, nil, "", noValidationOutput))["direct"],
"index": oneVersionPrompt("index", "v1", contractPrompt("offline", `{{index . "audience"}}`, nil, "", noValidationOutput))["index"],
}, profiles, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
cases := []struct {
name string
request promptkit.RunRequest
want error
}{
{name: "brief branch omits optional input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "brief"}}},
{name: "detailed branch needs optional input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "detailed"}}, want: promptkit.ErrPromptRender},
{name: "detailed branch accepts inline input", request: promptkit.RunRequest{PromptID: "optional", Vars: map[string]string{"mode": "detailed"}, Inputs: map[string]promptkit.ArtifactRef{"details": promptkit.Inline("Long form details")}}},
{name: "declared required input is needed on brief branch", request: promptkit.RunRequest{PromptID: "required", Vars: map[string]string{"mode": "brief"}}, want: promptkit.ErrPromptRender},
{name: "evaluated variable needs a supplied key", request: promptkit.RunRequest{PromptID: "direct"}, want: promptkit.ErrPromptRender},
{name: "empty value and unrelated variable are accepted", request: promptkit.RunRequest{PromptID: "direct", Vars: map[string]string{"audience": "", "unrelated": "accepted"}}},
{name: "missing index lookup remains optional", request: promptkit.RunRequest{PromptID: "index"}},
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
assertValidateRequestParity(t, engine, test.request, test.want)
})
}
if got := modelCalls.Load(); got != 0 {
t.Fatalf("preparation operations made %d model calls, want none", got)
}
}
type canceledValidationArtifactReader struct{}
func (canceledValidationArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
return nil, context.Canceled
}
func TestValidateRequestPreservesResourceAndSelectionFailures(t *testing.T) {
var modelCalls atomic.Int64
profiles := map[string]any{
"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
}
branch := `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`
definition := contractPrompt("offline", branch, []any{
map[string]any{"name": "details", "required": false},
}, "", noValidationOutput)
profilePrompt := contractPrompt("offline", "hello", nil, "", noValidationOutput)
schemaPrompt := contractPrompt("offline", "{}", nil, "", map[string]any{
"format": "json", "validation_mode": "json_schema", "schema_path": "missing-schema.json",
})
prompts := map[string]any{
"branch": oneVersionPrompt("branch", "v1", definition)["branch"],
"profile": oneVersionPrompt("profile", "v1", profilePrompt)["profile"],
"schema": oneVersionPrompt("schema", "v1", schemaPrompt)["schema"],
}
engine := requirementsEngine(t, prompts, profiles,
promptkit.WithSchemaFS(fstest.MapFS{}, "."),
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}),
)
t.Run("unselected supplied input is still loaded", func(t *testing.T) {
missingPath := filepath.Join(t.TempDir(), "missing-input.txt")
request := promptkit.RunRequest{
PromptID: "branch",
Vars: map[string]string{"mode": "brief"},
Inputs: map[string]promptkit.ArtifactRef{"details": promptkit.File(missingPath)},
}
assertValidateRequestParity(t, engine, request, promptkit.ErrArtifactLoad)
})
t.Run("missing selected profile", func(t *testing.T) {
request := promptkit.RunRequest{PromptID: "profile", ProfileID: "not-configured"}
assertValidateRequestParity(t, engine, request, promptkit.ErrProfileNotFound)
})
t.Run("unavailable selected schema", func(t *testing.T) {
request := promptkit.RunRequest{PromptID: "schema"}
assertValidateRequestParity(t, engine, request, promptkit.ErrValidation)
})
if got := modelCalls.Load(); got != 0 {
t.Fatalf("preparation operations made %d model calls, want none", got)
}
}
func TestValidateRequestPreservesCanceledLoaderCategory(t *testing.T) {
var modelCalls atomic.Int64
definition := contractPrompt("offline", `{{if eq .mode "detailed"}}{{input "details"}}{{end}}`, []any{
map[string]any{"name": "details", "required": false},
}, "", noValidationOutput)
engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{
"offline": map[string]any{"endpoint": "http://example.test/v1", "model": "offline-model"},
}, promptkit.WithArtifactReader(canceledValidationArtifactReader{}),
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
request := promptkit.RunRequest{
PromptID: "synthetic",
Vars: map[string]string{"mode": "brief"},
Inputs: map[string]promptkit.ArtifactRef{"details": promptkit.File("controlled-canceled-loader")},
}
assertValidateRequestParity(t, engine, request, promptkit.ErrArtifactLoad)
if got := modelCalls.Load(); got != 0 {
t.Fatalf("preparation operations made %d model calls, want none", got)
}
}
func TestValidateRequestForwardsProfileOverridesAndRequestSettings(t *testing.T) {
var modelCalls atomic.Int64
definition := contractPrompt("partial", "base message", nil, "", noValidationOutput)
engine := requirementsEngine(t, oneVersionPrompt("synthetic", "v1", definition), map[string]any{
"partial": map[string]any{"model": "partial-model"},
}, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
request := promptkit.RunRequest{
PromptID: "synthetic",
SessionID: "request-session",
Execution: &promptkit.ExecutionTargetOverride{Endpoint: "http://override.example/v1"},
AppendedMessages: []promptkit.RenderedMessage{{
Role: promptkit.RoleAssistant, Content: "caller suffix",
}},
}
prepared := assertValidateRequestParity(t, engine, request, nil)
if prepared.SelectedProfileID != "partial" || prepared.EffectiveModelParams.Model != "partial-model" ||
prepared.EffectiveModelParams.Endpoint != "http://override.example/v1" || prepared.SessionID != "request-session" ||
len(prepared.Messages) != 2 || prepared.Messages[1].Content != "caller suffix" {
t.Fatalf("preparation fixture did not exercise profile and request settings: %+v", prepared)
}
if got := modelCalls.Load(); got != 0 {
t.Fatalf("preparation operations made %d model calls, want none", got)
}
}