85 Commits

Author SHA1 Message Date
143c6e9b3a Prepare v0.14.0 release notes 2026-10-11 01:19:17 +00:00
48e4784fcd Retire completed configuration comparison roadmaps 2026-10-11 01:18:30 +00:00
78af5069c1 Update ConfigKit to v0.7.1 2026-10-11 01:17:10 +00:00
18963266d7 Upgrade ReleaseKit to v0.4.1 2026-10-11 01:11:19 +00:00
d039a019cb Document captured configuration comparison 2026-10-09 16:06:09 +00:00
d1c0281053 Verify captured configuration diff concurrency 2026-10-09 16:01:24 +00:00
870e4ca4e6 Verify configuration diff disclosure semantics 2026-10-09 15:59:58 +00:00
fd82524977 Add captured configuration comparison 2026-10-09 15:58:04 +00:00
3f76c75919 Adopt ConfigKit v0.7.0 2026-10-09 15:56:10 +00:00
fe764f72d8 Plan captured configuration comparison in focused stages 2026-10-09 15:46:41 +00:00
26ab566f38 Define captured configuration comparison roadmap 2026-10-09 15:41:53 +00:00
18ca21fa45 Retire completed roadmap documentation 2026-10-09 04:50:10 +00:00
e32fdc22f4 Document the v0.13.0 release 2026-10-09 04:49:55 +00:00
b02b802427 Upgrade ConfigKit to v0.6.0 2026-10-09 04:45:10 +00:00
301192b974 Document profile and prompt identity listings 2026-10-09 04:34:45 +00:00
f2aeb5eb30 Verify effective listing ownership and concurrency 2026-10-09 04:31:31 +00:00
a27ee8f7da Expose profile and prompt identity listings 2026-10-09 04:28:23 +00:00
4d1895c223 Capture profile and prompt listing identities 2026-10-09 04:27:00 +00:00
d7fe70520a Plan profile and prompt listing in focused stages 2026-10-09 04:23:27 +00:00
7325a34c07 Define profile and prompt listing 2026-10-09 04:19:09 +00:00
03c44923c8 Upgrade the release tooling to ReleaseKit v0.3.0 2026-10-09 03:48:52 +00:00
dc0adf5751 Align the release policy with Go compatibility guidance 2026-10-09 03:45:39 +00:00
eae21797f7 Describe the ReleaseKit upgrade scope 2026-10-09 03:41:56 +00:00
ea6c7d5f0f Release unused file buffers for empty artifacts 2026-10-09 01:52:06 +00:00
389ef26db3 Record the remaining file buffer cleanup issue 2026-10-09 01:38:14 +00:00
8a5c0de162 Finalize computational efficiency evidence 2026-10-09 01:16:14 +00:00
80aecb646d Reuse prepared execution snapshot ownership 2026-10-09 01:08:46 +00:00
b7b4000aab Record prepared snapshot ownership evidence 2026-10-09 00:57:23 +00:00
11ecb923ba Add prepared snapshot ownership benchmarks 2026-10-09 00:56:29 +00:00
f46d5a4591 Record assembled workflow benchmark results 2026-10-09 00:50:41 +00:00
743e78d514 Add assembled workflow benchmark baseline 2026-10-09 00:49:46 +00:00
3743221aa5 Correct roadmap progress summary 2026-10-09 00:46:00 +00:00
a8863d49d0 Record prompt inspection metadata results 2026-10-09 00:45:54 +00:00
c5f733af45 Select prompt metadata for inspection 2026-10-09 00:45:25 +00:00
c09d2c6e20 Record requirement inspection identity results 2026-10-09 00:38:19 +00:00
6c5dec761c Inspect requirements by profile identity 2026-10-09 00:38:01 +00:00
74b91fae99 Record lightweight inspection lookup results 2026-10-09 00:35:59 +00:00
b7ff37118c Add lightweight configuration lookups 2026-10-09 00:35:30 +00:00
ef012ea1e5 Record requirement inspection benchmarks 2026-10-09 00:31:52 +00:00
04d5d695e0 Add requirement inspection benchmarks 2026-10-09 00:30:51 +00:00
78bff91395 Encode generation payloads once on success 2026-10-09 00:27:24 +00:00
c63ca5352f Record request preparation baseline 2026-10-09 00:25:39 +00:00
3f157150e4 Add request preparation baseline and boundary tests 2026-10-09 00:25:21 +00:00
f1e1a45a05 Reserve capacity for regular file artifacts 2026-10-09 00:23:27 +00:00
996295d5c1 Add artifact file-read benchmarks 2026-10-09 00:20:15 +00:00
58a5bbe790 Use local builders for rendered templates 2026-10-09 00:18:39 +00:00
dd89be3b03 Record basic validation allocation savings 2026-10-09 00:15:06 +00:00
ed7c27324d Avoid copying output during basic validation 2026-10-09 00:13:23 +00:00
05fe13d80b Record validation and rendering baselines 2026-10-09 00:11:00 +00:00
0eb009ad5a Add validation and rendering benchmarks 2026-10-09 00:09:15 +00:00
3fde1f6cbb Plan the performance improvements in focused stages 2026-10-09 00:02:21 +00:00
481e2d494c Mark the ConfigKit performance request complete 2026-10-08 23:50:38 +00:00
eb830bb613 Refine performance roadmap after review 2026-10-08 23:44:12 +00:00
dd6d4591ae Plan computational efficiency improvements 2026-10-08 22:02:18 +00:00
de6443f3ae Align completed ReleaseKit documentation 2026-10-08 21:21:37 +00:00
5a63b649f9 Clarify inspected input identifier disclosure 2026-10-08 21:19:24 +00:00
c7e22c0dbe Avoid rescanning unsupported input names 2026-10-08 21:18:45 +00:00
e69a847cf2 Preserve caller scope during requirement inspection 2026-10-08 21:15:56 +00:00
21815ad7c0 Record requirement inspection review findings 2026-10-08 20:59:21 +00:00
a051edd82d Document prompt requirement inspection 2026-10-08 20:48:10 +00:00
ee0ad99e17 Test public prompt requirement inspection 2026-10-08 20:43:48 +00:00
bc5a2da0f2 Expose prompt requirement inspection 2026-10-08 20:40:16 +00:00
089e824a92 Resolve direct prompt input names 2026-10-08 20:36:17 +00:00
93e8874813 Classify simple prompt requirement branches 2026-10-08 20:31:45 +00:00
cc0b6183bf Enumerate prompt input and variable dependencies 2026-10-08 20:25:17 +00:00
bf472dad8a Capture prompt requirement metadata and templates 2026-10-08 20:17:34 +00:00
a6e0ffea8f Simplify requirement inspection plan 2026-10-08 19:58:57 +00:00
e459c7d54b Settle requirement inspection scope 2026-10-08 17:54:20 +00:00
71f6683fda Plan effective requirement inspection implementation 2026-10-08 17:28:04 +00:00
7827ca043b Plan effective prompt requirement inspection 2026-10-08 16:57:58 +00:00
b595dca8c7 Record ReleaseKit adoption review findings 2026-10-08 16:47:23 +00:00
d5f34642d7 Verify ReleaseKit adoption on final source snapshot 2026-10-08 16:42:21 +00:00
b1866946d1 Delegate Promptkit releases to ReleaseKit 2026-10-08 16:40:34 +00:00
05082c6fdf Test ReleaseKit validation failure handling 2026-10-08 16:35:39 +00:00
51b072965c Add successful ReleaseKit publication fixture 2026-10-08 16:33:43 +00:00
d5940856be Add native ReleaseKit bootstrap 2026-10-08 16:23:26 +00:00
0d163f8165 Extract example checks into release profile 2026-10-08 16:20:51 +00:00
1889915a81 Separate the Go minimum from the release toolchain 2026-10-08 16:14:50 +00:00
3ca01ae0b0 Add ReleaseKit as a release tool 2026-10-08 16:10:23 +00:00
6e283d16c3 Plan the ReleaseKit adoption 2026-10-08 13:56:36 +00:00
41b2021d8c Resolve the ReleaseKit adoption decisions 2026-10-08 13:36:01 +00:00
3add7024db Align release preparation and tooling with the current Go version 2026-10-08 13:35:06 +00:00
a148a2b1f4 Define the ReleaseKit adoption roadmap 2026-10-08 13:13:10 +00:00
7310ef34a4 Assess ReleaseKit adoption and shared release validation 2026-10-07 16:08:26 +00:00
4b8169da28 Prepare Promptkit v0.12.0 release 2026-10-03 01:26:43 +00:00
77 changed files with 7405 additions and 2099 deletions

4
.gitignore vendored
View File

@@ -22,6 +22,9 @@
go.work
go.work.sum
# Local ReleaseKit executable
.releasekit/
# env file
.env
@@ -67,4 +70,3 @@ Icon
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk

View File

@@ -37,6 +37,10 @@ boundary and constraints that framework work must preserve.
## Release Guidance
Consumers upgrading to `v0.12.0` should read the
[v0.12.0 release notes](docs/releases/v0.12.0.md) for the intentional ConfigKit
configuration and public construction API migration.
The [v0.11.2 release notes](docs/releases/v0.11.2.md) cover policy alignment
and copied-policy attribution updates.

View File

@@ -0,0 +1,237 @@
package promptkit_test
import (
"context"
"encoding/json"
"fmt"
"strings"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
var (
assembledPreparedSink *promptkit.PreparedRun
assembledResultSink *promptkit.RunResult
assembledDetailsSink promptkit.PreparedRun
assembledHandleSink *promptkit.PreparedExecution
)
type assembledWorkflowClient struct{ content string }
func (client assembledWorkflowClient) Generate(
context.Context,
promptkit.GenerateRequest,
) (*promptkit.GenerateResponse, error) {
return &promptkit.GenerateResponse{Content: client.content}, nil
}
type assembledWorkflowFixture struct {
engine *promptkit.Engine
request promptkit.RunRequest
expectedMessage string
response string
}
func assembledWorkflowSchema(b *testing.B) []byte {
b.Helper()
properties := make(map[string]any, 256)
for index := range 256 {
properties[fmt.Sprintf("property-%03d", index)] = map[string]any{"type": "string"}
}
data, err := json.Marshal(map[string]any{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": properties,
"additionalProperties": false,
})
if err != nil {
b.Fatal(err)
}
return data
}
func assembledWorkflowExtras(entries int) map[string]any {
providers := make(map[string]any, entries)
for index := range entries {
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
"enabled": true,
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
"limits": map[string]any{"burst": index + 1, "period": "minute"},
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
}
}
return map[string]any{"provider_options": providers}
}
func newAssembledWorkflowFixture(b *testing.B, name string) assembledWorkflowFixture {
b.Helper()
message := "Hello {{.name}}"
response := "ok"
output := map[string]any{"format": "text", "validation_mode": "none"}
profile := map[string]any{"endpoint": "http://example.test/v1", "model": "benchmark-model"}
options := []promptkit.Option{promptkit.WithLLMClient(assembledWorkflowClient{content: response})}
var expectedMessage string
switch name {
case "small":
expectedMessage = "Hello Ada"
case "large_message":
prefix := strings.Repeat("x", 256*1024)
message = prefix + "{{.name}}"
expectedMessage = prefix + "Ada"
case "mutable_payload":
message = `{"subject":"{{.name}}"}`
expectedMessage = `{"subject":"Ada"}`
response = "{}"
profile["extra_params"] = assembledWorkflowExtras(64)
output = map[string]any{
"format": "json", "validation_mode": "json_schema", "schema_path": "workflow-schema.json",
}
options[0] = promptkit.WithLLMClient(assembledWorkflowClient{content: response})
options = append(options, promptkit.WithSchemaFS(fstest.MapFS{
"workflow-schema.json": &fstest.MapFile{Data: assembledWorkflowSchema(b)},
}, "."))
default:
b.Fatalf("unknown assembled workflow fixture %q", name)
}
config := map[string]any{
"profiles": map[string]any{"profile": profile},
"prompts": map[string]any{"workflow": map[string]any{"v1": map[string]any{
"default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": message}},
"output": output,
}}},
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("assembled workflow benchmark", config),
}}}, options...)
if err != nil {
b.Fatal(err)
}
request := promptkit.RunRequest{
PromptID: "workflow", PromptVersion: "v1", Vars: map[string]string{"name": "Ada"},
}
return assembledWorkflowFixture{
engine: engine, request: request, expectedMessage: expectedMessage, response: response,
}
}
func verifyAssembledWorkflowFixture(b *testing.B, fixture assembledWorkflowFixture) {
b.Helper()
ctx := context.Background()
prepared, err := fixture.engine.Prepare(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
if prepared.PromptID != "workflow" || prepared.PromptVersion != "v1" || len(prepared.Messages) != 1 ||
prepared.Messages[0].Content != fixture.expectedMessage {
b.Fatalf("prepared fixture output is invalid: %+v", prepared)
}
handle, err := fixture.engine.PrepareExecution(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
details := handle.Details()
if details.PromptID != "workflow" || len(details.Messages) != 1 || details.Messages[0].Content != fixture.expectedMessage {
handle.Discard()
b.Fatalf("prepared execution details are invalid: %+v", details)
}
handle.Discard()
result, err := fixture.engine.Run(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
if result.RawOutput != fixture.response {
b.Fatalf("run output=%q, want %q", result.RawOutput, fixture.response)
}
if fixture.response == "{}" && result.Validation.Status != promptkit.ValidationPassed {
b.Fatalf("structured response validation=%+v, want passed", result.Validation)
}
}
func BenchmarkEnginePreparation(b *testing.B) {
for _, name := range []string{"small", "large_message", "mutable_payload"} {
b.Run(name, func(b *testing.B) {
fixture := newAssembledWorkflowFixture(b, name)
verifyAssembledWorkflowFixture(b, fixture)
ctx := context.Background()
b.Run("prepare", func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
prepared, err := fixture.engine.Prepare(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
assembledPreparedSink = prepared
}
b.StopTimer()
assembledPreparedSink = nil
})
b.Run("prepared_execution_creation_and_discard", func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
handle, err := fixture.engine.PrepareExecution(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
handle.Discard()
assembledHandleSink = handle
}
b.StopTimer()
assembledHandleSink = nil
})
})
}
}
func BenchmarkEngineRun(b *testing.B) {
for _, name := range []string{"small", "large_message", "mutable_payload"} {
b.Run(name, func(b *testing.B) {
fixture := newAssembledWorkflowFixture(b, name)
verifyAssembledWorkflowFixture(b, fixture)
ctx := context.Background()
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
result, err := fixture.engine.Run(ctx, fixture.request)
if err != nil {
b.Fatal(err)
}
assembledResultSink = result
}
b.StopTimer()
assembledResultSink = nil
})
}
}
func BenchmarkPreparedExecutionDetails(b *testing.B) {
for _, name := range []string{"small", "large_message", "mutable_payload"} {
b.Run(name, func(b *testing.B) {
fixture := newAssembledWorkflowFixture(b, name)
verifyAssembledWorkflowFixture(b, fixture)
handle, err := fixture.engine.PrepareExecution(context.Background(), fixture.request)
if err != nil {
b.Fatal(err)
}
defer handle.Discard()
initial := handle.Details()
if len(initial.Messages) != 1 || initial.Messages[0].Content != fixture.expectedMessage {
b.Fatalf("prepared details fixture is invalid: %+v", initial)
}
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
assembledDetailsSink = handle.Details()
}
b.StopTimer()
assembledDetailsSink = promptkit.PreparedRun{}
})
}
}

View File

@@ -0,0 +1,243 @@
package promptkit_test
import (
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestConfigurationInspectionDiffsCapturedValues(t *testing.T) {
left := comparisonInspection(t, "earlier-model")
right := comparisonInspection(t, "later-model")
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare captured configurations: %v", err)
}
if len(differences) != 1 {
t.Fatalf("difference count = %d, want 1: %+v", len(differences), differences)
}
difference := differences[0]
if got, want := difference.Path.String(), "/profiles/offline/model"; got != want {
t.Fatalf("difference path = %q, want %q", got, want)
}
if !strings.Contains(difference.Left.YAML, "earlier-model") || !strings.Contains(difference.Right.YAML, "later-model") {
t.Fatalf("difference snapshots = (%q, %q)", difference.Left.YAML, difference.Right.YAML)
}
reversed, err := right.Diff(left)
if err != nil {
t.Fatalf("compare reversed configurations: %v", err)
}
if len(reversed) != 1 || reversed[0].Path.String() != difference.Path.String() ||
!strings.Contains(reversed[0].Left.YAML, "later-model") ||
!strings.Contains(reversed[0].Right.YAML, "earlier-model") {
t.Fatalf("reversed difference = %+v", reversed)
}
equal := comparisonInspection(t, "earlier-model")
if equal, err := left.Diff(equal); err != nil || len(equal) != 0 {
t.Fatalf("compare equal configurations = (%+v, %v), want no differences", equal, err)
}
if equal, err := left.Diff(left); err != nil || len(equal) != 0 {
t.Fatalf("compare view with itself = (%+v, %v), want no differences", equal, err)
}
}
func TestConfigurationInspectionDiffRejectsZeroViews(t *testing.T) {
valid := comparisonInspection(t, "current-model")
var zero promptkit.ConfigurationInspection
for _, comparison := range []struct {
name string
left, right promptkit.ConfigurationInspection
}{
{name: "zero left", left: zero, right: valid},
{name: "zero right", left: valid, right: zero},
{name: "both zero", left: zero, right: zero},
} {
t.Run(comparison.name, func(t *testing.T) {
differences, err := comparison.left.Diff(comparison.right)
if !errors.Is(err, promptkit.ErrInvalidConfig) || len(differences) != 0 {
t.Fatalf("Diff() = (%+v, %v), want ErrInvalidConfig and no records", differences, err)
}
})
}
}
func TestConfigurationInspectionDiffIgnoresSourceHistory(t *testing.T) {
left := comparisonInspectionFrom(t, "earlier source", "same-model", nil)
right := comparisonInspectionFrom(t, "later source", "same-model", nil)
if !hasSourceLabel(left, "earlier source") || !hasSourceLabel(right, "later source") {
t.Fatal("comparison fixtures did not retain their distinct source labels")
}
differences, err := left.Diff(right)
if err != nil || len(differences) != 0 {
t.Fatalf("compare equal effective values with different sources = (%+v, %v), want no differences", differences, err)
}
}
func TestConfigurationInspectionDiffReportsPresenceAndSequences(t *testing.T) {
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
"sequence": []any{"earlier", "unchanged"},
})
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
"nullable": nil,
"sequence": []any{"later", "unchanged"},
})
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare presence and sequence values: %v", err)
}
if len(differences) != 2 {
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
}
byPath := differencesByPath(differences)
nullable, ok := byPath["/backends/local/extra_params/nullable"]
if !ok || nullable.Left.Present || !nullable.Right.Present || strings.TrimSpace(nullable.Right.YAML) != "null" {
t.Fatalf("absent-to-null difference = (%+v, %t)", nullable, ok)
}
sequence, ok := byPath["/backends/local/extra_params/sequence"]
if !ok || !sequence.Left.Present || !sequence.Right.Present ||
!strings.Contains(sequence.Left.YAML, "earlier") || !strings.Contains(sequence.Right.YAML, "later") {
t.Fatalf("atomic sequence difference = (%+v, %t)", sequence, ok)
}
}
func TestConfigurationInspectionDiffProtectsSensitiveValuesAndReturnedRecords(t *testing.T) {
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
"token": "left-secret-sentinel",
"visible": "before-value",
})
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
"token": "right-secret-sentinel",
"visible": "after-value",
})
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare sensitive values: %v", err)
}
if len(differences) != 2 {
t.Fatalf("difference count = %d, want 2: %+v", len(differences), differences)
}
byPath := differencesByPath(differences)
secret, ok := byPath["/backends/local/extra_params/token"]
if !ok || !secret.Left.Redacted || !secret.Right.Redacted || !secret.HiddenChanged {
t.Fatalf("sensitive difference = (%+v, %t)", secret, ok)
}
if strings.Contains(secret.Left.YAML, "left-secret-sentinel") || strings.Contains(secret.Left.YAML, "right-secret-sentinel") ||
strings.Contains(secret.Right.YAML, "left-secret-sentinel") || strings.Contains(secret.Right.YAML, "right-secret-sentinel") {
t.Fatalf("sensitive snapshots disclosed a value: %+v", secret)
}
visible, ok := byPath["/backends/local/extra_params/visible"]
if !ok || visible.Left.Redacted || visible.Right.Redacted || visible.HiddenChanged ||
!strings.Contains(visible.Left.YAML, "before-value") || !strings.Contains(visible.Right.YAML, "after-value") {
t.Fatalf("public sibling difference = (%+v, %t)", visible, ok)
}
differences[0].Path = configkit.Keys("caller", "mutation")
differences[0].Left.YAML = "caller mutation"
differences[0].Right.YAML = "caller mutation"
again, err := left.Diff(right)
if err != nil {
t.Fatalf("repeat comparison after mutating returned records: %v", err)
}
if len(again) != 2 {
t.Fatalf("repeat difference count = %d, want 2: %+v", len(again), again)
}
againByPath := differencesByPath(again)
if _, exists := againByPath["/caller/mutation"]; exists {
t.Fatal("mutating returned path reached retained inspection")
}
if _, exists := againByPath["/backends/local/extra_params/token"]; !exists {
t.Fatalf("original sensitive path was lost after mutation: %+v", again)
}
if _, exists := againByPath["/backends/local/extra_params/visible"]; !exists {
t.Fatalf("original public path was lost after mutation: %+v", again)
}
for _, difference := range again {
if difference.Left.YAML == "caller mutation" || difference.Right.YAML == "caller mutation" {
t.Fatalf("mutating returned snapshots reached retained inspection: %+v", difference)
}
}
}
func TestConfigurationInspectionDiffMasksSensitiveShapeReplacement(t *testing.T) {
left := comparisonInspectionFrom(t, "before", "same-model", map[string]any{
"details": map[string]any{
"token": "nested-secret-sentinel",
"label": "public detail",
},
})
right := comparisonInspectionFrom(t, "after", "same-model", map[string]any{
"details": "replacement",
})
differences, err := left.Diff(right)
if err != nil {
t.Fatalf("compare sensitive shape replacement: %v", err)
}
if len(differences) != 1 || differences[0].Path.String() != "/backends/local/extra_params/details" {
t.Fatalf("shape replacement differences = %+v", differences)
}
difference := differences[0]
if !difference.Left.Redacted || !difference.Right.Redacted || !difference.HiddenChanged {
t.Fatalf("shape replacement masking = %+v", difference)
}
if strings.Contains(difference.Left.YAML, "nested-secret-sentinel") || strings.Contains(difference.Right.YAML, "nested-secret-sentinel") {
t.Fatalf("shape replacement disclosed a sensitive value: %+v", difference)
}
}
func comparisonInspection(t *testing.T, model string) promptkit.ConfigurationInspection {
t.Helper()
return comparisonInspectionFrom(t, "comparison fixture", model, nil)
}
func comparisonInspectionFrom(t *testing.T, source, model string, extraParams map[string]any) promptkit.ConfigurationInspection {
t.Helper()
backend := map[string]any{"endpoint": "https://example.invalid/v1"}
if extraParams != nil {
backend["extra_params"] = extraParams
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: configkit.Mapping(source, map[string]any{
"backends": map[string]any{"local": backend},
"profiles": map[string]any{"offline": map[string]any{
"backend": "local",
"model": model,
}},
})}}}},
}})
if err != nil {
t.Fatalf("construct comparison engine: %v", err)
}
inspection, err := engine.InspectConfiguration()
if err != nil {
t.Fatalf("inspect comparison engine: %v", err)
}
return inspection
}
func differencesByPath(differences []configkit.Difference) map[string]configkit.Difference {
byPath := make(map[string]configkit.Difference, len(differences))
for _, difference := range differences {
byPath[difference.Path.String()] = difference
}
return byPath
}
func hasSourceLabel(view promptkit.ConfigurationInspection, label string) bool {
for _, source := range view.Sources() {
if source.Label == label {
return true
}
}
return false
}

View File

@@ -27,6 +27,23 @@ func (e *Engine) InspectConfiguration() (ConfigurationInspection, error) {
return ConfigurationInspection{source: e.internal.InspectConfiguration()}, nil
}
// Diff compares this view's captured effective configuration with other. This
// view is the earlier (left) value and other is the later (right) value. The
// comparison delegates to ConfigKit's frozen effective-value semantics, so
// source or resolution history alone does not create differences. It does not
// reread external sources or compare referenced file contents, resolved
// credential contents, later execution preparation, or request-specific
// overrides. Either zero view returns ErrInvalidConfig and no records; other
// ConfigKit errors are returned unchanged. An empty diff is not evidence of
// equivalent engine behavior or execution. Records can contain ordinary
// application data, so callers choose where and to whom to disclose them.
func (v ConfigurationInspection) Diff(other ConfigurationInspection) ([]configkit.Difference, error) {
if v.source == nil || other.source == nil {
return nil, ErrInvalidConfig
}
return configuration.Diff(v.source, other.source)
}
// Metadata returns detached effective presence and history records.
func (v ConfigurationInspection) Metadata() configkit.Metadata {
if v.source == nil {

View File

@@ -144,9 +144,17 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
if err != nil {
t.Fatalf("select inputs: %v", err)
}
otherSource := &unreadableAfterConstructionFS{FS: fstest.MapFS{
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
"config/body.tmpl": &fstest.MapFile{Data: []byte("hello")},
}}
otherInputs, err := promptkit.FilesystemInputs("comparison source", otherSource, "config")
if err != nil {
t.Fatalf("select comparison inputs: %v", err)
}
var modelCalls atomic.Int64
client := countingLLMClient{calls: &modelCalls}
endpoint, model := "https://example.invalid/v1", "offline-model"
endpoint, model, otherModel := "https://example.invalid/v1", "offline-model", "comparison-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: inputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("offline profile", map[string]promptkit.ProfileConfig{
"offline": {Endpoint: &endpoint, Model: &model},
@@ -155,11 +163,42 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
if err != nil {
t.Fatalf("construct engine: %v", err)
}
otherEngine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: otherInputs}, {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("comparison profile", map[string]promptkit.ProfileConfig{
"offline": {Endpoint: &endpoint, Model: &otherModel},
})}}}},
}}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatalf("construct comparison engine: %v", err)
}
source.closed.Store(true)
otherSource.closed.Store(true)
view, err := engine.InspectConfiguration()
if err != nil {
t.Fatalf("inspect configuration: %v", err)
}
otherView, err := otherEngine.InspectConfiguration()
if err != nil {
t.Fatalf("inspect comparison configuration: %v", err)
}
var expected configkit.Difference
for index := 0; index < 3; index++ {
differences, err := view.Diff(otherView)
if err != nil || len(differences) != 1 {
t.Fatalf("compare captured configurations = (%+v, %v), want one difference", differences, err)
}
if differences[0].Path.String() != "/profiles/offline/model" ||
!strings.Contains(differences[0].Left.YAML, model) ||
!strings.Contains(differences[0].Right.YAML, otherModel) {
t.Fatalf("captured difference = %+v", differences[0])
}
if index == 0 {
expected = differences[0]
} else if differences[0].Path.String() != expected.Path.String() ||
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML {
t.Fatalf("captured difference changed between reads: first=%+v next=%+v", expected, differences[0])
}
}
if _, found := view.Explain(configkit.Keys("prompts", "captured", "1", "messages").Index(0).Key("content_file")); !found {
t.Fatal("captured prompt content path was not inspectable")
}
@@ -167,24 +206,32 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
if err := view.WriteEffective(&report); err != nil || !strings.Contains(report.String(), "captured") {
t.Fatalf("write captured report = (%q, %v)", report.String(), err)
}
if source.reads.Load() != 0 || modelCalls.Load() != 0 {
t.Fatalf("inspection touched source or provider: reads=%d requests=%d", source.reads.Load(), modelCalls.Load())
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != 0 {
t.Fatalf("inspection or comparison touched source or provider: reads=(%d,%d) requests=%d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
}
const calls = 16
const calls = 24
var wait sync.WaitGroup
errs := make(chan error, calls)
for index := 0; index < calls; index++ {
wait.Add(1)
go func(index int) {
defer wait.Done()
if index%2 == 0 {
switch index % 3 {
case 0:
_, err := engine.InspectConfiguration()
errs <- err
return
case 1:
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
errs <- err
case 2:
differences, err := view.Diff(otherView)
if err == nil && (len(differences) != 1 || differences[0].Path.String() != expected.Path.String() ||
differences[0].Left.YAML != expected.Left.YAML || differences[0].Right.YAML != expected.Right.YAML) {
err = errors.New("concurrent comparison returned an unexpected difference")
}
errs <- err
}
_, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "captured", SessionID: "direct"})
errs <- err
}(index)
}
wait.Wait()
@@ -194,7 +241,7 @@ func TestConfigurationInspectionUsesCapturedSourcesWithoutProviderCalls(t *testi
t.Fatal(err)
}
}
if source.reads.Load() != 0 || modelCalls.Load() != calls/2 {
t.Fatalf("concurrent inspection/execution = source reads %d, model calls %d", source.reads.Load(), modelCalls.Load())
if source.reads.Load() != 0 || otherSource.reads.Load() != 0 || modelCalls.Load() != calls/3 {
t.Fatalf("concurrent inspection/comparison/execution = source reads (%d,%d), model calls %d", source.reads.Load(), otherSource.reads.Load(), modelCalls.Load())
}
}

25
doc.go
View File

@@ -4,15 +4,16 @@
// Applications construct an [Engine] with [NewEngine], select ConfigKit
// sources for prompts, profiles, and backends, and
// call [Engine.InspectConfiguration], [Engine.InspectPrompt],
// [Engine.InspectProfile], [Engine.Prepare],
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete
// registries, repositories, validators, and the built-in OpenAI-compatible
// client remain internal implementation details.
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.ListProfiles],
// [Engine.ListPrompts], [Engine.Prepare], [Engine.PrepareExecution],
// [Engine.Run], or [Engine.RunPrepared]. Concrete registries, repositories,
// validators, and the built-in OpenAI-compatible client remain internal
// implementation details.
//
// # Concurrency and ownership
//
// An Engine supports concurrent InspectConfiguration, InspectPrompt,
// InspectProfile, Prepare,
// InspectRequirements, InspectProfile, ListProfiles, ListPrompts, Prepare,
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
// bound admitted Run and RunPrepared calls and model generations where
// configured, while different backend pools and unlimited backends continue
@@ -20,11 +21,12 @@
// still receive concurrent calls and must be safe for that use.
//
// ConfigKit captures the selected configuration when NewEngine constructs an
// Engine. Prepare,
// PrepareExecution, and Run copy request maps, slices, pointer values, and
// JSON-compatible extra parameters before using them. InspectPrompt and
// InspectProfile return copied inspection values. Returned values and values
// passed to extension interfaces are likewise isolated from engine state.
// Engine. Prepare, PrepareExecution, and Run copy request maps, slices, pointer
// values, and JSON-compatible extra parameters before using them. InspectPrompt,
// InspectRequirements, and InspectProfile return copied inspection values.
// ListProfiles and ListPrompts return caller-owned identity slices.
// Returned values and values passed to extension interfaces are likewise
// isolated from engine state.
// Callers own those copies and may mutate them after the call that supplied or
// returned them. [CapacityError] values are caller-owned and may be mutated
// without affecting engine state or another error. Immutable [GenerationError]
@@ -58,7 +60,8 @@
// [ConfigurationSources], [ConfigurationInspection], [BackendConfig],
// [ProfileConfig], [RunRequest],
// [ArtifactRef], [ExecutionTargetOverride], [ProfileInspection],
// [PromptInputDefinition], [PromptInspection], [PreparedExecution],
// [PromptIdentity], [PromptInputDefinition], [PromptInspection],
// [RequirementInspectionRequest], [RequirementInspection], [PreparedExecution],
// [CapacityError], [GenerationError], [RetryExhaustedError], and
// [ExecutionEvent], do not have stable JSON
// representations. Direct API keys are nevertheless excluded from JSON for

View File

@@ -19,6 +19,20 @@ The following Go fragments are illustrative and omit surrounding package,
import, and error-handling code. Use the maintained examples for complete
programs.
## Go Compatibility
The `go` directive in the selected release's [module metadata](../../go.mod)
declares its minimum supported consumer Go version. Maintainers build and
validate releases with the locally selected Go patch release; the minimum
does not simply track that patch version.
Successful release validation on the selected patch is sufficient to presume
support across its Go series, subject to the declared minimum and concrete
evidence of incompatibility. Specific compatibility concerns are investigated
as they arise. Minor releases may raise the minimum; patch releases preserve
it. Release notes explain increases and any required consumer action. See the
[release policy](../policy/release.md#consumer-minimum-and-release-toolchain).
## Construct An Engine
Create an engine with [`NewEngine`](../../engine.go). Select each directory
@@ -104,6 +118,43 @@ engines and prepared handles retain their captured definitions; schema and
request artifact resources still follow their runtime lifecycle. Inspection
does not watch or reload sources automatically.
## Discover Available Profiles And Prompts
Use [`Engine.ListProfiles`](../../listing.go) and
[`Engine.ListPrompts`](../../listing.go) to discover exact identities from the
engine's final captured configuration, then inspect a selected prompt:
```go
profiles, err := engine.ListProfiles(ctx)
if err != nil {
return err
}
if len(profiles) > 0 {
_, err := engine.InspectProfile(ctx, profiles[0])
if err != nil {
return err
}
}
prompts, err := engine.ListPrompts(ctx)
if err != nil {
return err
}
if len(prompts) > 0 {
selected := prompts[0]
inspection, err := engine.InspectPrompt(ctx, selected.ID, selected.Version)
if err != nil {
return err
}
_ = inspection
}
```
These lists describe effective membership, not ConfigKit contribution history,
and do not promise that a profile is ready to execute. Use inspection for
details and preparation to resolve request resources. The linked method GoDoc
owns ordering, ownership, concurrency, and error behavior; identifiers may be
sensitive application data.
## Inspect The Captured Configuration
Use [`Engine.InspectConfiguration`](../../configuration_inspection.go) for an
@@ -130,6 +181,41 @@ shows backend-completed execution settings for one profile, and `Prepare` or
`PreparedExecution.Details` shows request-specific rendered messages, inputs,
and validation resources. None of those views is a substitute for the others.
To review changes between two independently captured configurations, compare
their inspection views:
```go
earlier, err := earlierEngine.InspectConfiguration()
if err != nil {
return err
}
later, err := candidateEngine.InspectConfiguration()
if err != nil {
return err
}
differences, err := earlier.Diff(later)
if err != nil {
return err
}
for _, difference := range differences {
if difference.HiddenChanged {
// Report a protected change without inferring its value.
}
if difference.Left.Redacted || difference.Right.Redacted {
// Apply the application's disclosure policy before presenting snapshots.
}
}
```
See the [`ConfigurationInspection.Diff` GoDoc](../../configuration_inspection.go)
and ConfigKit's [v0.7.0 effective-configuration diff guidance](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.7.0/docs/config.md#effective-configuration-diffs)
for the exact comparison contract. The operation compares only frozen effective
configuration values: it does not reread external files or compare their
contents, resolved credential contents, later execution preparation, or
request-specific overrides. An empty diff is not proof of equivalent execution.
Records can contain ordinary application data, so use the application's
disclosure policy when presenting them.
## Inspect A Prompt Before Preparation
Use [`Engine.InspectPrompt`](../../engine.go) to check one configured prompt's
@@ -154,6 +240,50 @@ rendered messages, and use prepared execution when that work must remain tied
to later execution. The method's [GoDoc](../../engine.go) owns exact fields,
hash, ownership, and error semantics.
## Inspect Requirements Before Preparation
Use [`Engine.InspectRequirements`](../../requirements.go) when the application
needs the prompt's recognizable input and caller-variable dependencies for a
particular context before it has input artifacts. Unlike `InspectPrompt`, it
also walks captured template syntax and selects a profile identity; unlike
`Prepare`, it does not load artifacts or render messages.
```go
requirements, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{
PromptID: "meeting.summary",
Vars: map[string]string{"audience": "project leads"},
})
if err != nil {
return err
}
for _, input := range requirements.Inputs {
if input.Kind == promptkit.RequirementRequired {
// Plan to bind this input before preparation.
}
if input.Kind == promptkit.RequirementOptional && input.Explanation != "" {
// A recognizable reference may occur only on a known skipped path.
}
}
for _, variable := range requirements.Variables {
if variable.Kind == promptkit.RequirementRequired && !variable.Supplied {
// The key is required even though this inspection did not receive it.
}
}
if !requirements.Complete {
// Preserve the partial inventory and account for its discovery issues.
}
```
Optional entries can include recognizable inputs and variables found only in
inactive branches. A missing required variable is reported separately from
discovery completeness: missing a value used only for direct text substitution
does not by itself make the inventory incomplete. Conversely, an unresolved
dependency-bearing branch can make discovery incomplete while retaining known
conditional entries. Do not treat an empty partial inventory as proof that no
input bindings are needed. See the method's [GoDoc](../../requirements.go) for
the exact classifications, supported syntax, ownership, and error contract.
## Prepare Without Model Execution
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,

View File

@@ -59,37 +59,43 @@ exist.
## Maintainer Validation
For ordinary work, select validation proportionate to the change. Do not repeat
successful checks for unchanged content merely to create a commit. The full
release-candidate checks have one executable entry point from the repository root:
successful checks for unchanged content merely to create a commit. During
development, run relevant Go tests and vet, format changed Go files, and execute
the maintained examples when they are affected:
```sh
./scripts/release-check.sh
go test ./...
go vet ./...
gofmt -w path/to/changed.go
python3 scripts/check-examples.py
git diff --check
```
The [script](../scripts/release-check.sh) owns the executable checks:
race-enabled tests, vet, the three offline consumer examples with semantic output
assertions, tracked Go formatting, local Markdown links, module/workspace
hygiene, whitespace, and the credential-pattern scan. It does not publish,
change files, validate release-note structure, or retire roadmaps. It requires Go,
Git, a POSIX shell, and Python 3. Tests and examples require no provider
credentials or live inference calls.
Release candidates use `./scripts/release.sh VERSION`, which runs the selected
profile checks and publishes the tag. The [release procedure](release.md)
describes its requirements and recovery. For an optional diagnostic of a
committed candidate without publication, run `./scripts/releasekit.sh check
VERSION`; this is not a required pre-publication step.
The script can run on intended uncommitted changes during development.
Formatting and documentation checks use tracked files; stage intended new
files before final validation so those checks include them. The release
procedure separately enforces a clean, committed candidate.
The standalone `python3 scripts/check-examples.py` command checks semantic
expectations for the three offline consumer examples. The focused
`python3 scripts/test-release.py` command exercises the release tooling against
disposable local repositories; use it when changing that tooling, not as a
recurring release gate. Tests and examples require no provider credentials or
live inference calls.
Ordinary validation can run on intended uncommitted work. Release validation
uses committed candidate contents and the selected local Go installation.
### Change Review
Inspect `git status --short --untracked-files=all` and the complete diff before
accepting a change. Reject unintended files, credentials, private data,
generated binaries, test output, downloaded assets, and template residue.
The automated credential scan does not replace review of the actual change.
For releases, follow the [release procedure](release.md). Its publication
command runs the full local checks once on the final committed candidate and
automates release-note structure and Git checks. Do not repeat ordinary change
review as a release sign-off. Roadmap retirement remains preparation housekeeping.
For releases, follow the [release procedure](release.md). Do not repeat its
candidate checks as a separate sign-off. Roadmap retirement remains preparation
housekeeping.
Follow the [agent commit policy](policy/agents.md#git-commit-policy) for local
commits. Ordinary commits may leave unrelated work in the workspace; preserve

View File

@@ -51,7 +51,8 @@ message files on ordinary runs; construct a new engine to observe edits.
Schema and request artifact reads still happen during preparation. Use
`InspectConfiguration` for an explicit effective-configuration or provenance
report, `InspectProfile` for backend-completed profile settings, and preparation
for request-specific rendered details.
for request-specific rendered details. For identity discovery before inspection,
see the consumer guide's [profile and prompt listing](consumers/pkg-promptkit.md#discover-available-profiles-and-prompts).
## Prompt Definitions
@@ -124,6 +125,12 @@ Optional `session_id` uses the same template variables and input helper. Its
rendered value is trimmed, omitted if empty, and limited to 256 Unicode code
points. A nonblank direct request session ID bypasses the template.
`InspectRequirements` can enumerate recognizable dependencies from captured
templates before artifact content is available. Its conservative syntax
coverage does not change template validity or rendering; see the [consumer
guide](consumers/pkg-promptkit.md#inspect-requirements-before-preparation) and
the method's [GoDoc](../requirements.go) for that inspection boundary.
### Cache Control
Optional `cache_control` has required `type: ephemeral` and optional `ttl`,

View File

@@ -78,9 +78,8 @@ The client conditionally includes:
The engine resolves backend, profile, and request extra-parameter maps by
whole-map replacement rather than key merging. The resulting effective map is
then merged directly into the top-level body after JSON serialization is
verified. Empty keys and collisions with these reserved fields are rejected
before any provider call:
merged into the top-level body. Empty keys, values that cannot be encoded, and
collisions with these reserved fields are rejected before any provider call:
- `model`
- `session_id`

View File

@@ -34,9 +34,13 @@ resolved request target may supply the endpoint. Generation preparation then:
1. validates shared execution-setting invariants and the final selected base
endpoint;
2. maps the internal request into the OpenAI-compatible chat payload;
3. validates and merges extra parameters;
3. checks extra-parameter keys against empty and reserved names, then merges
their values into the payload;
4. composes `/chat/completions` through parsed URL path operations;
5. resolves authentication and freezes the URL, body, and optional credential.
5. encodes the complete body once on success. If encoding fails, extra values
are checked individually to retain invalid-request classification; otherwise
the complete-request failure remains a request failure;
6. resolves authentication and freezes the URL, body, and optional credential.
Each execution of that prepared request creates a fresh HTTP request, body
reader, and attempt timeout context. It closes the response body and returns

View File

@@ -11,28 +11,28 @@ contributor workflow and validation.
| Component | Implemented responsibility | References |
| --- | --- | --- |
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, explicit inspection, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go) |
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, prompt/profile/requirement inspection and identity listing, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go), [identity listing](../../listing.go), [requirement facade](../../requirements.go) |
| `examples/go-library/prepare` | Demonstrates offline preparation and explicit configuration inspection with a prompt file, programmatic profile, and inline input. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
| `internal/backend` | Holds a defensive, immutable lookup and capacity-policy index for final backend definitions validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
| `internal/catalog` | Selects released keyed catalog assets and PromptKit-owned profile definitions as ConfigKit underlays; effective validation belongs to the configuration loader. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
| `internal/configuration` | Loads keyed backend, profile, and prompt configuration through ConfigKit; records effective field presence, validates final values, captures indexed prompt definitions and referenced message content, constructs backend and presence-aware profile snapshots, and retains an explicit read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [internal sources](sources.md#typed-configuration-loading) |
| `internal/configuration` | Loads keyed backend, profile, and prompt configuration through ConfigKit; records effective field presence, validates final values, captures indexed prompt definitions, referenced message content, compiled templates, input metadata snapshots, sorted identity-only profile and prompt inventories, backend and presence-aware profile snapshots, and an explicit read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [identity listing](../../internal/configuration/listing.go), [internal sources](sources.md#typed-configuration-loading) |
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) |
| `internal/engine` | Assembles selected catalog and caller configuration into one snapshot, then constructs capacity, rendering, validation, generation, repair, and the use-case runner for each engine. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) |
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, requirement inspection, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [requirement values](../../internal/domain/requirements.go), [endpoint invariant](../../internal/domain/endpoint.go) |
| `internal/engine` | Assembles selected catalog and caller configuration into one snapshot, then constructs capacity, rendering, validation, generation, repair, and the use-case runner for each engine. It delegates identity listing directly to the captured configuration inventory rather than the runner. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) |
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, and attempt observations. | [Internal generation coordination](generation.md) |
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data, and conservatively enumerates dependencies from captured template syntax. | [Go-template renderer](../../internal/prompt/go_renderer.go), [requirement discovery](../../internal/prompt/requirements.go) |
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts. | [Internal execution observation](observation.md) |
| `internal/usecase` | Selects captured prompts and profiles, applies request overrides and completion checks, and coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
| `internal/usecase` | Selects captured prompts and profiles, enumerates requirements from captured prompt syntax, applies request overrides and completion checks, and coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [requirement inspection](../../internal/usecase/requirements_inspection.go), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
The internal engine assembles these internal components without exposing their
representations. Consumers depend only on the root facade.

View File

@@ -32,16 +32,29 @@ The runner has no durable run or session store.
## Shared Prompt Selection
The runner uses one prompt-selection and hashing boundary for ordinary
preparation and exact prompt inspection. Preparation retains its early
request-ID check before direct-session normalization; both operations select
one captured definition and its construction-time hash. Referenced message
content and static template syntax were captured when the engine was built.
Ordinary preparation selects the full prompt definition and construction-time
hash. It retains its early request-ID check before direct-session
normalization. Referenced message content and static template syntax were
captured when the engine was built.
Inspection stops after that structural lookup. It does not render templates or
touch profile, artifact, schema, renderer, validator, admission, or model
collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns
the public operation's exact contract.
`InspectPrompt` selects captured metadata through an optional repository
capability, falling back to the full definition and existing hash resolution
for internal repositories without it. It returns owned metadata without
rendering templates or touching profile, artifact, schema, renderer, validator,
admission, or model collaborators. The root
[`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's
exact contract.
`InspectRequirements` uses the same metadata selection to resolve the
request-profile/default identity, then obtains the exact captured requirement
snapshot by the selected prompt ID and version. The analyzer receives only
those identity fields rather than an execution definition. It analyzes the
snapshot with the supplied caller variables and normalized direct-session
bypass; it does not resolve target readiness or enter the preparation
pipeline. Missing snapshot capability is a prompt-load failure rather than a
source fallback. The root
[`Engine.InspectRequirements`](../../requirements.go) GoDoc owns the public
operation's exact contract.
## Shared Profile Selection
@@ -92,8 +105,11 @@ one execution-precedence and error-ordering implementation while allowing a
full backend pool to reject work before expensive schema, artifact, and
rendering operations. `Prepare` discards the plan after returning its public
metadata. `Run` retains the plan through initial and repaired-output validation
and discards it when the operation ends. Prepared execution stores the same
kind of plan only in its private payload.
and discards it when the operation ends. Prepared execution transfers the
completed preparation's owned message, input-hash, and copied-schema values
into its private payload, while deep-copying mutable provider extras and
creating an independent details snapshot. Claim completion or discard clears
the execution payload without removing the details snapshot.
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
out-of-range values fail as invalid requests. Endpoint overrides do not change

View File

@@ -21,6 +21,21 @@ profile inheritance. It retains ConfigKit's result privately for explicit
inspection, not for per-run resolution. The runtime/domain packages do not
depend on ConfigKit.
The `ConfigurationInspection.Diff` facade method compares two retained results
through a private wrapper and a narrow `internal/configuration.Diff` bridge.
The wrapper keeps `Result[document]` inside the adapter; the bridge delegates
once to ConfigKit v0.7.0 and returns its public difference records. Comparison
uses only frozen effective values and does not reopen sources, resolve runtime
credentials, or enter the execution path. ConfigKit applies sensitivity from
both results; reports may still contain ordinary application data. See the
[consumer comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
and ConfigKit's [v0.7.0 comparison contract](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.7.0/docs/config.md#effective-configuration-diffs).
After prompt materialization and effective-profile snapshot construction, the
adapter captures lexically ordered, identity-only inventories from the final
prompt and indexed-profile maps. Its listing methods return caller-owned copies
of those inventories without consulting ConfigKit or copying definitions.
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
files under an explicit root, provide resource context, and leave imports
disabled. Each returned slice belongs to one strict-composition group; file
@@ -35,21 +50,68 @@ The configuration adapter indexes each prompt by exact ID and version,
materializes file-backed message content using the exact effective field's
origin and resource context, and checks static message/session template syntax
at construction. It stores captured content, compiled templates, declared
input metadata, and a definition hash. It closes content-file handles before
input metadata, and a definition hash. Alongside the unchanged hashed
definition, it retains content-type and description presence for each declared
input and provides an internal requirement snapshot with those declarations
and the captured parses. Metadata values come from the final normalized
declaration; absent and null fields remain distinguishable from explicit
strings, including empty strings. It closes content-file handles before
returning and retains no source capability for public inspection.
Role and cache-control normalization is applied to execution-owned message
copies. The retained ConfigKit configuration keeps effective input message
metadata and its resource origins: recording a sequence-wide normalization in
ConfigKit v0.5.0 would replace origins of unchanged content-file fields.
copies. ConfigKit v0.7.0 retains effective input message metadata and its
resource origins; recording a sequence-wide normalization in the retained
configuration would replace origins of unchanged content-file fields.
The runner selects from this index for preparation and `InspectPrompt`.
Neither path rescans configuration or reopens message files. `InspectPrompt`
The runner selects full definitions from this index for preparation and uses
captured prompt metadata for `InspectPrompt` and `InspectRequirements`; the
internal requirement snapshot lookup selects by exact prompt ID and version.
These paths do not rescan configuration or reopen message files. `InspectPrompt`
returns declared metadata without rendering or resolving profile, input, or
schema resources. Runtime rendering still checks request variables and input
helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
schema resources, and its public shape and prompt hashes do not include the
captured presence state. Runtime rendering still checks request variables and
input helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
cover resource origins, context containment, capture, and selection.
The configuration adapter shares exact ID/version selection across its
internal prompt lookups. The execution snapshot lookup returns an owned prompt
definition and cached hash; metadata inspection returns owned input metadata
without copying execution templates. Profile selection likewise has an
identity-only lookup for consumers that do not need the execution target,
while the full profile lookup continues to return an owned target and deep copy
of provider extras. These lookups use captured indexes and do not query
ConfigKit again.
The `internal/prompt` requirement walker uses the captured parse trees and
declaration metadata without executing templates or reading artifacts. It
collects literal `input` names, direct root caller keys, and literal-key
`index` lookups. For `if` and `with`, it classifies branches when their
conditions are simple supported values: direct values, literals, negation, or
two-string equality/inequality. It uses Go template truthiness, so a supplied
nonempty string such as `false` is true. Missing decision values and
content-dependent conditions leave dependency-bearing branches conditional
and report incomplete discovery; unresolved conditions with no branch
dependencies do not by themselves make discovery incomplete. Recognizable
references on known skipped paths remain optional inventory entries with an
inactive-use explanation, unless another applicable use establishes a
stronger classification. Skipped expressions are scanned only for
syntactically recognizable names and are not evaluated. A direct caller value
or literal-key lookup can also supply an input name, including through simple
parentheses or a direct pipeline to `input`; names are used exactly as
supplied. Unresolved names and unsupported dependency-bearing expressions
produce structured limitations while retaining recognizable references.
Where an unresolved name or scope could affect existing optional entries, those
entries are reported as conditional rather than inactive-only or definitively
optional. Direct root-map `with` forms retain caller-map identity. Known scalar
values remain separate from unknown scopes; accesses through untracked local
maps report incomplete discovery rather than silently losing caller keys.
Unsupported name expressions reuse value analysis's conservative syntax scan
instead of recursively rescanning the same subtree.
Invoked named templates are scanned conservatively, and
dependency-bearing range scopes make discovery incomplete. Uncalled definitions
and bypassed session templates are not traversed. This internal walker does not
change rendering behavior.
## Profiles And Built-Ins
`internal/catalog` selects the released keyed OpenRouter and Rakestrawhome
@@ -66,10 +128,16 @@ snapshots with backend defaults and presence/suppression state. A profile can
remain partial until runtime overrides complete it. `InspectProfile` selects
that captured target without prompt, artifact, schema, credential lookup, or
provider work. Preparation applies runtime overrides and then checks target
completeness. The [target tests](../../internal/configuration/targets_test.go)
cover effective target interpretation, while the [catalog tests](../../internal/catalog/catalog_test.go)
completeness. Identity-only profile selection can identify a partial captured
profile without copying its target or requiring readiness. The
[target tests](../../internal/configuration/targets_test.go) cover effective
target interpretation, while the [catalog tests](../../internal/catalog/catalog_test.go)
cover selected released assets.
The internal engine delegates listing through a narrow identity-inventory
interface directly to the configuration snapshot. The execution runner has no
listing responsibility.
## Ordinary Artifacts
`internal/artifact` accepts explicitly typed inline references even when their
@@ -79,10 +147,15 @@ and after opening the file. It copies content into an artifact, records
metadata and an opaque content-equality value, and applies a content-type
fallback.
Regular files are read synchronously in bounded chunks. Cancellation is
checked before opening, before and after every read, and before publishing the
artifact, so a canceled read never publishes partial content. The ordinary
reader does not detach file reads into background goroutines.
Regular files are read synchronously in bounded chunks. The opened file's size
is only an initial-capacity hint, capped at 1 MiB; reads continue to EOF, and
the returned body determines artifact size and hash. If an overreported hint
leaves unused capacity greater than the actual body length, the completed body
is copied to release the excess backing storage, including when the body is
empty. Cancellation is checked before opening, before and after every read,
and before publishing the artifact, so a
canceled read never publishes partial content. The ordinary reader does not
detach file reads into background goroutines.
This ordinary reader does not implement an inbound HTTP security boundary. In
particular, it does not constrain files to an application root or impose an
@@ -106,7 +179,9 @@ between artifact conversion chunks, around each message, and before publishing
the complete prompt. It cannot interrupt template work already in progress and
never publishes a partial prompt after observing cancellation. It validates and
canonicalizes message roles before carrying roles, session IDs, and cache
control into the rendered prompt. The
control into the rendered prompt. Message and session templates execute into
per-call string builders, so their rendered strings do not require a final
buffer-to-string copy. The
[renderer tests](../../internal/prompt/renderer_test.go) own rendering behavior.
## Schemas And Output Validation
@@ -114,7 +189,8 @@ control into the rendered prompt. The
`internal/validate` provides validators backed by an operating-system
filesystem or an `fs.FS`. Invalid generated content is returned as a validation
result; inability to load, register, or compile a schema is an operational
error.
error. Basic validation checks body emptiness on the byte slice directly
instead of first making a response-sized string copy.
Every preparation operation creates one operation-local validation plan. None,
basic, and JSON modes retain the effective output contract without source

View File

@@ -6,29 +6,37 @@ Material: policy documents under `docs/policy/` and the Policy Reading Guide in
`docs/development.md`, adapted for this repository; root `AGENTS.md`, copied
without modification from upstream `agents/AGENTS.md`.
The release policy uses upstream commit
`e833faddf120c789a9b45348edaf75720de16348`; the remaining material uses
`d84a38ebab26f048cafbe59a2246700c35d29843`.
Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: CC-BY-SA-4.0
License and notices: [CC BY-SA 4.0](licenses/CC-BY-SA-4.0.txt) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE).
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/LICENSE) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/LICENSE).
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843).
[Source](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348).
## ConfigKit
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.5.0`.
Selected module: `gitea.maximumdirect.net/eric/ConfigKit` at `v0.7.1`.
Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: LGPL-3.0-or-later
License and notices: [LGPL 3.0](../../LICENSE) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd/LICENSE).
[COPYRIGHT](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77/LICENSE).
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd).
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/6fffde4af8f31f99d3ee295c426eb3234cb0fb77).
## Promptkit OpenRouter Catalog v3

View File

@@ -1,10 +1,10 @@
# Release Policy
**Base Policy Version:** 1.4
**Base Policy Version:** 1.6
**Last Updated**: 2026-09-27
**Last Updated**: 2026-10-08
**Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/d84a38ebab26f048cafbe59a2246700c35d29843/docs/policy/release.md)
**Upstream Source:** [release.md](https://gitea.maximumdirect.net/eric/policy/src/commit/e833faddf120c789a9b45348edaf75720de16348/docs/policy/release.md)
## Purpose
@@ -12,6 +12,9 @@ This policy defines a proportionate workflow for validating and publishing an
intended source revision under a fresh version. Releases should require little
routine intervention, avoid repeated work, and preserve published contents.
Adopt stronger verification or reproducibility requirements through an explicit
project-specific decision justified by the intended audience and support needs.
The repository-specific commands belong in `docs/release.md`. Historical release
summaries belong in `docs/releases/`. The procedure must agree with the adopted
policy and link to it rather than introduce additional release requirements.
@@ -59,7 +62,8 @@ requires a deliberate update to the policy and procedure where needed.
For releases at or after `v1.0.0`:
- a major release may make intentional incompatible changes;
- a minor release may add backward-compatible behavior; and
- a minor release may add backward-compatible behavior or raise the Go consumer
minimum as described below; and
- a patch release may make backward-compatible fixes.
Before `v1.0.0`, a minor release may change a documented public, operational,
@@ -71,14 +75,44 @@ Additional compatibility promises and deliberate exceptions belong in the
adopted policy or canonical contract documentation. Release notes explain
changes; they do not redefine current compatibility contracts.
## Release Documentation
## Consumer Minimum And Release Toolchain
Distinguish the minimum toolchain supported for consumers from the toolchain
maintainers use to build and validate a release. Record the consumer minimum in
the language's standard metadata and explain its support meaning in canonical
documentation. Define the selected release toolchain and its setup commands in
`docs/release.md`. Toolchain selection and upgrade cadence are repository-specific
choices.
For Go repositories, normally declare the initial stable release of the selected
series as the consumer minimum, such as `go 1.26.0`. Use a higher patch minimum
when a dependency or an identified correctness fix requires it, and document
the reason.
For release-policy purposes, successful build and release validation on the
selected Go patch release are sufficient to presume support across that Go
series, subject to the declared consumer minimum and concrete evidence of
incompatibility. Investigate specific compatibility concerns as they arise.
Use the selected release toolchain for candidate validation and release builds,
including CI jobs performing those tasks. A newer validation patch version alone
leaves the consumer minimum unchanged.
A minor release may raise the Go consumer minimum, including from one Go series
to the next, both before and after `v1.0.0`. This is an explicit exception to the
backward-compatibility requirement for minor releases. Patch releases preserve
the consumer minimum. Explain an increase's impact and required consumer action
in the release note. Other incompatible changes remain subject to the versioning
rules above.
## Release Documentation And Preparation
Every new release must include `docs/releases/<version>.md` in the tagged
source tree. Use the complete version tag as the filename and begin with a
title containing the product name and that exact version.
Include a short summary and any relevant compatibility impact or upgrade
instructions. Use the Promptkit section structure below and keep the note
instructions. Choose sections suited to the release and keep the note
proportionate to the change.
Keep exact current contracts in their canonical documents and update affected
@@ -186,7 +220,8 @@ release without replacing its contents or reusing its version.
Maintain `docs/release.md` as a short, executable maintainer runbook. It must
identify:
1. prerequisites, primary branch, and authoritative remote;
1. prerequisites, selected release toolchain, primary branch, and authoritative
remote;
2. version and release-note preparation;
3. one publication command with its selected local checks and automated guards;
4. declared deliverables, required build targets, and automation responsibilities;
@@ -210,20 +245,39 @@ annotated stable SemVer tags. It has no binaries, hosted download pages, or
asynchronous deliverables. The source commit is pushed through main before the
tag. The [release procedure](../release.md) owns commands and recovery.
The existing publication script requires Summary, Compatibility, Upgrade, and
Changes headings beneath a title of `# Promptkit vMAJOR.MINOR.PATCH`. State
“No action required” or “None” where appropriate. This retained local structure
matches the automated release-note guard; changing it is separate work.
Release preparation uses the latest Go installation available in the development
environment to build the maintainer release executable and validate the
candidate, including the consumer examples. Keep that same local toolchain
active throughout those operations.
The [publication script](../../scripts/release.sh) runs the
[local checks](../../scripts/release-check.sh) once for the final committed
candidate: race-enabled tests, vet, three offline consumer examples, tracked
Go formatting, Markdown links, module/workspace hygiene, whitespace, and a
credential-pattern scan. These protect concurrency, assembled library use,
source distribution, and documentation integrity. A separate ordinary test run
or standalone build is not an additional release gate. Candidate, tag, and
synchronized-main checks remain automated. No CI or module-availability wait
is required. Licensing footer URLs are preparation work, not a network gate.
For a minor release, advance Promptkit's consumer minimum to the initial stable
release of that Go series, retaining a higher patch minimum when required by a
dependency or an identified correctness fix. Patch releases preserve the
existing consumer minimum. Commit any module update before validating the final
candidate. The [release procedure](../release.md#prepare-the-candidate) owns
setup commands, and the consumer guide explains
[Go compatibility](../consumers/pkg-promptkit.md#go-compatibility).
Use a title containing `Promptkit` and the exact version, followed by a short
summary and any relevant compatibility impact or upgrade instructions. The
note is committed with the candidate and supplies the annotated tag message.
No fixed set of section headings is required.
The [publication command](../../scripts/release.sh) delegates to the released
ReleaseKit tool selected in `go.mod`. The committed
[profile](../../releasekit.json) selects the race-enabled Go suite, full vet,
committed Go formatting, local Markdown links, workspace and filesystem
replacement inspection, and the offline semantic checks for the three
maintained consumer examples. These are the selected checks for the final
candidate; a separate ordinary test run or standalone build is not an
additional release gate. The upstream tool owns candidate, remote, and tag
publication checks. It returns native status 0 for confirmed publication, 1
for a known validation or publication failure, 2 for invalid input or profile,
and 3 when the push outcome is unknown. For status 3, resolve the exact
captured publication endpoint before retrying; the repository-specific
[procedure](../release.md#failure-recovery) links to the upstream recovery
instructions. Do not wait for CI or Go module availability. Licensing footer
URLs are preparation work, not a network gate.
## Catalog Profile Compatibility

View File

@@ -58,17 +58,24 @@ and repository hygiene.
Introducing hosted CI later would supplement, not silently redefine, this
documented validation model.
Full release-candidate validation includes race-enabled package tests, explicit
static analysis, and execution with automated semantic checks of the three maintained
offline consumer examples. The test run builds the packages; the examples
also exercise ordinary compilation. Separate ordinary tests and a standalone
build are not additional release gates. Ordinary tests remain useful during
development. The preparation example protects assembled preparation and
The selected ReleaseKit profile runs the race-enabled package suite and full
vet, checks committed Go formatting, local Markdown links, and module
distribution hygiene, and executes semantic assertions for the three
maintained offline consumer examples. ReleaseKit owns this shared validation
machinery and publication workflow. The test run builds the packages; the
examples also exercise ordinary compilation. Separate ordinary tests and a
standalone build are not additional release gates. Ordinary tests remain useful
during development. The preparation example protects assembled preparation and
inspection behavior. The execution example separately protects assembled
`Run`, injected-client, validation, usage, and result behavior. The observation
example protects assembled built-in retries, structured proposal and assessment
runs, and application-owned interpretation of metadata.
The standalone `python3 scripts/test-release.py` command exercises Promptkit's
ReleaseKit wiring against disposable local repositories. It is focused
development validation when release tooling changes, not part of the default
Go test suite or a recurring release check.
Tests in the default suite must be deterministic, offline, and independent of
real credentials. They must not invoke paid APIs, use live network
dependencies, or depend on mutable external services. Tests that require live

View File

@@ -1,64 +1,101 @@
# Release Procedure
This runbook implements the [release policy](policy/release.md). A successful
push of the selected annotated tag to origin completes the release. Never wait
for CI to finish or succeed.
This runbook implements the [release policy](policy/release.md). Promptkit
publishes source-only Go module tags from `main` to `origin`. A successful push
of the selected annotated tag, after its source commit is available on `main`,
completes the release. Do not wait for CI or Go module availability.
## Prepare The Candidate
Required tools are Git, Go with race-detector support, POSIX shell, and Python
3. Publication requires push access to origin. Work from the repository root.
3. Work from the repository root with push access to `origin`.
Select an unused version and create `docs/releases/<version>.md` using the
[required structure](policy/release.md#release-documentation). Describe
compatibility impact and consumer actions and update affected canonical docs.
For catalog updates, record selected catalog versions and any consumer action,
linking to the catalog policy rather than repeating its contracts.
Use the latest Go installation available locally for the release tool, the
candidate validation, and the maintained examples. Make it the active `go`
command and record its version:
Follow the [OSS policy](policy/oss.md) when preparing licensing and attribution
changes. Each new note must end with the following navigation, replacing
`vMAJOR.MINOR.PATCH` with the selected release tag:
```markdown
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vMAJOR.MINOR.PATCH/docs/oss/attribution.md)
```sh
export GOTOOLCHAIN=local
go version
```
Check these file paths against the candidate; a future tag need not resolve
remotely during preparation. The annotated tag derives from this same note,
including its footer. Promptkit publishes no separate hosted release body or
download page. If such a channel is deliberately introduced, reuse the note
and preserve its footer. No extra network checks, artifacts, or approval gates
are introduced, and published notes are not retrofitted.
For a minor release, advance Promptkit's consumer minimum to the initial stable
release of that Go series before committing the candidate:
Retire completed roadmaps, preserving planned or deferred work and repairing
links. This is housekeeping, not a separate approval gate. Review changes
through the ordinary development workflow, commit them, and push main to
origin. No repeated release-specific diff review is required.
```sh
release_go_version=$(go env GOVERSION)
release_go_series=${release_go_version#go}
release_go_series=${release_go_series%.*}
go mod edit "-go=${release_go_series}.0"
```
For example, Go 1.26.8 normally results in `go 1.26.0`. Keep a higher patch
minimum if a dependency or identified correctness fix requires it. For a patch
release, leave the consumer minimum unchanged; using a newer validation
toolchain does not itself raise that minimum. If the candidate needs a higher
minimum, prepare a minor release instead. See the
[consumer-minimum policy](policy/release.md#consumer-minimum-and-release-toolchain).
Prepare module caches before validation; this may update `go.sum` and belongs
before the candidate commit:
```sh
GOTOOLCHAIN=local go mod download
```
Choose an unused stable version and create `docs/releases/<version>.md` with a
title containing `Promptkit` and that exact version, a short summary, and any
relevant compatibility impact or upgrade instructions. No fixed section
headings are required. Describe a raised Go minimum and its compatibility
impact under the [versioning policy](policy/release.md#versioning-and-compatibility).
End each new note with the licensing footer below, using the selected version:
```markdown
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/vX.Y.Z/docs/oss/attribution.md)
```
Check the footer paths against the candidate; the future tag need not resolve
remotely during preparation. Follow the [OSS policy](policy/oss.md) for these
links. Promptkit publishes no binaries, hosted download page, or other
asynchronous deliverables.
Retire completed roadmaps while preserving deferred work and repairing incoming
links. Review the intended changes through the ordinary development workflow,
commit the candidate on `main`, and push `main` to `origin` before publication.
Preparation may update module metadata; the release bootstrap and validation
preserve the committed candidate.
## Validate And Publish
Run the following with the chosen unused version (replace the placeholder):
For an optional local diagnostic of a committed candidate, run:
```sh
./scripts/releasekit.sh check vMAJOR.MINOR.PATCH
```
This is not a required step before publication. The upstream
[candidate-check contract](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/consumers/check.md)
describes its requirements.
Run the single publication command with the chosen version:
```sh
./scripts/release.sh vMAJOR.MINOR.PATCH
```
The [publication script](../scripts/release.sh) checks a clean, synchronized
main candidate, the version and unused tag, and the release note's title and
required headings. It runs the [local checks](../scripts/release-check.sh)
once, rechecks candidate identity and publication state, generates the
annotation from the committed note plus the validated commit, and pushes only
the selected annotated tag.
The familiar wrapper delegates to the native ReleaseKit executable selected in
`go.mod`. The committed [profile](../releasekit.json) selects the race-enabled
Go suite, full vet, committed Go formatting, local Markdown links, workspace
and filesystem replacement inspection, and semantic checks for the three
maintained offline consumer examples. ReleaseKit also verifies the committed
candidate and publication state, then publishes only the selected tag. Its
checks run once for this candidate; do not repeat them as a separate sign-off.
There is no separate tag-message authoring, example-output inspection,
metadata inspection, or human sign-off. Compatibility and release-note content
remain ordinary editorial responsibilities. Development validation is useful
but does not require a second full run before invoking the publication script.
The script stops successfully after the tag push. CI completion, CI success,
remote read-back, and Go module availability are not release completion gates.
Do not wait or poll for them. Provider probes and inference calls are never
part of release validation.
The native command returns status 0 for confirmed publication, 1 for a known
validation or publication failure, 2 for invalid input or profile, and 3 when
the push outcome remains unknown. Successful publication needs no remote
read-back, CI wait, or installation check. Provider probes and inference calls
are not part of validation.
## Catalog Profile Compatibility
@@ -67,28 +104,32 @@ catalog compatibility boundary. This heading preserves older incoming links.
## Failure Recovery
If local checks fail, fix the candidate, commit and push changes, then rerun
the publication command. It never automatically removes a tag.
For status 1, use the diagnostic to distinguish a validation failure from a
known publication conflict. Correct and commit candidate defects, push `main`,
and rerun publication with an unused version when appropriate. Status 2 means
the invocation or profile must be corrected.
If a push fails or its result is ambiguous, set `RELEASE_VERSION` to the
attempted version and inspect origin before retrying:
For status 3, do not retry or remove the local tag until the remote outcome is
resolved. ReleaseKit reports the attempted version, tag object, and recovery
state, including the captured publication endpoint when available. Use that
endpoint—not merely the current `origin` URL—to query the exact tag; follow the
upstream
[failure-recovery procedure](https://gitea.maximumdirect.net/eric/ReleaseKit/src/tag/v0.4.1/docs/release.md#resolve-failures):
```sh
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
git rev-parse "refs/tags/$RELEASE_VERSION" "refs/tags/$RELEASE_VERSION^{}"
releasekit_endpoint='COPY_CAPTURED_ENDPOINT'
releasekit_version='vX.Y.Z'
test "$(git ls-remote --get-url -- "$releasekit_endpoint")" = "$releasekit_endpoint" &&
git ls-remote --refs -- "$releasekit_endpoint" "refs/tags/$releasekit_version"
```
Matching remote tag and peeled commit values confirm publication, even if the
push command reported an error. If the remote query fails, publication status
is still unknown. Do not delete or recreate the local tag until absence on
origin is confirmed. Once confirmed unpublished, an unsuccessful attempt's
local tag can be removed and the publication command rerun. Never overwrite
a published tag.
CI or module-resolution problems may be investigated separately; temporary
service, network, or authentication failures do not require another release.
Only an actual defect requiring changes to published contents warrants a new
appropriate version. See the [recovery policy](policy/release.md#immutability-and-failure-recovery).
Compare the returned object ID with the reported tag object. An exact match
confirms publication. A missing ref confirms absence only when the endpoint
identity check and query both succeed. If either check fails, the outcome
remains unknown; correct the transport configuration and resolve it before
retrying. Only after authoritative absence is confirmed may an unpublished
local tag be removed before correcting the candidate or retrying. Never move,
overwrite, or delete a published tag.
## Policy Changes

74
docs/releases/v0.12.0.md Normal file
View File

@@ -0,0 +1,74 @@
# Promptkit v0.12.0
This release replaces Promptkit's configuration loader with ConfigKit and
migrates maintained catalogs to keyed `/v3` assets. It intentionally changes
the public construction API and configuration format.
## Summary
Backends, profiles, and versioned prompts now share one layered configuration
snapshot built when `NewEngine` is called. ConfigKit supplies composition,
inheritance, presence, and source history; Promptkit validates the final
effective definitions and captures prompt content and templates. Consumers can
inspect the captured configuration explicitly without triggering a reload.
## Compatibility
This is an incompatible minor release under Promptkit's pre-1.0
[versioning policy](../policy/release.md#versioning-and-compatibility). The
former source options, `Config.PromptDir`, `Config.ProfileDir`, and the old
profile/backend convenience values have been removed. Configuration documents
now use literal keys beneath `backends`, `profiles`, and `prompts`; embedded
`id`/`version` fields and `base_profile` are no longer accepted. Source groups
compose strictly within a group and overlay in the supplied order. Explicit
zero, false, empty, and null values can now differ from omission; maps overlay
recursively, and an empty map does not clear an inherited map.
All final definitions, including unused ones, are validated during engine
construction. Prompt text and static templates are captured then; an existing
engine does not automatically reload changed configuration or message files.
Request artifacts, schemas, credentials, and provider operations retain their
runtime boundaries. See the [format reference](../formats.md) for the current
configuration contract and the [consumer migration guide](../consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api)
for API and source-selection changes.
Maintained OpenRouter and Rakestrawhome catalogs are selected as `/v3` modules,
both at `v3.20260930.0`. Their asset format changed; consumers that read catalog
assets directly must adopt the keyed format. The maintained Promptkit profile
IDs remain available under the [catalog compatibility policy](../policy/release.md#catalog-profile-compatibility),
which does not promise fixed profile data.
## Upgrade
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.12.0
```
Replace removed source and backend registration options with
`Config.Sources`, explicitly ordered ConfigKit underlays, and optionally a
highest-priority main input. Use `DirectoryInputs` or `FilesystemInputs` for
selected directories, `ProfileSource` and `BackendSource` for typed
programmatic values, or `configkit.Mapping` for explicit null and combined
documents. Move profile `base_profile` to `extends` and embedded identities to
mapping keys. Review zero/false/null values and any assumptions that replacing
a model or backend discards unrelated explicit settings. Rebuild the engine
when definitions or referenced message files change. The
[consumer guide](../consumers/pkg-promptkit.md#migrate-from-the-previous-configuration-api)
provides the migration details and maintained examples.
## Changes
- Add ConfigKit v0.5.0 as the configuration boundary and consume released
OpenRouter and Rakestrawhome `/v3` catalog modules.
- Add keyed backend, profile, and prompt definitions with final-value domain
validation, explicit presence/null behavior, and source-relative content
capture.
- Add `ConfigurationSources`, directory discovery helpers, programmatic
profile/backend source builders, and read-only configuration inspection.
- Preserve execution workflows while rendering captured templates, applying
backend defaults after inheritance, and honoring explicit suppression of
provider settings.
- Remove superseded loader, catalog preflight, and inheritance machinery;
update examples, format guidance, and internal documentation.
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.12.0/docs/oss/attribution.md)

69
docs/releases/v0.13.0.md Normal file
View File

@@ -0,0 +1,69 @@
# Promptkit v0.13.0
This release adds prompt requirement discovery and profile/prompt listing,
reduces avoidable configuration and execution allocations, and standardizes
maintainer publication through ReleaseKit. The minimum supported Go version
increases to 1.26.0.
## Discover Requirements And Available Definitions
- `Engine.InspectRequirements` reports recognizable input and caller-variable
dependencies for a selected prompt and supplied caller context before input
artifacts exist. It classifies dependencies as required, conditional, or
optional and reports structured limitations when discovery is incomplete.
Discovery coverage is not proof that requirements are satisfied or execution
will succeed. See the [requirement inspection guide](../consumers/pkg-promptkit.md#inspect-requirements-before-preparation)
and [GoDoc](../../requirements.go).
- `Engine.ListProfiles` returns effective profile IDs, and `Engine.ListPrompts`
returns exact ID/version pairs as `PromptIdentity` values. Lists come from the
engine's captured configuration, are lexically ordered and caller-owned, and
include consumer definitions and maintained profiles that remain effective.
Listing performs no provider or execution work. See the
[discovery guide](../consumers/pkg-promptkit.md#discover-available-profiles-and-prompts)
and [GoDoc](../../listing.go).
## Efficiency And Tooling
- Reduce response-sized copies in basic validation and rendered-message output,
and encode valid provider extra parameters only as part of the complete
request rather than serializing them twice.
- Use bounded initial file-size reservations without limiting accepted content,
releasing excess backing storage for small or empty completed artifacts.
- Reuse owned preparation values when creating prepared-execution snapshots,
while preserving independent inspection details and caller-owned data.
- Use lightweight captured profile identities and prompt metadata for inspection
instead of copying execution targets and definitions unnecessarily.
- Upgrade ConfigKit to v0.6.0 for more efficient loading, resolution, metadata,
and inspection processing. Configuration semantics remain unchanged; these
improvements primarily benefit construction and explicit operator inspection,
not upstream inference latency.
- Adopt module-selected ReleaseKit v0.3.0 for maintainer validation and tag
publication. The [release procedure](../release.md) owns the current commands
and recovery workflow; successful tag publication remains the completion
point, without waiting for CI.
These changes preserve existing execution and ownership contracts. Benefits
depend on workload; this release introduces no fixed performance guarantee.
## Compatibility And Upgrade
Consumers upgrading from v0.12.0 need **Go 1.26.0 or newer**, rather than the
previous 1.25.5 minimum. Update application build environments and any CI images
still using Go 1.25 before upgrading:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.13.0
```
No application-code or configuration migration is required from v0.12.0 beyond
the Go toolchain update. The new discovery methods are optional additions;
existing construction, inspection, preparation, and execution workflows remain
available. See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility)
for the distinction between the consumer minimum and maintainer toolchain.
Maintainers should follow the current release runbook rather than the retired
`scripts/release-check.sh`. The publication entry point remains
`scripts/release.sh`; ReleaseKit now owns its shared validation and publication
mechanics.
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.13.0/docs/oss/attribution.md)

48
docs/releases/v0.14.0.md Normal file
View File

@@ -0,0 +1,48 @@
# Promptkit v0.14.0
This release adds comparison of captured configuration inspections and updates
ConfigKit and the maintainer release tool. The minimum supported Go version
remains 1.26.0.
## Compare Captured Configurations
`ConfigurationInspection.Diff` compares the effective configuration captured by
two engines. Consumers can use it to review a candidate configuration, inspect
an overlay's effect, or present configuration changes to operators.
Comparison delegates to ConfigKit and returns its difference records, including
presence information, paired redaction, and protected-change indicators. It
does not reload sources or contact providers. Source-history differences alone
do not produce value differences.
An empty diff is not proof of equivalent execution: referenced file contents,
resolved credentials, later execution preparation, and request-specific
overrides are outside the comparison. Records can contain ordinary application
data; callers retain responsibility for their disclosure.
See the [configuration comparison guide](../consumers/pkg-promptkit.md#inspect-the-captured-configuration)
and [GoDoc](../../configuration_inspection.go) for usage and the exact contract.
## Dependency And Tooling Updates
- Upgrade ConfigKit from v0.6.0 to v0.7.1, providing the upstream comparison API.
v0.7.1 also updates ConfigKit's maintainer tooling without changing its runtime
behavior from v0.7.0.
- Upgrade the module-selected ReleaseKit tool from v0.3.0 to v0.4.1. Promptkit
retains its existing root-module profile, publication entry point, and release
gates. See the [release procedure](../release.md) for maintainer commands.
## Compatibility And Upgrade
No application-code, configuration, or Go-toolchain migration is required from
v0.13.0. Configuration comparison is an optional API addition; existing
construction, inspection, preparation, and execution workflows remain available.
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.14.0
```
See [Go compatibility](../consumers/pkg-promptkit.md#go-compatibility) for the
distinction between the consumer minimum and maintainer toolchain.
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/COPYRIGHT) | [LICENSE](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/LICENSE) | [ATTRIBUTION](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.14.0/docs/oss/attribution.md)

View File

@@ -1,505 +0,0 @@
# ConfigKit Adoption Assessment
Status: assessment aligned to released ConfigKit v0.5.0. Upstream capability
prerequisites are available; PromptKit integration and migration decisions remain
for the feature roadmap and implementation plan. No runtime changes are part of
this assessment.
## Goal And Recommendation
Use ConfigKit to standardize PromptKit's configuration semantics and simplify
its loading machinery, improving ConfigKit where a reusable capability is
justified. Preserve substantive consumer capabilities, not historical loading
sequences, file syntax, or Go signatures. Document intentional breaking changes
and their migration paths.
Recommend adoption through an eager, immutable definition-loading boundary.
ConfigKit should own YAML interpretation, source-tier combination, strict typed
decoding, sibling inheritance, presence, and source diagnostics. PromptKit should
own definition selection, LLM-specific rules, and execution. Do not retain the lazy loading
architecture merely to minimize migration, or introduce a second generic
configuration framework inside PromptKit.
The expected benefits are consistent configuration semantics for applications
using both libraries and removal of repeated discovery, partial metadata parsing,
lazy error arbitration, and repeated profile-chain resolution. Some of those
savings come from the loading redesign rather than ConfigKit itself. ConfigKit
adds ongoing value by owning the generic decoding behavior and its tests.
## Released ConfigKit Capabilities
ConfigKit v0.5.0 satisfies the two requested prerequisites and also implements
sibling inheritance. Adopt the released public contracts rather than waiting for
the earlier proposals to be implemented:
1. **Filesystem sources:** `NewFilesystem(label, fsys).File(name)` borrows an
`fs.FS` with reusable namespace identity. Imports and resource contexts retain
that namespace. `Result.SourceContext` and `Resolution.SourceContext` expose
passive `ResourceContext` values with `FS` and `BaseDir`.
2. **Source-backed underlays:** `LoadLayered` accepts a main `Input`
and ordered `Underlay` groups. Each group composes strictly; groups overlay
recursively from lowest to highest priority. `Input{Source: NoSource()}` omits
the main input. `Mapping(label, map[string]any)` snapshots supported built-in
scalar/list/map values with explicit presence; it does not ingest arbitrary
typed structs. Generic programmatic input is not represented as CLI overrides.
3. **Sibling inheritance:** the reserved `extends` key names one immediate sibling
mapping. Relationships resolve after precedence assembly and before typed
decoding, with inherited origins, history, and resource context preserved.
This is built-in behavior, not the opt-in/registered-collection API originally
suggested by PromptKit.
Adopt ConfigKit's recursive combination model rather than preserving PromptKit's
whole-profile shadowing or atomic provider-map replacement across source tiers.
No hidden replacement or inheritance pass should run after ConfigKit loading.
Use released sibling inheritance for profiles, leaving domain validation in
PromptKit. This assessment does not authorize code changes in either repository.
## Evidence And Scope
PromptKit's runtime baseline remains v0.11.2; this assessment has since received
documentation-only revisions. Updated upstream review on 2026-09-30:
- PromptKit: `12d6d80024e80c552d122a076c22429bf6f2f6c2` (v0.11.2).
- ConfigKit: `dc09959cca4e105f49ec316c6d4ba585bd2340bd` (v0.5.0), verified
against the published tag. Its clean checkout passed `go test ./...` during
the release review. This replaces the earlier v0.4.2 and uncommitted-roadmap
comparison as the upstream basis for adoption.
PromptKit evidence: [consumer workflows](../consumers/pkg-promptkit.md),
[file formats](../formats.md), [source implementation](../internal/sources.md),
[runner](../internal/runner.md), and the implementation owners identified below.
ConfigKit evidence at that release: `docs/config.md`, `docs/releases/v0.5.0.md`,
`layered.go`, `source.go`, `resource_context.go`, and ADR 0009 in the sibling
checkout. That checkout is review context, not a build dependency or required
directory layout. The released configuration contract owns exact upstream behavior;
the summaries here identify its consequences for the proposed PromptKit design.
This is a source-and-contract assessment, not a performance benchmark or full
correctness audit. It does not claim measured allocation, latency, or line-count
savings. Current code and canonical contracts remain authoritative until an
approved implementation changes them.
## 1. Capability Inventory
| Preserve the ability to… | Replaceable mechanics and representation |
| --- | --- |
| Define named, versioned prompts, inspect their declared inputs, and select a version explicitly | Directory scans on each operation, YAML metadata prepasses, filename layout, constructor signatures |
| Use inline and file-backed message templates, variables, input helpers, session templates, roles, and cache control | Timing of definition/content reads and private repository interfaces |
| Configure named profiles, alias/inherit another profile, and override selected settings | Zero-as-absence fields, per-call chain traversal, raw YAML decoding types |
| Supply embedded application defaults, operator definitions, maintained catalogs, and programmatic profiles | Repository-wrapper stacks and exact source-option API |
| Register custom/local backends, use built-ins or endpoint-only profiles, and control backend capacity | Configuration DTO shapes and registry assembly order |
| Prepare, inspect, execute, or freeze an execution for later single-claim execution | Source reloading during ordinary preparation; exact preparation error ordering |
| Override per-run settings, credentials, reasoning, session, and output contract; append rendered messages | Public value signatures if presence or clearer configuration separation requires revision |
| Supply inline/file artifacts and inject artifact readers, model clients, and HTTP clients | Assembly APIs; runtime collaborators need not be YAML-decodable |
| Validate output, use provider-native structured output, bounded repair, transport retries, cancellation, and observation | Internal wiring, not the substantive execution behaviors |
| Receive results, usage, identities, hashes, validation outcomes, and inspectable failures | Configuration error types and when definition failures appear |
Preserve independent profile and prompt inspection as useful operations. They
need not retain the old guarantee that engine construction has never examined
other definitions. Preserve custom backend identity and capacity behavior,
optional upstream credentials, and the distinction between failed generated
content and operational errors. This project does not authorize silently
dropping execution features to simplify the configuration layer.
Add explicit configuration inspection for consumers implementing operator-facing
CLI commands or debug output. This is a new capability motivated by this adoption,
not an assertion that the current prompt/profile inspection methods already
provide configuration provenance.
## 2. Responsibility Mapping
| Concern | Recommended owner and disposition |
| --- | --- |
| YAML grammar, scalar interpretation, one-document enforcement, unknown fields, typed ranges, null legality | Delegate to ConfigKit; remove corresponding PromptKit parser logic and redundant parser tests |
| Field presence, source coordinates, and configuration provenance | Delegate to ConfigKit; retain its inspection state with the loaded snapshot for explicit operator queries, separate from execution-ready values |
| Operator-facing configuration inspection | Provide a PromptKit consumer API reusing ConfigKit inspection facilities; applications own CLI commands, presentation, and disclosure policy |
| File/directory/embedded source selection and definition ID indexing | Retain a small PromptKit collection loader; ConfigKit loads documents, not a prompt catalog |
| Configuration filesystem sources, relative imports, and generic layer origins | Delegate to the prerequisite ConfigKit APIs; do not maintain parallel reader-based loading machinery |
| Definition discovery and relative message-content references | Retain PromptKit's domain resource adapters, using ConfigKit source context and standard `io/fs` facilities where sufficient |
| Profile-source priority and combination | PromptKit selects tiers; ConfigKit owns strict within-tier composition and recursive cross-tier overlay; eliminate repository fallback and whole-definition shadowing |
| Named profile inheritance | Delegate sibling graph resolution, combination, and derivation to ConfigKit's `extends`; remove the custom profile resolver |
| Backend selection and domain normalization | Retain PromptKit domain rules; use recorded application resolution for configuration transformations not already handled by structural inheritance |
| LLM setting bounds, role/session normalization, output rules, reserved provider fields | Retain shared PromptKit domain owners; ConfigKit validation callbacks may invoke them |
| Definition defaults | Use ConfigKit defaults where they do not erase inheritance presence; apply effective execution defaults after inheritance |
| Programmatic configuration | Prefer labeled mapping input with explicit presence; evaluate whether typed PromptKit convenience constructors justify conversion code, without YAML round trips or zero-value inference |
| Runtime clients, observers, artifact readers, request data | Retain normal Go injection and request APIs; do not load them through ConfigKit |
| Credentials | Retain optional environment-name and direct request-key semantics; do not replace these with ConfigKit's required typed secret resolution |
| Catalog manifest, ownership, reserved names, secret-safety checks | Retain the catalog adapter; reuse the common ConfigKit profile decoder |
| JSON Schema resources and generated-output validation | Retain JSON Schema implementation; these are not application configuration documents |
| Application imports, CLI overrides, deployment overlays, secrets-directory bootstrap | Leave with consuming applications unless a concrete PromptKit need is accepted |
| Generic source/decoding diagnostics | Reuse ConfigKit; add only PromptKit identity/context and domain relationship diagnostics |
## 3. Recommended Target Loading Model
### Definition Snapshot
Create one immutable definition snapshot during engine construction. Discover
the explicitly supplied sources, assemble their contributions through ConfigKit,
decode the effective configuration, normalize domain values, and resolve relationships
before publishing the engine. No provider call or credential availability check
belongs in this phase. Publish no partially initialized engine on failure.
Prefer keyed collections, such as `profiles.<name>`, assembled before one typed
loading lifecycle instead of independently decoding each profile and recombining
the results. Multiple files can contribute to those collections; this does not
require a monolithic file. Reconsider the previous one-typed-document-per-definition
recommendation. Settle prompt ID/version representation and catalog format migration
in the feature roadmap. Support OS directories, individual files, embedded/custom
`fs.FS` sources, and programmatic input through the new source model; source selection
and definition discovery remain PromptKit concerns, not a second composition engine.
ConfigKit expands structural inheritance before typed defaults/decoding. Perform
any remaining domain-specific resolution through its recorded lifecycle before
semantic validation and result publication. Reuse its presence and provenance;
do not add a generic merge/defaulting pass or reconstruct origins after loading.
Runtime client assembly and resource interpretation remain outside typed decoding.
Load and freeze referenced message text with definitions. Validate static prompt
structure and template syntax at loading, using the same template function setup
as execution; render only when request variables and artifacts exist. Preserve
source-relative content resolution and containment without expanding the threat
model. A snapshot is a set of captured reads, not an atomic filesystem transaction.
Resolve each effective content reference using its contributing source's context,
not the highest tier's directory. Use ConfigKit's released direct context
query rather than a consumer-maintained filesystem registry. Composite origins
must not be assigned an arbitrary directory; ambiguous or absent context needs
an actionable resource error or explicit consumer-supplied context. ConfigKit
does not snapshot external text: PromptKit opens it explicitly and freezes it
before engine publication. Caller-owned filesystem lifetimes remain caller-owned;
close resource handles acquired by PromptKit without retaining ConfigKit load handles.
Keep request artifacts and schema compilation at preparation. A request can
replace the output contract, so an unused default schema should not need eager
compilation. Prepared execution continues to freeze its effective resources.
The snapshot covers definitions/message text, not environment values, request
files, every schema file, or execution state.
### Composition And Validation
Use explicit source categories in the existing useful priority order:
programmatic profiles, operator source, application fallback, PromptKit-owned
profiles, then external catalogs. Express these as explicit ordered tiers, not
source discovery order. Within a tier, mappings compose recursively and overlapping
non-mapping contributions conflict, even when equal. Disjoint contributions to
the same profile may therefore combine; do not preserve a blanket duplicate-ID
rejection. Across tiers, mappings overlay recursively, retaining omitted lower
children. Effective prompt identities/versions must remain addressable, with an
omitted version ambiguous when multiple versions exist; finalize their keyed
representation without reintroducing whole-definition shadowing.
All sources undergo syntax, acquisition, and composition checks; higher tiers
cannot hide those failures. Decode and validate effective values after assembly,
not every intermediate tier against the final typed schema. A replaced lower
value may consequently have a type that would not fit the final schema. An unused
invalid effective definition still fails construction. Catalog ownership,
reserved-name, and secret-safety checks remain distinct source-policy requirements,
not a reason to impose intermediate typed validation on ordinary inputs.
Preserve profile templates that become usable through descendants or runtime
overrides: eager loading must not require every consumer profile to be an
independently runnable endpoint/model pair. Validate supplied fields and graph
integrity eagerly (ConfigKit handles missing bases and cycles), and check final execution
requirements after runtime selection/overrides. Prompt default-profile references
can be checked against the assembled collection; a prompt with no default remains
valid for callers that supply a profile explicitly.
ConfigKit's ordering is typed defaults, source-underlay tiers, strictly composed
main input, selected overlay, then CLI overrides. PromptKit's API must map its
source priorities explicitly onto supported roles; do not mislabel programmatic
input as CLI flags. Underlay-only loading avoids inventing a main document.
Imports remain opt-in, and the named-overlay convenience need not be exposed by
PromptKit merely because ConfigKit supports it.
A prompt's `version` is a domain identifier, not automatically a ConfigKit
application-schema version. Do not configure `VersionRule` for prompt versions.
If a separate configuration schema version is later needed, respect ConfigKit's
designated main-input ownership and pre-underlay check timing; do not assume
last-wins version selection across catalogs.
### Source Presence And Released Sibling Inheritance
Adopt ConfigKit source-layer semantics: absence retains lower input; explicit
zero, false, and empty strings replace it; mappings combine recursively; empty
mappings retain lower children; sequences replace as units; null explicitly
replaces, subject to final schema legality. No reset/delete operator is assumed.
Preserve presence through domain resolution, backend/framework
default application, request overrides, and outbound request construction.
Otherwise accepting `temperature: 0` during decoding would accomplish nothing.
Use ConfigKit metadata for both file and programmatic input. A final Go value
alone does not distinguish absent from explicit zero. Avoid eagerly defaulting
profile fields in ways that erase that distinction before relationship resolution.
Provider `extra_params` maps combine recursively across source tiers too: `{}`
does not clear lower keys. Retire the proposed atomic source-map replacement and
null-as-unspecified conventions; do not emulate them after loading. An operator
overriding only `model` also retains a lower `extends` declaration and other
omitted settings. Diagnose and document these consequences explicitly.
Replace `base_profile` with `extends` on sibling profile mappings. A string names
one exact immediate sibling; null cancels the relationship while preserving the
child's own settings. Omission retains a lower-tier directive. There is no ancestor
scope search or cross-collection reference. Superseded references are history,
not active relationships to validate; every effective relationship resolves,
including profiles not selected for execution.
Children recursively overlay their resolved parents with the same value rules:
zero/false/empty strings and sequences/null replace, mappings combine, and `{}`
does not clear inherited keys. Inherited values count as explicit input for
presence queries and absence-based defaults, even when false, zero, or null.
Do not repeat inheritance in PromptKit's application-resolution callback. Use
`Origin.Inheritance` and retained history for explicit operator inspection.
Inherited resource references retain their declaring source's context.
Enclosing inheritance resolves before nested relationships; nested inherited
directives resolve at each destination against its siblings. Defaults and
application-derived values do not trigger another inheritance pass. Model this
deliberately in PromptKit configuration rather than promising arbitrary graph
resolution over runtime values. Partial bases must fit the typed schema and
PromptKit's validation; ConfigKit grants them no special validation exemption.
`extends` is reserved in every input mapping, including provider-parameter maps,
across all source kinds. There is no opt-in switch or ordinary-data escape in the
released contract. Audit catalog and consumer payload needs before finalizing
their format; a literal provider field with that name cannot silently pass through
the configuration loader. Do not invent an encoding workaround in this assessment.
The existing request-level override API remains a separate runtime boundary.
The feature roadmap must review its interaction with configured values without
silently subjecting runtime request data to configuration inheritance directives.
### Reloading, Errors, And Application Integration
Later definition edits take effect through a newly constructed engine. Do not add
watchers, mutable registry reloads, or a second lazy compatibility mode. Applications
choose how to swap engines and let in-flight work finish. Old prepared handles
remain associated with their original engine. Simultaneously active engines have
separate backend capacity pools; migration guidance must not imply a global limit
across a reload overlap.
Return configuration failures from construction with file/path/location and
useful definition identity where available. Recommend preserving ConfigKit's
public `LoadError` through wrapping for `errors.As`, together with a PromptKit
configuration category; do not copy its entire diagnostic API into the facade.
Domain-only relationship failures remain PromptKit errors with relevant source
locations. Start with deterministic fail-fast loading across files and ConfigKit's
existing multi-diagnostic behavior within a load, rather than inventing a batch
diagnostic subsystem. Do not expose raw credentials or add implicit logging.
Runtime failures continue to distinguish invalid requests, missing selections,
rendering/artifact/schema errors, provider/transport errors, capacity rejection,
and content-validation results. Cancellation remains effective at
PromptKit-controlled boundaries; ConfigKit's current synchronous `Load` has no
context argument, so do not promise interruption inside it or create goroutines
to simulate that guarantee.
Applications using ConfigKit can present the same underlying loading diagnostics
for their application settings and PromptKit files. They can also translate
application-owned typed settings into PromptKit profiles/backends. Do not require
applications to encode Go clients into YAML or adopt ConfigKit merely to call
PromptKit's runtime API. The operator-inspection capability below supplies the
concrete consumer purpose for exposing effective configuration and its provenance.
### Explicit Configuration Inspection
Provide a consumer-facing API for inspecting effective configuration and its
provenance so applications can implement targeted CLI commands or debug output.
Keep three activities distinct:
- **Construction:** load and resolve configuration, retain ConfigKit's provenance
and inspection state, and publish execution-ready definitions. Resolve any
presence and origin information needed for definition construction here.
- **Ordinary execution:** use the ready definitions directly. Do not analyze
configuration provenance, traverse origin/history records, reconstruct
inheritance explanations, or render effective-configuration reports as part of
a prompt run. Carry execution-relevant presence explicitly in resolved values
rather than querying configuration metadata during each run.
- **Explicit inspection:** query effective settings and their contributing
sources or explanations only when a consumer invokes the inspection API.
Inspection uses the same loaded configuration snapshot as the engine; it does
not reread configuration files, open referenced resources, or contact providers.
Reuse ConfigKit's inspection and redaction facilities rather than maintaining a
second provenance system. The API should make targeted queries practical; exact
signatures, query scope, and safe result ownership belong in the feature roadmap.
Do not imply that configuration inspection predicts all runtime request overrides
or reveals the values of execution-time credentials. Existing prompt/profile
inspection, prepared-run details, and execution observations have distinct purposes;
configuration inspection does not replace them or add provenance to run events.
Applications decide whether and how to expose inspection through operator tools.
PromptKit adds no CLI, implicit debug logging, or automatic explanation output.
Redacted configuration can still contain sensitive application content, so callers
retain responsibility for access control and disclosure. Retention has a memory
cost, but explanation generation and provenance analysis stay off the run path.
## 4. Simplification Accounting
| Current implementation | Expected disposition |
| --- | --- |
| `internal/profile/filesystem_repository.go`: metadata scans, selected strict decode, stream-end checks, malformed-file selection arbitration | Replace with source acquisition, ConfigKit assembly/decoding, and indexed effective values; remove the metadata prepass and per-lookup filesystem path |
| `internal/promptdef/filesystem_repository.go`: metadata selection and repeated decoding/content opening | Replace with eager prompt loads and an ID/version index; keep domain normalization and contained content reading |
| Profile filesystem, `fs.FS`, eager, memory, and overlay repository variants | Collapse loading into source adapters plus immutable collections; preserve only interfaces that still serve a real boundary |
| `internal/profile/resolving_repository.go` | Remove custom graph traversal, cycle detection, and zero-sentinel merging in favor of ConfigKit sibling inheritance; retain only PromptKit domain validation |
| `internal/engine` raw repository-stack assembly | Replace with ConfigKit source tiers and one effective typed collection; engine capacity/client assembly remains |
| `internal/catalog` profile metadata extraction | Obtain explicit field information from ConfigKit; retain manifests, ownership, namespace, connection-metadata and secret checks |
| `internal/filecatalog` | Retain/reduce discovery and containment helpers actually needed by the eager loader; do not claim the entire package disappears |
| `internal/domain` source YAML tags and runtime/source coupling | Separate configuration DTOs from execution values where presence requires it; reuse domain validation rather than duplicating it |
| Direct `gopkg.in/yaml.v3` use for definition loading | Remove once all such consumers migrate; ConfigKit brings its own private YAML dependency |
New integration work is real: collection DTOs, ConfigKit calls and error wrapping,
source-context access, collection indexing, programmatic mapping ergonomics, and
conversion to runtime definitions. Catalog documents may need new keyed shapes;
do not hide that migration behind a permanent second loader.
Some defensive copies remain necessary at public/injected-client boundaries;
ConfigKit's detached load result is not permission to expose shared engine maps.
`internal/jsonvalue` still serves programmatic values, request parameters, and
prepared schema ownership, so adoption does not eliminate it.
Maintenance savings depend on actually deleting the old lazy paths and their
special-case tests. Move generic YAML/scalar behavior coverage to ConfigKit's
ownership; PromptKit retains representative adapter tests plus domain, precedence,
snapshot, inheritance, embedded-source, error, and execution integration tests.
Do not duplicate ConfigKit's entire parser suite or discard execution regression
coverage because construction fixtures need migration.
Runtime benefit: ordinary runs no longer rediscover definition directories,
redecode definitions, reread template text, or traverse inheritance chains.
Cost: construction loads even unused definitions and retains their text and
resolved values. Retain ConfigKit inspection state needed by the explicit operator
API without duplicating its history in runtime definitions or per-run values.
These are reasoned tradeoffs, not measured performance
claims. Adoption is not justified by the decoder replacement alone.
### Algorithmic Efficiency
Efficiency is an explicit design criterion for both the upstream work and this
shared-library refactor. Prefer optimal or near-optimal algorithms relative to
input and produced-data size, not a blanket O(n) claim with an undefined n.
- Acquire/parse each accepted source occurrence once, assemble input, and run
one typed loading lifecycle rather than decoding intermediate tiers. Reuse
captured content instead of reparsing it for metadata, selection, and validation.
- Build identity indexes in one pass, with expected constant-time map lookup.
Do not scan whole collections for every lookup or precedence decision.
- Delegate inheritance expansion to ConfigKit once during loading rather than
rebuilding its graph in PromptKit. Assess scaling against input, materialized
inherited values, nested destination resolution, and retained history; do not
claim unconditional O(V + E) total work from the count of profiles and edges.
ConfigKit bounds expansion and retained evidence through its existing budgets;
representative catalog sizes must fit them without new per-run processing.
- Avoid whole-collection copying per layer, repeated serialization, and repeated
prefix/history traversal. Analyze layer processing against total input nodes
and produced values/provenance, including intentionally retained history.
- Keep configuration discovery, decoding, and provenance analysis out of ordinary
execution. Perform explanation work only on explicit inspection calls. Copy only
where ownership requires it; do not trade caller isolation for speed.
These are design targets, not promised complexity for every operation. Sorting
for deterministic discovery may require O(F log F) for F files, and processing
message text costs work proportional to its bytes. Construction time, retained
memory, and frequent-call costs all matter. Prefer straightforward indexes and
clear immutable ownership over speculative global caches or intricate optimizers.
The implementation plan should include focused benchmarks for consequential
repeated-work risks, such as shared-ancestor profiles and many layers, rather than
machine-dependent timing gates or benchmarks for every helper.
## 5. Upstream Readiness And Remaining Opportunities
Filesystem sources, underlays, programmatic mappings, source contexts, and sibling
inheritance are released in v0.5.0. No additional upstream feature is currently a
demonstrated prerequisite. Context-aware loading remains a possible future request
if concrete consumers need it, not a reason to delay adoption. Existing loading
is synchronous; do not simulate cancellation with detached background work.
Do not propose a public raw AST, permissive identity-only decoder, or dynamic
subtree conversion solely to preserve PromptKit's current scanning design.
Do not upstream backend/model selection, credential timing for inference,
profile catalog reservations, or prompt-template functions.
ConfigKit and PromptKit both declare LGPL-3.0-or-later. Adoption must still update
dependency notices under the OSS policy and inspect the exact selected release
and its transitive notices. Their current Go minimums do not require a PromptKit
toolchain increase. Use a released module dependency, not a sibling-directory
replacement. Select v0.5.0 as the assessed adoption baseline; any newer selection
needs its own review. This assessment changes neither dependencies nor licenses.
## 6. Decisions And Migration
The agreed direction now incorporates ConfigKit's v0.5.0 contracts. This supersedes
the earlier recommendations for independently typed documents, whole-definition
shadowing, intermediate typed validation, and atomic maps/null-as-unspecified at
the source boundary.
1. **Loading lifetime and failure scope.** Keep eager construction and frozen
definitions/content. Syntax/acquisition/composition failures remain unmaskable;
typed and domain validation apply to effective values. A separate reusable
public snapshot lifecycle remains possible but is not currently needed.
2. **Source organization.** Prefer keyed collections composed across files and
tiers before typed loading. Settle exact profile and prompt/version shapes,
source-role mapping, and catalog migration in the feature roadmap. Independent
typed loads followed by custom combination are not the selected direction.
3. **Presence and relationships.** Accept recursive source-tier overlay, retained
children for empty mappings, explicit null replacement, and effective metadata.
Adopt built-in sibling inheritance, including reserved `extends`, recursive
mapping combination, null cancellation, and inherited explicit presence. Do
not preserve a parallel PromptKit structural resolver.
4. **Diagnostic boundary.** Keep wrapping ConfigKit's public loading errors rather
than mirroring them. Retain actionable domain relationship and resource errors
without duplicating ConfigKit's generic diagnostic/provenance system.
5. **Operator inspection.** Expose effective configuration and provenance through
an explicit consumer API backed by the engine's loaded snapshot. Retain the
required ConfigKit inspection state at construction; never analyze it during
ordinary prompt execution. Finalize targeted query shapes and ownership in the
feature roadmap, with presentation remaining application-owned.
Use released mapping-based programmatic input and direct filesystem resource-context
queries. If real consumer examples show
mapping construction to be repetitive, consider small typed PromptKit conveniences
with explicit presence rather than requiring an upstream typed-ingestion API.
Remaining PromptKit decisions are the exact collection/version format, catalog
migration, programmatic ergonomics, operator-inspection surface, and validation
of partial base profiles. Also settle how operators discard unwanted provider
settings or old relationships using the released null semantics without a hidden
replacement pass. Check the reserved-key impact on arbitrary provider maps. These
are integration decisions, not unresolved ConfigKit API proposals.
Expected consumer migration:
- Update definitions for ConfigKit's strict scalar rules, rejected YAML
anchors/aliases/tags, input limits, and null/type checks. Quote prompt version
strings rather than introducing a special numeric-version conversion merely
for historical YAML coercion.
- Move standalone definitions into the agreed keyed contribution format. Review
same-name contributions: disjoint fields within a tier can combine, and higher
tiers retain omitted lower fields, including `extends` references. Convert
`base_profile` to sibling `extends`; use `extends: null` to cancel a lower-tier
relationship, not to erase all settings already supplied to the child.
- Review zero/false/empty/null values explicitly. Provider maps merge across
tiers, `{}` does not clear them, and null replaces rather than means omission.
Use labeled programmatic mappings or approved presence-aware conveniences.
Apply those same combination rules to inherited settings. Rename ordinary
configuration keys named `extends`; examine provider maps for cases where
renaming would change their substantive meaning and resolve those explicitly.
- Handle effective-definition errors at construction, fix syntax/composition
failures even in overridden inputs, and deliberately rebuild/swap engines
after definition changes.
- Keep runtime credential lookup optional, avoid baking resolved secrets into
definition snapshots, and preserve request-scoped key overrides.
- Update maintained catalog data for collection shapes and any incompatible syntax;
coordinate catalog releases before the PromptKit dependency update. Preserve
maintained profile names and catalog ownership rules, not every old scalar
spelling. Do not validate live provider/model availability.
- Update error inspection, constructor/source options, and programmatic profile
literals as required. Keep representative preparation, execution, inspection,
custom backend, fallback, and embedded workflows in the migration guidance.
The subsequent feature roadmap should carry forward these choices, resolve the
remaining PromptKit integration decisions, and specify the target end state.
The staged implementation plan can now adopt released ConfigKit support without
waiting for upstream feature work, coordinating any catalog format releases before
their consumption. Cover deletions/replacements, contract tests, canonical documentation,
examples, attribution, and minor-release migration notes. Neither document should
become a reason to preserve obsolete compatibility machinery.

View File

@@ -1,339 +0,0 @@
# ConfigKit Adoption
Status: agreed feature roadmap. ConfigKit v0.5.0 supplies the required upstream
capabilities. The [implementation plan](implementation.md) owns execution steps.
The [configuration-boundary ADR](../adr/0001-configuration-boundary.md) records
the handoff decision. This document owns the intended end state, not
implementation authorization.
## Purpose
Use ConfigKit to standardize PromptKit's configuration semantics and simplify
its loading machinery, replacing custom logic wherever ConfigKit is the
appropriate owner. Preserve substantive consumer capabilities, not historical
file syntax, loading order, or Go signatures. Pre-1.0 breaking changes are
acceptable and must have migration guidance.
**This is an intentional breaking change. We are intentionally replacing PromptKit's configuration handling code in favor of adopting ConfigKit's simpler, standardized API. Our goal is to arrive at a new version of PromptKit that is leaner, simpler, and uses a ConfigKit-standard configuration syntax. We MUST NOT give any weight to preserving arbitrary implementation details from PromptKit's current codebase--those should be discarded in favor of the simplest viable implementation that supports PromptKit's substantive LLM-focused consumer capabilities.**
Applications using both libraries should share configuration interpretation,
layering, inheritance, presence, and inspection semantics. PromptKit should
concentrate on LLM-specific validation, resource interpretation, and execution.
The [adoption assessment](configkit-assessment.md) records the supporting analysis;
this roadmap owns the agreed feature end state. Where the earlier assessment
differs, this roadmap and the maintainer's direction above take precedence.
Do not preserve source-option wrappers, repository interfaces, incidental error
ordering, or old field shapes solely because they exist today. Reuse existing
code only when it remains the simplest appropriate owner of a substantive
capability. Migration documentation replaces compatibility machinery, not vice versa.
## Baseline And Dependency
The current PromptKit runtime baseline is v0.11.2. Its
[source implementation](../internal/sources.md), [format contract](../formats.md),
and [consumer guide](../consumers/pkg-promptkit.md) describe existing behavior,
not the target described here.
The reviewed ConfigKit release is v0.5.0 at
`dc09959cca4e105f49ec316c6d4ba585bd2340bd`. It provides native `fs.FS` sources,
`LoadLayered`, source-backed underlays, labeled programmatic mappings, passive
resource contexts, and built-in sibling inheritance. Its released configuration
contract and GoDoc own exact upstream semantics. No further upstream feature is
currently a demonstrated prerequisite. Review any newer version before selection.
Adopt a released Go module, not a local `replace` or workspace requirement.
Update attribution and transitive license notices under the OSS policy. This
feature does not change either project's license or require a Go minimum increase
relative to the reviewed versions.
## Substantive Capabilities To Preserve
- Named/versioned prompts, declared-input inspection, templates, inline and
referenced message content, variables, input helpers, roles, and cache control.
- Maintained catalogs, application defaults, operator configuration, programmatic
profiles, aliases/inheritance, built-in/custom/local backends, and endpoint-only
profiles.
- Preparation without generation, ordinary execution, and frozen single-claim
prepared execution; independent prompt and profile inspection.
- Runtime execution-setting, reasoning, session, credential, and output-contract
overrides; appended already-rendered messages.
- Inline/file input artifacts and injected artifact, model, and HTTP collaborators.
- Backend capacity management, cancellation, transient retries, structural output
validation, provider-native structured output, bounded repair, and observations.
- Results, effective identities/settings, usage, hashes, and meaningful errors.
Retain optional environment credentials and direct request credentials. ConfigKit
secret resolution must not introduce a new requirement that an upstream API key
exist. Domain validation must not probe provider/model availability.
## Target Configuration Model
### One Shared Loading Lifecycle
Construction supplies selected configuration inputs to ConfigKit, which composes,
overlays, expands inheritance, and decodes them. PromptKit interprets and validates
the final effective configuration at the ConfigKit handoff and produces
execution-ready definitions. ConfigKit's effective-value resolution and validation
hooks may implement that handoff. Publish no partially initialized engine on failure.
ConfigKit owns generic parsing, combination, presence, origins, and inheritance.
PromptKit owns input selection/discovery, definition indexing, LLM-specific rules,
and translation of final values into runtime settings. Input adapters select or
encode contributions; they do not inspect or validate raw source contents.
Effective presence/null information and resource context supplied by ConfigKit
support interpretation at handoff. Historical inspection remains an explicitly
invoked operator capability, separate from domain enforcement.
Runtime clients and callbacks remain normal Go dependencies, not YAML types.
Use `backends.<id>`, `profiles.<id>`, and `prompts.<id>.<version>` keyed mappings
assembled before typed decoding rather than independently decoding definitions and combining them
afterward. Keys own identity; do not duplicate `id`/`version` fields. These paths
describe mapping levels, not dot-splitting of literal IDs or version keys.
Multiple files can contribute to a collection; a monolithic file is not required.
When a prompt ID has multiple versions, selection without a version remains
ambiguous. This shape fits sibling inheritance without extra identity reconciliation.
Supply explicit source groups through a small construction surface compatible
with ConfigKit's `Input`/underlay concepts. Offer convenient file/directory/embedded
entry points where useful, not a parallel set of historical loading semantics.
Accept ConfigKit's labeled mapping sources for programmatic configuration. Add
only small typed conveniences justified by the minimal local-backend and
application-profile workflows. Helpers encode explicit presence and feed the same
loading lifecycle; they neither merge configurations nor infer omission from zeros.
Backend helpers also produce inputs to this lifecycle. Backend definitions,
including catalog connection and capacity settings, cross the same ConfigKit
handoff as prompts and profiles. The immutable runtime registry is built from
effective backend values. HTTP/model clients and other runtime collaborators
remain outside typed configuration.
Keep the public package a deliberate facade and implementation private. Do not
expose internal engine/domain types or introduce a parallel generic loader merely
to preserve old source options. ConfigKit's supported public types may be used
where intentional integration is simpler than maintaining equivalent wrappers.
### Sources, Tiers, And Presence
Support ordinary files/directories, embedded or caller-supplied `fs.FS`, and
programmatic definitions. Explicitly selected directories may be discovered by
PromptKit; ConfigKit handles their configuration contributions. Do not infer
priority from traversal order.
Retain the useful priority categories, lowest to highest: external
catalogs, PromptKit-owned profiles, application fallback, operator configuration,
and programmatic configuration. Express intentional priority as ordered tiers.
Within each tier, independent contributions compose strictly; overlapping
non-mapping values conflict, even when equal. Disjoint fields under the same
identity can combine. Across tiers, mappings recursively overlay.
These rules also govern backend configuration; consumers can add backend IDs or
overlay existing definitions through their selected tiers. Registry construction
validates the resulting values, not the identity of their contributors.
Use ConfigKit's source roles honestly: underlay groups are ordered defaults,
main input overlays underlays, and optional named overlay/CLI facilities retain
their upstream meaning. Underlay-only loading is supported; do not manufacture
a main document or disguise programmatic values as CLI flags. Exposing every
ConfigKit convenience through PromptKit is not a requirement.
Adopt the shared value rules: omission retains lower input; explicit zero, false,
empty string, sequence, and null replace; mappings combine recursively; empty
mappings do not clear lower children. Null remains explicit and subject to the
final schema. Provider-parameter maps obey these rules too. No hidden whole-profile
replacement, arbitrary merge callback, or new reset/delete operator belongs here.
Preserve presence into execution-ready settings so an explicit zero does not
disappear during backend/default resolution or outbound request construction.
Use ConfigKit metadata at construction, not as a per-run lookup mechanism.
### Sibling Inheritance
Use ConfigKit's reserved `extends` rather than PromptKit's `base_profile` and
custom resolver. A string identifies an immediate sibling mapping; null cancels
the relationship without deleting the child's own settings. Layering completes
before inheritance. Only effective relationships resolve, but all of them do,
including entries never selected for execution.
Children recursively overlay resolved parents. Inherited values count as explicit
input. Origins and resource context remain attached to their declaring source.
Nested relationships follow ConfigKit's destination-local rules; defaults and
application-derived values do not trigger another inheritance pass.
The keyword is reserved throughout configuration, including dynamic provider
maps; it is not opt-in. Keep runtime request provider-parameter maps ordinary
JSON-compatible data outside configuration parsing. Consumers needing a literal
provider field named `extends` supply it at that runtime boundary. Do not rename
the outbound key or introduce a second opaque configuration format to escape it.
Document and test this distinction, including any affected maintained payloads.
### Validation And Errors
ConfigKit owns acquisition, syntax, composition, inheritance, and typed-decoding
failures. PromptKit preserves those errors and validates final effective domain
values. An unused invalid effective definition fails construction; a lower-layer
value legitimately replaced by ConfigKit is not a separate validation target.
This boundary applies equally to application and maintained catalog configuration.
PromptKit does not perform catalog preflight, raw-source or intermediate-state
validation, or history-based enforcement, including equivalent test-only checks.
Catalog maintainers own publishing checks in their respective repositories.
Domain validators remain shared with runtime request validation where appropriate.
Allow schema-valid partial profiles without an `abstract` flag. Validate supplied
domain values at loading; final runnable-target requirements are checked after
profile selection and runtime overrides at execution preparation.
Do not require credentials, artifacts, or provider connectivity during construction.
Preserve absence versus explicit null in resolved settings. Null on an optional
provider setting deliberately suppresses it; it must not restore the same setting
from backend defaults. A null provider map suppresses the configured lower map,
while null entries inside a provider map remain literal JSON null values. This
supports reusable bases and deliberate removal of stale settings without changing
ConfigKit's merge rules. Framework-required settings such as timeouts have explicit
field-specific null semantics rather than blanket nil defaulting.
Preserve ConfigKit public loading errors through wrapping for `errors.As`, with a
PromptKit configuration category. Use source-aware domain/resource failures where
ConfigKit is not the owner. Do not mirror its diagnostic framework or promise
aggregation beyond what the underlying operation provides. Runtime operational
errors remain distinct from generated-content validation failures.
Prompt versions are domain identifiers, not ConfigKit `VersionRule` declarations.
Do not introduce an application schema-version policy merely to adopt ConfigKit.
## Snapshot, Resource, And Execution Boundaries
The engine captures effective definitions and referenced message text during
construction. Validate static template syntax then; render with request data
later. Resolve content references from the effective field's contributing source
context, not the last layer's directory. Ambiguous/contextless resource references
need an explicit source context or an actionable error, not an invented directory.
ConfigKit returns passive resource contexts, not snapshots of referenced text.
PromptKit explicitly opens and closes its resource handles. Caller filesystem
ownership remains unchanged. Preserve appropriate source containment without
claiming that every `fs.FS` is a security sandbox. Captured reads are not an atomic
transaction over a concurrently changing filesystem.
Definition changes require new engine construction. No watcher, background
refresh, or lazy compatibility mode is added. Applications own engine replacement;
prepared handles remain bound to their original engine and overlapping engines
retain separate capacity pools.
Request artifacts and schema preparation remain invocation work. A request can
replace the output contract, so construction need not compile unused schemas.
Prepared execution continues to freeze its effective resources. Environment
credential lookup remains at the established execution/preparation boundary,
not frozen into the configuration snapshot.
Ordinary execution uses indexed ready definitions and resolved presence values.
It performs no configuration discovery, parsing, inheritance traversal, or
provenance analysis. Context cancellation remains cooperative at controlled
boundaries; do not add detached work to pretend synchronous ConfigKit loading is
instantly interruptible.
## Explicit Operator Inspection
Add an API consumers can use to implement targeted CLI commands or debug output
for effective configuration, contributing sources, and explanations. Retain
ConfigKit inspection state with the engine's configuration snapshot and reuse
its redaction and provenance facilities.
Inspection is explicitly invoked and uses that snapshot without reloading files,
opening resources, or contacting providers. It does not predict every runtime
override, reveal execution-time credential values, or replace prepared-run details
and execution observations. No provenance traversal, explanation generation, or
configuration-report rendering occurs during ordinary prompt runs.
Expose a read-only inspection view backed by retained ConfigKit state, with
targeted path queries, source/history information, and explicit redacted effective
and explanation output. Reuse suitable public ConfigKit inspection values rather
than mirroring them. Do not expose mutable runtime definitions, resource capabilities
merely for diagnostics, or request credentials. The scope is generic targeted
inspection, not just narrow profile/prompt origin summaries.
PromptKit ships no CLI and emits no automatic debug reports. Applications own
presentation, access control, and disclosure; redaction does not make all prompt
or configuration content safe to publish.
## Required Refactoring And Coordination
- Replace profile/prompt metadata prepasses, selected-file decoding, and per-call
scans with ConfigKit collection loading and immutable indexed definitions.
- Remove the custom profile inheritance resolver and error-driven repository
overlay stack. Do not retain an old-format compatibility engine alongside the
new semantics.
- Route catalog and programmatic backend definitions through ConfigKit; retire
the separate backend-manifest decoder and pre-handoff registration validation.
Retain backend endpoint, capacity, and provider-parameter domain validation on
the final effective values.
- Retain only needed discovery and message-resource adapters; remove direct YAML
dependency usage once no remaining owner needs it. Preserve JSON Schema,
runtime JSON-value ownership, domain validation, and execution collaborators.
- Adapt maintained catalogs and PromptKit-owned definitions to the agreed keyed
format and released inheritance semantics, including keyed backend assets.
Preserve maintained name commitments and the intended shipped backend settings.
Catalog source-policy enforcement belongs to catalog maintainers. Coordinate
compatible catalog releases before consuming the changed asset format.
Publish new keyed assets directly in the existing data-only catalog projects,
recording the format transition rather than maintaining duplicate legacy assets
or a permanent conversion adapter. Catalog maintainers select release/version
handling under their own policies; old consumers must not be assumed capable
of decoding the new shape. Pin the compatible released versions in PromptKit.
This document requests no immediate changes in those repositories.
- Keep configuration conversion, normalization, and inspection connected to
ConfigKit's lifecycle instead of reconstructing effective values/history after
loading. Translate once into execution-owned values with necessary copying.
- Update canonical formats, consumer guidance, GoDoc, internal inventory, examples,
architecture boundaries, dependencies, and OSS notices when implementation lands.
## Efficiency And Validation Expectations
Target linear or near-linear work relative to actual input, output, and retained
history, accounting separately for sorting, resource reads, and domain work.
Acquire/parse each accepted source occurrence once, avoid decoding intermediate
tiers, and index definition selection. Do not duplicate inheritance expansion or
copy complete accumulated collections for every added layer.
Materialized inherited data and provenance can exceed source size. Exercise real
catalog shapes within ConfigKit's budgets; do not claim unconditional O(n) runtime
or memory. Prefer clear ownership and straightforward indexes over global caches
or elaborate optimization. Operator inspection may perform work proportional to
the requested report; that work remains off the execution path.
Tests should use real ConfigKit with offline sources and synthetic provider
collaborators. PromptKit owns its adapter, domain, lifecycle, and execution tests,
not a duplicate ConfigKit parser/graph suite. Retire tests whose only purpose was
the removed loading machinery or source-policy enforcement. Exercise catalog
integration through the final assembled configuration and resulting behavior.
Preserve meaningful execution regressions.
Representative acceptance evidence must cover:
- Minimal local-backend construction and execution without elaborate configuration.
- Embedded defaults plus operator and programmatic contributions, including
accurate inherited relative-resource origins.
- Profile inheritance, changing model/backend, canceling old relationships, and
discarding unwanted settings with documented semantics.
- Explicit zero/false/null behavior reaching execution, construction failures,
independent snapshots, and unchanged substantive preparation/execution features.
- Targeted inspection with useful redacted origins and no provenance work during
ordinary runs.
Use focused scaling/allocation benchmarks for large collections, shared bases,
and many tiers where they expose repeated-work risks. Do not add timing-based
release gates or provider-network tests.
## Migration And Completion
Ship as an intentionally incompatible minor release with concrete migration
guidance. Cover keyed files, `base_profile` to `extends`, reserved-key behavior,
recursive maps, explicit null versus omission, changed error timing, programmatic
construction, backend layering in place of registration-only restrictions, and
explicit engine replacement. Explain that overriding a model
does not discard lower endpoint, reasoning, provider, or inheritance settings.
The end state is one shared configuration model with the old loading/resolution
machinery removed, useful operator inspection, and all substantive execution
workflows still available. The six previously open design choices have been
approved and incorporated into the owning sections above. This is not a
compatibility-preserving reinterpretation of the existing configuration engine.

View File

@@ -42,20 +42,21 @@ checkpoint equality or when a broader semantic identity design is selected.
### Eager Source Validation
**Reason for deferral:** Exact prompt and profile inspection may already
provide a sufficiently small validation surface. Experience from downstream
adoption should establish whether an engine-wide operation would add enough
value to justify its broader contract.
provide a sufficiently small validation surface alongside construction-time
definition validation. Experience from downstream adoption should establish
whether a separate operation would add enough value to justify its broader
contract.
Promptkit could provide an explicit offline operation that discovers and
structurally validates configured prompt, profile, and schema sources without
model generation. The normal `NewEngine` path would remain lazy.
Promptkit could provide an explicit offline operation that also checks schema
resources without model generation. `NewEngine` already validates effective
definitions and captures prompt content; schema resources remain
request-dependent.
An eager operation would need coherent handling for duplicate prompt IDs and
versions, strict YAML decoding, referenced content files, profile/backend
membership, schema syntax and transitive references, context cancellation,
and source-specific public errors. Credential declarations must remain
separate from credential values; checking current environment availability,
if supported at all, should be an explicit option and must not expose secrets.
Such an operation would need coherent handling for schema syntax and transitive
references, context cancellation, and source-specific public errors. Credential
declarations must remain separate from credential values; checking current
environment availability, if supported at all, should be an explicit option
and must not expose secrets.
Reconsider this idea after downstream use of `InspectPrompt`,
`InspectProfile`, and fixture-based preparation demonstrates a concrete gap.

View File

@@ -1,642 +0,0 @@
# ConfigKit Adoption Implementation Plan
Status: ready for staged implementation under the agreed configuration boundary.
This is a planning document; writing it does not implement the feature or
authorize publishing dependencies.
## Authority And Working Method
The [ConfigKit feature roadmap](configkit.md) owns the intended end state and
approved policy. Its intentional-breaking-change direction is authoritative:
preserve substantive prompting capabilities, not the old loader, signatures,
source wrappers, incidental error ordering, or configuration syntax.
The [configuration-boundary ADR](../adr/0001-configuration-boundary.md) records
the ownership decision: PromptKit interprets and validates the final effective
configuration supplied by ConfigKit. Presence/null information and effective
resource context are part of that handoff. Source history serves explicit
operator inspection, not validation or catalog enforcement. This applies to
tests as well as production code.
Implement the stages below in order, one stage per coding-agent prompt. Each
stage includes its tests and affected current-state documentation. Read
[development](../development.md) and the applicable policies before starting;
in particular, follow [agents](../policy/agents.md),
[architecture](../policy/architecture.md), [testing](../policy/testing.md),
[documentation](../policy/documentation.md), and [OSS](../policy/oss.md).
Catalog work also follows each catalog repository's own instructions and release
procedure. Do not turn this plan's internal organization into new public promises.
Begin each stage by inspecting the worktree and earlier stages. Commit completed
units under the agent policy. Keep stages buildable; temporary private wiring
needed to connect successive stages must be removed by Stage 11. Do not ship a
legacy mode, dual loader, or old-format conversion layer. Do not add prototype
applications or a generic configuration framework of our own.
Planning baselines:
- PromptKit runtime v0.11.2, commit
`12d6d80024e80c552d122a076c22429bf6f2f6c2`; approved roadmap at `521291d`.
- ConfigKit v0.5.0, commit
`dc09959cca4e105f49ec316c6d4ba585bd2340bd`, module
`gitea.maximumdirect.net/eric/ConfigKit`.
- Both catalog dependencies currently select `/v2` at `v2.20260926.0`.
Use ConfigKit v0.5.0 for this work. Do not silently upgrade to a different
contract. The catalog asset migration needs coordinated releases, not a committed
workspace or local replacement. Changing the backend asset contract is broader
than the catalog profile-data compatibility exception: migrate the catalogs to
`/v3` with `v3.YYYYMMDD.N` tags, preserving their date/revision convention and
updating their own release documentation accordingly. Select exact dates and
unused revisions at publication; record versions and full commits consumed.
Publishing any repository requires separate authority.
## Implementation Decisions
### Construction Surface
Keep `NewEngine(Config, ...Option)` and the execution/injection surface where it
still serves its substantive purpose. Replace profile/prompt source options with
these fields and helpers; this is an intentional API break:
```go
type ConfigurationSources struct {
Underlays []configkit.Underlay
Main *configkit.Input
}
// Add Sources; remove PromptDir and ProfileDir.
// Retain SchemaDir, Timeout, HTTPClient, and TransientRetries.
type Config struct { /* existing non-definition settings; Sources ConfigurationSources */ }
func DirectoryInputs(directory string) ([]configkit.Input, error)
func FilesystemInputs(label string, filesystem fs.FS, root string) ([]configkit.Input, error)
```
The pointer distinguishes omitted main input without inspecting ConfigKit's
opaque Source. Nil main becomes `configkit.NoSource()`; a supplied invalid Input
is not silently ignored. Prepend two private underlays: external catalogs, then
PromptKit-owned definitions. Append caller underlays in their supplied order;
main is last and keeps its ConfigKit meaning. Imports follow each Input's flag.
Do not expose named overlays, CLI overrides, or VersionRule through another
PromptKit abstraction in this feature.
Show the usual application categories as caller underlays ordered fallback,
operator, programmatic, with main omitted. For a single highest-priority document,
main is also useful. These are explicit selections, not directory-name heuristics
or a second automatic precedence algorithm. Each directory helper returns inputs
for one strict-composition group, recursively finding `.yaml` and `.yml` files;
sorted discovery makes diagnostics reproducible, not one file higher priority.
An explicitly selected missing/unreadable root fails; an empty directory supplies
no contributions. Helpers set imports false; callers may enable them explicitly.
Use one ConfigKit filesystem namespace per helper invocation and root resource
contexts at the supplied directory. Single files use ConfigKit directly.
Remove `WithPromptFS`, `WithPromptFile`, `WithProfileFS`, `WithProfileFile`,
`WithFallbackProfileFS`, `WithProfiles`, and `WithBackend` at public cutover.
Retain schema source options and runtime collaborator injection. Backend
configuration moves into ConfigKit inputs as specified below. Do not require any
prompts to exist merely to construct an engine for profile inspection.
Programmatic configuration uses labeled `configkit.Mapping` inputs. Replace the
old zero-sentinel `Profile`/`OpenAICompatibleProfileConfig` pair with one small
`ProfileConfig` convenience and a source builder:
```go
type ProfileConfig struct {
Extends, Backend, Endpoint, Model *string
Temperature, TopP *float64
MaxTokens, TimeoutSeconds *int
ServiceTier, ReasoningEffort, APIKeyEnv *string
APIKeyRequired *bool
ExtraParams map[string]any
}
func ProfileSource(label string, profiles map[string]ProfileConfig) configkit.Source
```
Nil convenience fields mean omitted; nonnil pointers preserve zero/false/empty.
Nil ExtraParams omits the map; a nonnil empty map supplies an empty mapping, not a
clear operation. Use raw Mapping for explicit null, prompt definitions, or richer
configuration. Do not introduce a public generic optional-value type just for
this helper. Convert fields explicitly to ConfigKit-supported built-in values,
then call Mapping for snapshot ownership and load-time errors. Do not marshal
through YAML or infer absence from a value. Helpers encode values; domain
validation occurs on the final configuration at handoff.
Replace the registration-only Backend value with a presence-aware convenience:
```go
type BackendConfig struct {
Endpoint, APIKeyEnv *string
ConcurrencyLimit, QueueCapacity *int
ExtraParams map[string]any
}
func BackendSource(label string, backends map[string]BackendConfig) configkit.Source
func LocalBackend(endpoint string, concurrencyLimit int) BackendConfig
```
Use the same omission/pointer/map conventions as ProfileSource. LocalBackend
supplies endpoint and concurrency pointers and leaves the remaining fields
omitted; callers choose the ID as the map key. Neither helper validates settings
before ConfigKit processes them. For example, place BackendSource with
`map[string]BackendConfig{"local": LocalBackend(endpoint, 4)}` in an application
underlay. Raw Mapping supports explicit null and combined backend/profile/prompt
documents. Remove the old Backend registration value and its reservation rules;
retain useful built-in ID constants. All backend contributions follow the same
strict-composition and overlay rules as other configuration.
### Typed Configuration And Execution Values
Add `internal/configuration` as the cohesive ConfigKit adapter. Its private
document DTO has `backends: map[string]backendConfig`,
`profiles: map[string]profileConfig`, and
`prompts: map[string]map[string]promptConfig`. Collection entries are mappings,
not nullable definition pointers. No duplicate identity fields; ConfigKit consumes
`extends`, so no DTO field or second resolver handles it. Use `configkit.Keys`,
`Key`, and `Index` for semantic paths, never split IDs on dots.
Profile DTO fields match ProfileConfig's YAML spellings (`backend`,
`reasoning_effort`, etc.). Nullable pointers plus ConfigKit presence metadata
distinguish absent, value, and explicit null. Store that distinction in small
private execution-owned fields or presence bits, not retained metadata queries.
Runtime/domain packages must not import ConfigKit. Do not duplicate its tree,
merge engine, inheritance graph, diagnostics, or presence framework.
Backend DTO fields match BackendConfig using `endpoint`, `api_key_env`,
`concurrency_limit`, `queue_capacity`, and `extra_params`. Validate the final
endpoint as required, absolute HTTP(S) with the existing URL rules. An absent or
null API-key environment name means no lookup; a supplied name follows existing
normalization and naming rules. Absent/null concurrency means zero (unlimited);
supplied values must be nonnegative. Absent/null queue capacity selects the
existing default of 1024 for positive concurrency and no queue for unlimited
concurrency. Explicit queue capacity, including zero, requires positive
concurrency and must satisfy existing nonnegative/overflow checks. Null provider
maps mean no configured extra parameters; entries may contain JSON null.
Required endpoint failures, unknown fields, and domain errors apply to final
definitions, including unused ones. Registry construction consumes these final
normalized values; it does not replay registration or source-reservation checks.
Use ConfigKit's Resolve/Validate callbacks only on its effective post-layering,
post-inheritance typed values. These public lifecycle hooks implement the domain
handoff and record normalization; they are not hooks for inspecting earlier
contributions. Final presence lookup and resource-context lookup are sufficient
for interpretation. History/Explain/HistoricalPaths are operator inspection
facilities, not inputs to construction-time policy checks.
Use the existing prompt fields under each keyed version, without `id`/`version`.
Use pointers for `content`, `content_file`, and optional clearable strings;
preserve existing input, cache-control, output-contract, and role rules.
`messages` must be nonempty; each message has exactly one nonnull content source.
An explicitly supplied empty inline string is a valid template. Null content
removes that selection; null content_file removes its reference. Output remains a
required mapping with valid format and validation mode. Optional description,
session template, default profile, cache control, inputs, and schema path can be
cleared where their domain permits absence. Null required role/format/mode or a
null whole definition is invalid. Scalar `required` and `repair_attempts` retain
their false/zero defaults and are not nullable. Null inputs means no declarations.
Unknown fields are ConfigKit errors, including old `id`, `version`, and
`base_profile`. Prompt versions must be string mapping keys.
Literal definition keys remain exact; require nonblank identity but do not
normalize/rekey maps after ConfigKit has resolved relationships. Update selection
to use exact nonblank IDs/versions rather than silently trimming them into a
different identity. Scalar normalization such as endpoint trimming is recorded
through ConfigKit's resolution callbacks when
mutating its typed configuration. Message roles and cache control are normalized
on execution-owned copies instead: ConfigKit v0.5.0 treats sequence mutations as
whole-value replacements, losing unchanged content_file origins. Inspection
retains the effective input message metadata and its source context. Field
references to definition IDs remain exact as well.
Profile field rules after configuration overlay/inheritance:
| Fields | Absent | Explicit value | Explicit null |
| --- | --- | --- | --- |
| `backend` | No selected backend | Exact registered ID; blank means none | No backend |
| `endpoint` | Selected backend endpoint, if any | Normalize/validate nonblank URL; empty clears | Clear, do not recover backend endpoint |
| `model` | Unspecified | Nonblank supplied model; empty clears | Unspecified |
| `temperature`, `top_p`, `max_tokens` | Unspecified | Validate existing bounds; zero is explicit and sent | Suppress outbound control |
| `reasoning_effort`, `service_tier` | Unspecified | Existing normalization; blank suppresses | Suppress outbound control |
| `timeout_seconds` | Framework 600-second baseline | Nonnegative; zero disables generation deadline | Reset to framework baseline, not lower profile timeout |
| `api_key_env` | Selected backend's optional lookup name | Trimmed value; blank disables lookup | Disable lookup, do not recover backend name |
| `api_key_required` | False | Explicit boolean, including false overriding true | False |
| `extra_params` | Backend defaults | ConfigKit-composed profile map, recursively overlaid on backend map | Suppress entire configured provider map |
Profile API-key environment names follow the existing lookup contract, including
host-supported non-shell-identifier names. Do not extend the backend's naming
regular expression to profiles. Trimming, explicit clearing, optional lookup,
and direct request-key precedence remain unchanged.
Backend registry defaults are applied once during snapshot construction, after
effective profile inheritance. Keep them out of the inheritance input: a parent's
derived endpoint/key/defaults must not follow a child that changes backend.
Provider map null entries are literal JSON null; empty mappings retain children.
For backend-to-profile provider-map overlay, use a narrow JSON-value merge helper
with the same recursive-map/value-replacement semantics; it is not a configuration
loader. Both backend and profile configuration maps have already passed through
ConfigKit's reserved-key semantics; only request-level maps bypass that lifecycle.
Record any defaults/normalizations written into the typed configuration via
ConfigKit Resolve, with dependencies on relevant configuration paths. Do not
project runtime collaborators or credentials into the configuration tree just to
report them. Inspection includes effective backend configuration. It describes
resolved configuration, while InspectProfile and preparation describe
backend-completed execution settings. Make this boundary
explicit rather than inventing synthetic provenance for registry defaults.
Retain APIKeyRequired's direct-key workflow: true suppresses a backend-derived
environment source. An explicitly configured profile environment source, or a
request environment override, remains usable; false restores ordinary optional
credential semantics. No credential values belong in the DTO, inspection, or
construction-time environment resolution. Raw profile `api_key` remains invalid.
Validate all effective supplied values and referenced backend IDs at construction.
Do not require a complete model/endpoint there or reject unused partial profiles.
Do not require a prompt default_profile to be runnable before a request can
override it. Final target completeness is checked after runtime overrides.
InspectProfile can report a partial target without credential or provider access.
Keep request override contracts otherwise unchanged, including complete nonempty
runtime ExtraParams replacement and reasoning clearing. A runtime literal
`extra_params["extends"]` remains ordinary provider data. Do not expand this work
into a redesign of every request override.
### Snapshot And Inspection Design
The loader returns indexed, execution-owned definitions plus a private retained
ConfigKit result. An adapter may implement the runner's narrow lookup interfaces
over maps during migration; no repository chain, fallback-on-error, or per-call
configuration load survives. Preserve defensive ownership across injected clients
and public results. Cache prompt definition hashes at construction; hash effective
captured content, not a stale resource path. Hash bytes need not match the old
configuration format. Preserve rendered-prompt hashing semantics.
Resolve each effective content_file using its exact field origin and ConfigKit
SourceContext. Join its relative name to BaseDir inside the returned rooted FS;
reject absolute/outside-root paths and close every opened handle. Preserve path
text, including meaningful whitespace, after separately checking nonblankness.
Do not choose the child's source for an inherited reference. If there is no unique
contributing source with context, fail with an actionable path/source error; the
caller can supply an explicit Input filesystem context or override the field.
Retain no resource capability for the purpose of public diagnostics.
Capture message text and parse message/session template syntax at construction
with the same function names/options used for runtime rendering. Do not invoke
input helpers or require Vars/artifacts while parsing. Reuse compiled templates
where practical, binding request-specific input functions on safe per-call clones;
never mutate a shared template function closure with request state. Runtime
missing-variable/input failures remain runtime errors.
Expose a concrete read-only facade value:
```go
func (e *Engine) InspectConfiguration() (ConfigurationInspection, error)
func (v ConfigurationInspection) Metadata() configkit.Metadata
func (v ConfigurationInspection) Sources() []configkit.SourceInfo
func (v ConfigurationInspection) Diagnostics() []configkit.Diagnostic
func (v ConfigurationInspection) Explain(path configkit.Path) (configkit.Explanation, bool)
func (v ConfigurationInspection) WriteEffective(w io.Writer) error
func (v ConfigurationInspection) WriteExplanation(w io.Writer, path configkit.Path) error
```
Store an unexported narrow implementation reference; do not expose Result.Config,
ConfigKit SourceContext, mutable runtime values, or internal types. Delegate
queries and redaction to the retained result. Nil Engine returns ErrInvalidConfig;
the zero inspection view has empty query results and writer operations return
ErrInvalidConfig. Writer errors remain inspectable. Mark credential-shaped dynamic
keys sensitive using Definition.Secret; the existing secret-key vocabulary can
serve as a redaction predicate, not a source-rejection rule. Do not redact
`api_key_env` names as credential values. This is not a
promise that arbitrary prompt/provider content is safe to disclose.
Keep the runner independent of this view. Inspection calls perform no discovery,
resource reads, environment credential lookup, or provider operations. Schemas,
artifacts, preparation ownership, generation admission, retry/repair, observations,
and cancellation remain at their established runtime boundaries.
## Stage 1 — Add The Dependency And Typed Loading Core
**Completed:** 2026-09-30.
Scope: module metadata, `internal/configuration` DTOs, real ConfigKit loading,
and its focused tests. Leave engine assembly on the old path for this stage.
1. Add ConfigKit v0.5.0; update go.sum and factual OSS notices for it and its
selected transitive dependencies. Preserve project licenses and the Go floor.
2. Add keyed backend/profile/prompt DTOs and a private load entry point accepting
ConfigKit Inputs and underlays. Call LoadLayered once, with no VersionRule or
secret resolution bindings for API credentials. Use callback-scoped
Resolve/Validate correctly.
3. Implement effective presence extraction and the field rules above. Validate
final domain values through existing helpers, resolving backend references
against the final backend map; profiles have no final target requirement yet.
Record normalization/default mutations, not arbitrary DTO bookkeeping fields.
4. Preserve `*configkit.LoadError` through wrapping. Domain callback messages
identify paths/problems without interpolating raw values or credentials.
5. Add representative real-loader tests: keyed literal IDs, strict group conflict,
recursive cross-tier overlay, a shared base with explicit zero/null/false, an
unused invalid effective definition, and preservation of a ConfigKit load
error. Cover an invalid lower-layer domain value replaced by a valid final
value: construction succeeds. Do not reproduce ConfigKit's parser/inheritance
matrix or add preflight/source-policy tests.
Exit: focused tests and `go test ./...` pass; old engine behavior is not presented
as migrated. Internal inventory documents only the loader now actually present.
## Stage 2 — Add Source Discovery And Prompt Resource Capture
**Completed:** 2026-09-30.
Scope: source selection helpers and construction-time prompt materialization.
1. Implement private directory/FS discovery behind the chosen helper signatures;
wrap discovered paths as ConfigKit sources, retaining explicit root context.
Reuse useful filecatalog path handling without carrying profile metadata scans.
2. Materialize effective prompt message text using field origins and passive
contexts. Move reusable prompt/input/cache/output validation into the loader
or shared domain owner; do not copy entire legacy parsers.
3. Build immutable prompt indexes by exact ID/version and an ambiguity indicator
for omitted versions. Capture declared-input inspection data and prompt hashes.
4. Validate static template syntax using the renderer's grammar/function setup.
Refactor that setup into one private owner if needed; do not render at load.
5. Test native and embedded sources, imported/inherited relative content from a
different directory, explicit mapping context, contextless failure, handle
closure, malformed static syntax, and resource mutation after capture. Use
minimal fixtures, sharing scenarios where the same adapter is exercised.
Exit: loader creates usable frozen prompt definitions without request data.
`go test ./internal/configuration ./internal/prompt ./internal/promptdef` passes;
existing execution tests still work through the old assembly until cutover.
## Stage 3 — Make Execution Settings Presence-Aware
**Completed:** 2026-09-30.
Scope: domain settings, target resolution, wire-presence propagation, profile
inspection. No new source discovery in execution packages.
1. Build the immutable backend registry from the final loaded backend map. Add
execution-owned presence/suppression state and construct indexed profiles
from loader output plus that registry. Apply registry-derived
values after inheritance, using the field table above.
2. Replace scalar-zero/blank merge assumptions in target resolution. Configuration
explicit zero reaches injected clients and the built-in request body; null does
not resurrect a provider setting. Extend existing TargetPresence plumbing
through preparation/results as necessary rather than creating a parallel path.
3. Preserve request-level override precedence, API-key availability behavior,
endpoint-only profiles, backend identity/capacity, and partial-profile runtime
completion. Move final completeness checks after overrides. InspectProfile
returns partial settings without treating missing model/endpoint as load errors.
4. Test explicit numeric zero versus omission, map overlay and null, backend
change without derived defaults leaking from parent, timeout null/zero,
credential-source suppression, and completion through request overrides.
Include one built-in-client wire test, not just an injected-client assertion.
Exit: focused domain/usecase/client tests pass. Any temporary adaptation for the
old assembly is private, clearly limited to reaching Stage 7, and does not become
a compatibility promise. Update affected target/presence GoDoc with this change.
## Stage 4 — Migrate The OpenRouter Catalog Assets
**Completed:** 2026-09-30. Published OpenRouter catalog `v3.20260930.0`
from `f17d208cb7ad1fb043cda2266cd3011d11b40aa0`.
Scope: `../promptkit-backend-openrouter` only. Catalog maintainers own source and
publishing checks here; PromptKit consumes its configuration through ConfigKit.
1. Read that repository's instructions; preserve unrelated work. Convert each
profile asset to `profiles: {literal-id: {settings...}}`, preserving inventory
and values, changing `base_profile` to `extends`, and removing embedded ID.
Keep per-profile files and the exported FS/Root package; no dependency on
PromptKit and no second legacy asset tree.
2. Audit zero/empty values that previously meant unspecified. Remove those when
omission was intended; retain intentional explicit values. Review any dynamic
provider `extends` keys rather than silently changing outbound payloads.
3. Replace `catalog/backend.json` with `catalog/backend.yml` containing a
`backends: {openrouter: {settings...}}` mapping. Preserve endpoint, API-key
environment name, concurrency, queue, and extra-parameter values; remove the
duplicate id and old manifest schema_version. The file is an ordinary ConfigKit
input, not a separately decoded manifest. Update embedded assets, module path
and imports to `/v3`, tests, and catalog format/release documentation, with an
incompatible-format migration note. Keep FS/Root as the minimal export surface.
Catalog-local publishing checks stay in that repository. No provider probes.
4. Validate under its procedure and commit. If publication is authorized, publish
the next unused `v3.YYYYMMDD.N`; otherwise report the exact commit awaiting
publication. The date/revision is selected then, not guessed in this plan.
Exit: one reviewed migrated catalog commit; existing immutable tags still serve
old consumers. The new major module path identifies the changed asset contract;
old consumers retain the published `/v2` assets.
## Stage 5 — Migrate The Rakestrawhome Catalog Assets
**Completed:** 2026-09-30. Published Rakestrawhome catalog `v3.20260930.0`
from `993443d88c451901f4c917fa00c204fd3b54d28d`.
Scope: `../promptkit-backend-rakestrawhome` only. Apply the same keyed-format,
presence, notice, validation, and publication treatment as Stage 4 to its smaller
inventory, including the `/v3` transition and keyed `backends.rakestrawhome`
definition in `catalog/backend.yml`. Preserve backend endpoint, credentials
metadata, and capacity policy.
Do not couple its date/revision to OpenRouter's release.
Exit: both compatible catalog versions are published and ordinarily resolvable
before Stage 6 pins them. If permission or publication is unavailable, stop at
this external dependency checkpoint; do not commit a local replace, guessed
version, or legacy-to-new runtime converter to bypass it.
## Stage 6 — Integrate Catalog Sources And Maintained Profiles
**Completed:** 2026-09-30.
Scope: `internal/catalog`, maintained definitions, catalog integration tests,
dependency pins/notices. Keep this separate from public API migration.
1. Pin the published compatible catalog versions and exact attribution commits.
Update imports to `/v3`. Replace manifest decoding with ConfigKit file Inputs.
2. Supply backend and profile assets as ConfigKit Inputs in the first
underlay. Supply PromptKit-owned keyed Mapping definitions as the next group:
standard extends the selected catalog's current maintained base; auto retains
its current direct model/backend definition. Preserve both literal names.
3. Retire source-layout allowlists, catalog preflight, source ownership/reserved
prefix checks, connection-field prohibitions, and secret-key source rejection.
The adapter selects known asset entry points and discovers configuration files;
ConfigKit acquires their contents. Domain validation sees only the final
assembled configuration. Consumers may overlay catalog backends, profiles,
and their bases through the same configuration model.
4. Test real imported assets offline with the new loader, maintained-name
resolution, consumer override of a maintained base/backend, and rejection of
an invalid final backend. Test a replaced lower-layer domain error succeeds.
Do not independently validate catalog-only intermediate configurations, parse
raw assets in PromptKit tests, or audit origins to establish ownership.
Exit: catalog and owned definitions participate in the new keyed configuration
model without a legacy profile parser on that path. Catalog integration tests
pass against released modules with `GOWORK=off` and no replacements.
## Stage 7 — Cut Over Engine Assembly And Public Construction
**Completed:** 2026-09-30.
Scope: `engine.go`, configuration facade/helpers, `internal/engine`, facade
adapters, and migration of their direct test setup.
1. Introduce the Construction Surface exactly as specified above. Remove the
superseded source/profile options and convenience types rather than keeping
deprecated pass-through wrappers. Keep intentional runtime injection APIs.
2. Load the combined backend/profile/prompt configuration, construct its runtime
registry and indexed snapshot, then assemble renderer, validator, capacity
manager, and runner. Return no Engine on construction failure.
Translate construction errors to ErrInvalidConfig while preserving LoadError
via errors.As. Resource errors include semantic path and safe source context.
3. Connect indexed prompt/profile lookups to preparation and inspection. Missing
selections keep meaningful not-found errors; prompt version ambiguity stays a
selection error. Old parse/load error categories need not be manufactured for
errors now detected at construction.
4. Move root test fixtures/setup and maintained example construction to ConfigKit
sources, keyed definitions, ProfileSource, and BackendSource so all consumers
still compile. Update local-backend examples to choose their ID as a map key.
Update expected timing only for the deliberate eager-validation change. Keep
behavioral assertions for prompting, overrides, and errors.
5. Add public tests for a minimal custom local backend and programmatic profile,
plus embedded defaults/operator/programmatic tiers. No elaborate consumer
setup should be needed merely to send one prompt.
Exit: the public Engine uses only the new loading path. Root/package tests pass;
all package/example sources compile. Update GoDoc and consumer construction
guidance in this stage so removed options are not presented as usable.
## Stage 8 — Complete Frozen Rendering And Lifecycle Integration
**Completed:** 2026-09-30.
Scope: renderer/runtime consumption of the snapshot, ownership, and lifecycle
regressions. Stage 2 already owns acquisition and static syntax validation.
1. Use captured text and parsed templates without per-run configuration reads,
parsing, or inheritance. Bind input helpers per invocation safely; keep missing
request data errors and direct-session template bypass behavior.
2. Verify preparation and execution still handle artifacts and schema compilation
at invocation time, honor output overrides, freeze prepared resources, and
keep prepared handles bound to their engine. Credentials remain at runtime.
3. Test an old engine after configuration/resource edits and a newly constructed
engine observing the edits. Test concurrent renders with different Vars/input
helpers and no data leakage. Use an FS that rejects reads after construction
to demonstrate no configuration/content-file reopening during ordinary runs;
do not incorrectly prohibit request artifact/schema reads.
4. Preserve appended-message ordering, structural repair, retries, admission,
cancellation, observers, and result metadata through existing regression
tests; add only distinct missing cross-boundary cases.
Exit: focused race tests pass for assembled rendering/preparation and lifecycle.
No retained callbacks capture ConfigKit resolution contexts or request state.
## Stage 9 — Expose Explicit Configuration Inspection
**Completed:** 2026-09-30.
Scope: the read-only facade and retained ConfigKit inspection adapter.
1. Implement ConfigurationInspection and its specified methods without exposing
Result.Config or filesystem capabilities. Retain a private narrow view of the
ConfigKit result; no regeneration of origins from runtime values.
2. Ensure effective/explanation reports use ConfigKit redaction, including the
selected sensitivity predicate. Preserve safe value-free domain diagnostics.
3. Test a targeted literal-key path with layered/inherited history, detached
returned slices/records, writer failures, zero/nil behavior, and redacted secret
shaped provider values. Demonstrate inspection still works after source removal
and makes no filesystem/provider calls.
4. Document construction inspection versus InspectProfile and prepared details.
Consumers choose report disclosure; do not add CLI commands or automatic logs.
Exit: public inspection tests pass, including concurrent inspection/execution.
The runner has no reference to the inspector or ConfigKit metadata types.
## Stage 10 — Migrate Examples And Canonical Format Guidance
**Completed:** 2026-09-30.
Scope: maintained examples and user-facing documentation.
1. Complete the preparation, execution, and observation example migration begun
at public cutover. Preserve their distinct existing semantic assertions; extend
the preparation example with a small explicit inspection query if useful.
2. Update `docs/formats.md` to own the new keyed syntax, source-group semantics,
inheritance/presence/null table, eager validation, snapshot lifetime, and
resource-context rules. Link ConfigKit for generic grammar; do not copy its
complete parser/inheritance contract.
3. Update README and `docs/consumers/pkg-promptkit.md` with minimal local-backend,
file/embedded source, programmatic Mapping/ProfileSource, explicit inspection,
and engine-replacement workflows. Explain main versus ordered underlays and
backend layering in place of registration-only restrictions.
4. Add concise old-to-new migration guidance in the consumer guide: removed API
options, exact IDs, `base_profile` to `extends`, zero/false/null, recursive maps,
no clear via `{}`, reserved `extends` runtime escape, failure timing, and the
fact that changing a model/backend does not clear unrelated explicit fields.
Remove old automatic reload claims. Release notes later link to this owner.
Exit: all three offline examples run with their semantic checks; documentation
links and snippets match the implemented API. No unreleased version is tagged or
published, and no speculative release-note version is invented.
## Stage 11 — Remove Superseded Machinery And Consolidate Tests
**Completed:** 2026-09-30.
Scope: dead loading/resolution code and its obsolete tests, module cleanup, and
internal architectural documentation.
1. Delete lazy prompt/profile metadata prepasses, filesystem repositories,
fallback-on-error overlays, custom base-chain resolver/depth cap, zero-sentinel
profile merging, and temporary stage adapters once no caller needs them.
Remove the separate backend-manifest decoder, source-policy audits, and
pre-handoff backend registration validation. Keep final-value domain checks.
Retain narrow lookup interfaces only where they still simplify real consumers.
2. Retain useful filecatalog/schema utilities and JSON-value ownership/validation.
Remove direct yaml.v3 use and dependency if no remaining owner requires it;
do not claim all YAML dependencies disappear when ConfigKit still uses YAML.
3. Move meaningful regressions from deleted package tests to their new owning
boundary. Retire tests about old internals, repeated parser coverage,
source-policy enforcement, or superseded semantics. Do not replace lost runtime
protection with loader-only tests. Keep private assembly checks separate from
external facade tests.
4. Update `docs/internal/overview.md`, `docs/internal/sources.md`, relevant renderer
and engine documents, and the implemented-structure paragraphs in architecture
policy. Describe the new package/dependency boundary without turning names or
chosen adapters into permanent invariants. Update OSS notices with module tidy.
Exit: no legacy source path remains, no internal package imports the public root,
runtime/domain packages do not import ConfigKit, and `go test ./...` passes with
the reduced implementation/test inventory.
## Stage 12 — Validate Scale And The Complete Consumer Workflow
**Completed:** 2026-09-30.
Scope: final integration evidence and high-confidence simplification, not another
feature expansion or release gate.
1. Add focused benchmarks for construction with many independent definitions,
shared inherited bases, and several tiers; add an already-constructed selection
benchmark. Report allocations and input/output shape; inherited materialized
output/history may dominate. Do not impose timing thresholds or new caps.
2. Inspect for repeated full-map copies, per-run provenance calls, rescans,
repeated inheritance, and unnecessary double parsing. Fix demonstrated local
inefficiencies with straightforward ownership/indexing; no global cache.
3. Run the repository's existing complete local check script once on the final
candidate, including race tests and offline examples. Investigate failures;
do not weaken meaningful tests to get a green result. Stage intended new
files before the script's tracked-file checks.
4. Review the feature-roadmap capability list against actual public workflows:
independent inspection, partial profile completion, optional credentials,
source contexts, artifacts/schemas, prepared ownership, backend capacity,
overrides, appended messages, repair/retries, observations, and hashes.
5. Report completed stages, selected catalog versions, validation evidence, and
remaining issues. Minor-release documentation/version selection and roadmap
retirement belong to the separately authorized release preparation workflow.
Exit: the feature roadmap's end state is demonstrably implemented, with one
configuration lifecycle, explicit operator inspection, no legacy loader, and
no newly invented provider checks or human release gates.
The catalog-validation placement question is resolved by the configuration-boundary
ADR. No design question remains open here. Catalog publication authority and
ordinary module availability remain operational checkpoints for the catalog stages.

View File

@@ -1,126 +0,0 @@
# ConfigKit Adoption Review Findings
Review date: 2026-10-03. Reviewed implementation commits `2b847ac` through
`66c55b7` against the [feature roadmap](configkit.md),
[implementation plan](implementation.md), and accepted
[configuration-boundary ADR](../adr/0001-configuration-boundary.md).
## Summary
The principal architecture is in place: one ConfigKit load assembles backend,
profile, and prompt configuration; PromptKit validates effective values, captures
resources/templates, and builds execution-owned indexes. Catalog preflight and
legacy profile resolution are removed. Operator inspection delegates to the
retained ConfigKit result separately from ordinary execution.
Both findings have been addressed. The original review evidence is retained
below; the resolution section records the changes and regression coverage.
## 1. P2 — Message Normalization Loses Content-File Resource Origins
Location: `internal/configuration/prompts.go`, particularly `resolvePrompts`
lines 69–99 and `readContent`'s effective-origin lookup.
When a role or cache-control value needs normalization, `resolvePrompts` records
a normalization operation for the entire `messages` sequence in addition to the
field-level operations. The resulting metadata no longer provides the source
location needed by `readContent` for an unchanged `content_file`. Construction
then rejects an otherwise valid prompt.
Confirmed through the public API using FilesystemInputs over an fs.FS containing
this document and a readable `body.txt`:
```yaml
prompts:
p:
v1:
messages:
- role: " USER "
content_file: body.txt
output:
format: text
validation_mode: none
```
NewEngine fails with
`/prompts/p/v1/messages/0/content_file: content_file needs one contributing source with a filesystem context`.
Changing only the role to `user` succeeds. Role trimming/lowercasing and
source-relative message capture are both intended capabilities; requiring
already-normalized metadata is an accidental restriction.
Required outcome:
- Preserve ConfigKit's effective resource context while normalizing neighboring
message metadata. Review the broad sequence normalization record and how the
final input origin is retained across domain normalization.
- Use effective handoff metadata/context, not historical traversal, raw-source
inspection, source rereading, or an invented fallback directory.
- Add focused regressions combining file-backed content with role and
cache-control normalization, including inherited/imported content. Existing
tests cover normalization and resource capture separately but miss this
interaction. Verify inspection remains accurate for untouched content fields.
## 2. P2 — Profiles Gain An Unrequested Environment-Name Restriction
Location: `internal/configuration/load.go`, profile validation at lines 278–279,
using the package's `envName` regular expression.
Profile `api_key_env` values now must match `[A-Za-z_][A-Za-z0-9_]*`. The previous
profile path accepted environment lookup names without this shell-identifier
restriction; the regular expression previously governed backend registration.
The refactor extends it to profiles even though arbitrary environment names
supported by the host remain usable through Go's environment lookup and the
request override path.
Confirmed through NewEngine with an otherwise valid endpoint/model profile:
`api_key_env: NORMAL_KEY` succeeds, while `api_key_env: service.api-key` fails
construction. The latter is a usable environment name on the reviewed Linux
platform. This restriction is unrelated to configuration layering, ConfigKit
typing, or optional credential availability.
The implementation plan's reference to existing naming rules is not evidence
of an express user decision to introduce this new profile restriction. The
requested compatibility break permits adopting ConfigKit semantics; it does
not itself authorize additional unrelated name constraints.
Required outcome:
- Remove the newly introduced profile shell-identifier check, preserving the
established trimming, explicit clearing, optional environment lookup, and
direct-key precedence behavior.
- Add a public-path regression for a supported non-shell-identifier environment
name and for its unset/empty optional behavior, with controlled test environment
state and a synthetic credential. Keep any prerequisite validation limited to
actual supported environment lookup requirements.
- Correct the plan's profile-field guidance so later work does not reintroduce
this restriction. Treat any broader change to existing backend naming policy
separately rather than silently extending or weakening it in this fix.
## Validation Performed
- `go test ./...` passed on Go 1.26.8.
- `GOTOOLCHAIN=go1.25.5 go test ./...` passed on the declared minimum toolchain.
- `go test -race ./...` passed.
- Separate offline public-API reproductions confirmed both findings. Missing
message/session variables also remained runtime errors in targeted checks.
The passing suite does not cover the two failing cases above. No provider
requests were made. Beyond these findings, the reviewed implementation did not
show another actionable ownership-boundary violation or strengthened guarantee.
## Resolution
1. Message role/cache-control normalization now occurs on execution-owned copies
during materialization. ConfigKit's captured message sequence and its origins
remain intact. The imported/inherited content regression now includes role
and cache-control normalization and asserts the unchanged field's source in
inspection metadata. This also removes duplicate cache-control normalization.
2. Removed the profile environment-name regular expression check, leaving backend
validation unchanged. A public engine/HTTP-transport regression covers a
non-shell environment name, unset/empty/whitespace values, explicit clearing,
trimming, and direct-key precedence. The implementation plan now states this
distinction explicitly.
Both regressions failed against the reviewed implementation before the fixes.
After remediation, the full ordinary and race-enabled suites, affected packages
on Go 1.25.5, `go vet ./...`, and documentation/whitespace checks passed.

View File

@@ -82,12 +82,12 @@ var (
ErrValidation = errors.New("failed to validate output")
)
// Engine inspects prompts and profiles and prepares and runs Promptkit prompt
// requests.
// Engine inspects prompts and profiles, enumerates prompt requirements, and
// prepares and runs Promptkit prompt requests.
//
// An Engine is safe for concurrent calls to [Engine.InspectConfiguration],
// [Engine.InspectPrompt],
// [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
// [Engine.InspectRequirements], [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
// backend-capacity pools that coordinate Run and RunPrepared admission and
// model generation. Injected collaborators may still be invoked concurrently

View File

@@ -36,6 +36,17 @@ func main() {
if _, found := view.Explain(configkit.Keys("profiles", "offline-example", "model")); !found {
exit(fmt.Errorf("configured profile model was not found"))
}
requirements, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
PromptID: "example.prepare",
})
if err != nil {
exit(err)
}
if !requirements.Complete || len(requirements.Inputs) != 1 ||
requirements.Inputs[0].Name != "note" ||
requirements.Inputs[0].Kind != promptkit.RequirementRequired {
exit(fmt.Errorf("required note input was not discovered before preparation"))
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "example.prepare",

9
go.mod
View File

@@ -1,15 +1,20 @@
module gitea.maximumdirect.net/eric/promptkit
go 1.25.5
go 1.26.0
require (
gitea.maximumdirect.net/eric/ConfigKit v0.5.0
gitea.maximumdirect.net/eric/ConfigKit v0.7.1
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
)
require (
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 // indirect
github.com/yuin/goldmark/v2 v2.1.6 // indirect
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
golang.org/x/mod v0.40.0 // indirect
golang.org/x/text v0.14.0 // indirect
)
tool gitea.maximumdirect.net/eric/ReleaseKit/cmd/releasekit

10
go.sum
View File

@@ -1,5 +1,7 @@
gitea.maximumdirect.net/eric/ConfigKit v0.5.0 h1:2KwG10+kApWM3avOoYA+p768xcbHUp7tQ2rqizBNpyg=
gitea.maximumdirect.net/eric/ConfigKit v0.5.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
gitea.maximumdirect.net/eric/ConfigKit v0.7.1 h1:q3HrX4WWCPt1WfMydqFEaAmCf9s6Dq+vknJj0C/mdcI=
gitea.maximumdirect.net/eric/ConfigKit v0.7.1/go.mod h1:C82sbyKHYMV1OwaD2GJanFrwTfcZfjlW2JsGze5b0kA=
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 h1:Qn+mdebYOzCVCEXgCbuGSO/WHmJtItwaQ8e3XwY3qtI=
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1/go.mod h1:LVodqNQ3OIKv9NBq+fFeyMByd1lHY25YOmmpah/NaNI=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
@@ -8,8 +10,12 @@ github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxK
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
github.com/yuin/goldmark/v2 v2.1.6 h1:igy0q9JcqovoFXFx/XjtU8sHVhcUHavPj2sA3EoKJUE=
github.com/yuin/goldmark/v2 v2.1.6/go.mod h1:G6M4/qOFtfn01/o14BU1UR2Lo5N3S9Qo7xCuz/sHjGQ=
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

View File

@@ -21,6 +21,7 @@ var (
)
const fileReadChunkSize = 64 * 1024
const fileReadInitialCapacityLimit = 1 << 20
// Reader resolves artifact references into actual artifacts.
type Reader interface {
@@ -133,7 +134,11 @@ func readFileArtifact(ctx context.Context, path string, open func(string) (artif
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFile, path)
}
data := make([]byte, 0)
capacity := 0
if size := openedInfo.Size(); size > 0 {
capacity = int(min(size, int64(fileReadInitialCapacityLimit)))
}
data := make([]byte, 0, capacity)
chunk := make([]byte, fileReadChunkSize)
for {
if err := ctx.Err(); err != nil {
@@ -153,6 +158,13 @@ func readFileArtifact(ctx context.Context, path string, open func(string) (artif
return nil, fmt.Errorf("failed to read file %s: %w", path, readErr)
}
}
if cap(data)-len(data) > len(data) {
// A stale size hint should not make a small completed artifact retain a
// disproportionately large backing array.
compact := make([]byte, len(data))
copy(compact, data)
data = compact
}
contentType := mime.TypeByExtension(filepath.Ext(path))
if contentType == "" {

View File

@@ -0,0 +1,180 @@
package artifact
import (
"bytes"
"context"
"crypto/sha256"
"fmt"
"mime"
"os"
"path/filepath"
"runtime"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
var artifactFileReadBenchmarkSink *domain.Artifact
func BenchmarkArtifactFileRead(b *testing.B) {
for _, fixture := range makeArtifactReadBenchmarkFiles(b) {
b.Run(fixture.name, func(b *testing.B) {
reader := &fileReader{open: openArtifactFile}
ref := domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: fixture.path}
checkArtifactFileRead(b, reader, ref, fixture.body)
b.ReportAllocs()
b.SetBytes(int64(len(fixture.body)))
b.ResetTimer()
for range b.N {
artifact, err := reader.Read(context.Background(), ref)
if err != nil {
b.Fatalf("read artifact: %v", err)
}
artifactFileReadBenchmarkSink = artifact
}
b.StopTimer()
runtime.KeepAlive(artifactFileReadBenchmarkSink)
})
}
}
// BenchmarkRetainedArtifactFileRead measures live artifact bodies when an
// opened file reports a size larger than the content that reaches EOF.
func BenchmarkRetainedArtifactFileRead(b *testing.B) {
for _, fixture := range []struct {
name string
body []byte
}{
{name: "small", body: bytes.Repeat([]byte("x"), 256<<10)},
{name: "empty", body: []byte{}},
} {
b.Run(fixture.name, func(b *testing.B) {
benchmarkRetainedArtifactFileRead(b, fixture.body)
})
}
}
func benchmarkRetainedArtifactFileRead(b *testing.B, body []byte) {
path := filepath.Join(b.TempDir(), "overreported.txt")
if err := os.WriteFile(path, body, 0o600); err != nil {
b.Fatal(err)
}
mime.TypeByExtension(filepath.Ext(path))
reader := &fileReader{open: func(path string) (artifactFile, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
return &sizeReportingArtifactFile{artifactFile: file, reportedSize: 16 << 20}, nil
}}
ref := domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
checkArtifactFileRead(b, reader, ref, body)
const retainedBatchSize = 4
retained := make([]*domain.Artifact, retainedBatchSize)
artifactFileReadBenchmarkSink = nil
runtime.GC()
var setupMemory runtime.MemStats
runtime.ReadMemStats(&setupMemory)
runtime.KeepAlive(reader)
runtime.KeepAlive(ref)
b.ReportAllocs()
b.SetBytes(int64(len(body)))
b.ResetTimer()
for iteration := 0; iteration < b.N; iteration++ {
artifact, err := reader.Read(context.Background(), ref)
if err != nil {
b.Fatalf("read retained artifact: %v", err)
}
retained[iteration%len(retained)] = artifact
}
b.StopTimer()
for i, artifact := range retained {
if artifact != nil {
continue
}
artifact, err := reader.Read(context.Background(), ref)
if err != nil {
b.Fatalf("fill retained artifact batch: %v", err)
}
retained[i] = artifact
}
runtime.GC()
var retainedMemory runtime.MemStats
runtime.ReadMemStats(&retainedMemory)
b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
runtime.KeepAlive(retained)
runtime.KeepAlive(reader)
runtime.KeepAlive(ref)
}
type artifactReadBenchmarkFile struct {
name string
path string
body []byte
}
func makeArtifactReadBenchmarkFiles(b *testing.B) []artifactReadBenchmarkFile {
b.Helper()
fixtures := []struct {
name string
size int
}{
{name: "small_4KiB", size: 4 << 10},
{name: "1MiB", size: 1 << 20},
{name: "large_4MiB", size: 4 << 20},
}
results := make([]artifactReadBenchmarkFile, 0, len(fixtures))
for _, fixture := range fixtures {
body := bytes.Repeat([]byte("x"), fixture.size)
path := filepath.Join(b.TempDir(), fixture.name+".txt")
if err := os.WriteFile(path, body, 0o600); err != nil {
b.Fatal(err)
}
mime.TypeByExtension(filepath.Ext(path))
results = append(results, artifactReadBenchmarkFile{
name: fixture.name,
path: path,
body: body,
})
}
return results
}
func checkArtifactFileRead(b *testing.B, reader Reader, ref domain.ArtifactRef, want []byte) {
b.Helper()
artifact, err := reader.Read(context.Background(), ref)
if err != nil {
b.Fatalf("check artifact read: %v", err)
}
if !bytes.Equal(artifact.Body, want) || artifact.Size != int64(len(want)) {
b.Fatalf("artifact size=%d body length=%d, want %d", artifact.Size, len(artifact.Body), len(want))
}
wantHash := fmt.Sprintf("%x", sha256.Sum256(want))
if artifact.Hash != wantHash {
b.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
}
}
type sizeReportingArtifactFile struct {
artifactFile
reportedSize int64
}
func (file *sizeReportingArtifactFile) Stat() (os.FileInfo, error) {
info, err := file.artifactFile.Stat()
if err != nil {
return nil, err
}
return reportedArtifactFileInfo{FileInfo: info, size: file.reportedSize}, nil
}
type reportedArtifactFileInfo struct {
os.FileInfo
size int64
}
func (info reportedArtifactFileInfo) Size() int64 { return info.size }

View File

@@ -3,7 +3,9 @@ package artifact
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
@@ -232,6 +234,64 @@ func TestFileReaderFailuresAndMetadata(t *testing.T) {
})
}
func TestFileReaderUsesOpenedSizeOnlyAsACapacityHint(t *testing.T) {
tests := []struct {
name string
body []byte
reportedSize int64
}{
{name: "overreported size", body: []byte("actual file content"), reportedSize: 128},
{name: "overreported size with empty content", body: []byte{}, reportedSize: 16 << 20},
{name: "underreported size", body: bytes.Repeat([]byte("x"), fileReadChunkSize+17), reportedSize: 1},
{name: "very large hint with small content", body: []byte("small actual content"), reportedSize: int64(^uint64(0) >> 1)},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
filePath := filepath.Join(t.TempDir(), "artifact.txt")
if err := os.WriteFile(filePath, tc.body, 0o600); err != nil {
t.Fatal(err)
}
reader := &fileReader{open: func(path string) (artifactFile, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
info, err := file.Stat()
if err != nil {
file.Close()
return nil, err
}
return &reportedInfoFile{
artifactFile: file,
info: reportedFileSizeInfo{FileInfo: info, size: tc.reportedSize},
}, nil
}}
artifact, err := reader.Read(context.Background(), domain.ArtifactRef{
Type: domain.ArtifactRefFile,
URI: filePath,
})
if err != nil {
t.Fatalf("read artifact: %v", err)
}
if !bytes.Equal(artifact.Body, tc.body) {
t.Fatalf("artifact body length=%d, want %d", len(artifact.Body), len(tc.body))
}
if artifact.Size != int64(len(tc.body)) {
t.Fatalf("artifact size=%d, want %d", artifact.Size, len(tc.body))
}
if artifact.Name != filepath.Base(filePath) || artifact.URI != filePath || artifact.ContentType == "" {
t.Fatalf("unexpected artifact metadata: name=%q URI=%q content type=%q", artifact.Name, artifact.URI, artifact.ContentType)
}
wantHash := fmt.Sprintf("%x", sha256.Sum256(tc.body))
if artifact.Hash != wantHash {
t.Fatalf("artifact hash=%q, want %q", artifact.Hash, wantHash)
}
})
}
}
func TestFileReaderCancelsAfterReadProgress(t *testing.T) {
filePath := filepath.Join(t.TempDir(), "artifact.bin")
content := bytes.Repeat([]byte("x"), fileReadChunkSize*2)
@@ -268,6 +328,13 @@ func (f *reportedInfoFile) Stat() (os.FileInfo, error) {
return f.info, nil
}
type reportedFileSizeInfo struct {
os.FileInfo
size int64
}
func (info reportedFileSizeInfo) Size() int64 { return info.size }
type cancelAfterProgressFile struct {
artifactFile
cancel context.CancelFunc

View File

@@ -13,9 +13,12 @@ import (
const benchmarkDefinitionCount = 128
var (
constructionResult *loaded
selectedPrompt *domain.PromptDefinition
selectedProfile *domain.ExecutionProfile
constructionResult *loaded
selectedPrompt *domain.PromptDefinition
selectedPromptHash string
selectedPromptInspection *domain.PromptInspection
selectedProfile *domain.ExecutionProfile
selectedProfileIdentity string
)
type configurationShape struct {
@@ -177,3 +180,116 @@ func BenchmarkConfigurationSelection(b *testing.B) {
b.ReportMetric(float64(shape.effectiveEntries), "effective-entries/op")
})
}
func inspectionExtraParams(entries int) map[string]any {
providers := make(map[string]any, entries)
for index := range entries {
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
"enabled": true,
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
"limits": map[string]any{"burst": index + 1, "period": "minute"},
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
}
}
return map[string]any{"provider_options": providers}
}
func inspectionDefinitions(extraParams map[string]any) configurationShape {
profiles := map[string]any{
"small": map[string]any{"model": "inspection-model", "extra_params": extraParams},
}
prompt := benchmarkPrompt("small")
prompt["inputs"] = []any{map[string]any{"name": "transcript", "required": true}}
prompts := map[string]any{
"inspection": map[string]any{"1": prompt},
}
return configurationShape{
main: configkit.Input{Source: configkit.Mapping("inspection definitions", map[string]any{
"profiles": profiles,
"prompts": prompts,
})},
inputEntries: 2, effectiveEntries: 2, tiers: 1,
}
}
func BenchmarkInspectionSelection(b *testing.B) {
for _, scenario := range []struct {
name string
entries int
}{
{name: "small_extra", entries: 2},
{name: "nested_extra", entries: 64},
} {
shape := inspectionDefinitions(inspectionExtraParams(scenario.entries))
result, err := load(shape.main, nil)
if err != nil {
b.Fatal(err)
}
ctx := context.Background()
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection", "1")
if err != nil || definition.ID != "inspection" || definition.Version != "1" || hash == "" ||
len(definition.Inputs) != 1 || definition.Inputs[0].Name != "transcript" || !definition.Inputs[0].Required {
b.Fatalf("prompt lookup fixture is invalid: definition=%+v hash=%q err=%v", definition, hash, err)
}
profile, err := result.GetProfile(ctx, "small")
if err != nil || profile.ID != "small" || profile.ResolvedTarget.Model != "inspection-model" ||
len(profile.ResolvedTarget.ExtraParams["provider_options"].(map[string]any)) != scenario.entries {
b.Fatalf("profile lookup fixture is invalid: profile=%+v err=%v", profile, err)
}
inspection, err := result.GetPromptInspection(ctx, "inspection", "1")
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion != "1" ||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "transcript" {
b.Fatalf("prompt inspection fixture is invalid: inspection=%+v err=%v", inspection, err)
}
identity, err := result.GetProfileIdentity(ctx, "small")
if err != nil || identity != "small" {
b.Fatalf("profile identity fixture is invalid: identity=%q err=%v", identity, err)
}
b.Run("prompt/"+scenario.name, func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
definition, hash, err := result.GetPromptSnapshot(ctx, "inspection", "1")
if err != nil {
b.Fatal(err)
}
selectedPrompt = definition
selectedPromptHash = hash
}
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
})
b.Run("prompt_inspection/"+scenario.name, func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
inspection, err := result.GetPromptInspection(ctx, "inspection", "1")
if err != nil {
b.Fatal(err)
}
selectedPromptInspection = inspection
}
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
})
b.Run("profile/"+scenario.name, func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
profile, err := result.GetProfile(ctx, "small")
if err != nil {
b.Fatal(err)
}
selectedProfile = profile
}
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
})
b.Run("profile_identity/"+scenario.name, func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
identity, err := result.GetProfileIdentity(ctx, "small")
if err != nil {
b.Fatal(err)
}
selectedProfileIdentity = identity
}
b.ReportMetric(float64(scenario.entries), "extra-entries/op")
})
}
}

View File

@@ -16,6 +16,15 @@ type Inspection interface {
Explain(configkit.Path) (configkit.Explanation, bool)
WriteEffective(io.Writer) error
WriteExplanation(io.Writer, configkit.Path) error
capturedResult() *configkit.Result[document]
}
type inspectionResult struct {
*configkit.Result[document]
}
func (inspection inspectionResult) capturedResult() *configkit.Result[document] {
return inspection.Result
}
// Inspection returns only the retained result's inspection operations.
@@ -23,7 +32,19 @@ func (state *loaded) Inspection() Inspection {
if state == nil {
return nil
}
return state.result
return inspectionResult{Result: state.result}
}
// Diff compares two captured ConfigKit results without reconstructing them.
func Diff(left, right Inspection) ([]configkit.Difference, error) {
var leftResult, rightResult *configkit.Result[document]
if left != nil {
leftResult = left.capturedResult()
}
if right != nil {
rightResult = right.capturedResult()
}
return configkit.Diff(leftResult, rightResult)
}
func secretProviderPath(path configkit.Path) bool {

View File

@@ -0,0 +1,29 @@
package configuration
import (
"context"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func (state *loaded) ListProfiles(ctx context.Context) ([]string, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
profiles := append([]string(nil), state.profileIDs...)
if err := ctx.Err(); err != nil {
return nil, err
}
return profiles, nil
}
func (state *loaded) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
prompts := append([]domain.PromptIdentity(nil), state.promptIDs...)
if err := ctx.Err(); err != nil {
return nil, err
}
return prompts, nil
}

View File

@@ -0,0 +1,142 @@
package configuration
import (
"context"
"errors"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestLoadedListsSortedEffectiveIdentities(t *testing.T) {
state, err := Load(mapping("identities", map[string]any{
"backends": map[string]any{
"local": map[string]any{"endpoint": "http://localhost:8080/v1"},
},
"profiles": map[string]any{
"z/profile": map[string]any{"backend": "local", "model": "z-model"},
"partial/profile": map[string]any{"backend": "local"},
"a/profile": map[string]any{"backend": "local", "model": "a-model"},
" z/profile ": map[string]any{"backend": "local", "model": "spaced-model"},
},
"prompts": map[string]any{
"z/prompt": map[string]any{"v1": listingPromptDefinition()},
"a/prompt": map[string]any{"v1": listingPromptDefinition()},
" z/prompt ": map[string]any{
"2": listingPromptDefinition(),
"10": listingPromptDefinition(),
},
},
}), nil)
if err != nil {
t.Fatal(err)
}
profiles, err := state.ListProfiles(context.Background())
if err != nil {
t.Fatal(err)
}
wantProfiles := []string{" z/profile ", "a/profile", "partial/profile", "z/profile"}
if !reflect.DeepEqual(profiles, wantProfiles) {
t.Fatalf("profile IDs = %#v, want %#v", profiles, wantProfiles)
}
prompts, err := state.ListPrompts(context.Background())
if err != nil {
t.Fatal(err)
}
wantPrompts := []domain.PromptIdentity{
{ID: " z/prompt ", Version: "10"},
{ID: " z/prompt ", Version: "2"},
{ID: "a/prompt", Version: "v1"},
{ID: "z/prompt", Version: "v1"},
}
if !reflect.DeepEqual(prompts, wantPrompts) {
t.Fatalf("prompt identities = %#v, want %#v", prompts, wantPrompts)
}
}
func TestLoadedListsAreIndependent(t *testing.T) {
state, err := Load(mapping("identities", map[string]any{
"profiles": map[string]any{"profile": map[string]any{}},
"prompts": map[string]any{"example": map[string]any{"v1": listingPromptDefinition()}},
}), nil)
if err != nil {
t.Fatal(err)
}
profiles, err := state.ListProfiles(context.Background())
if err != nil {
t.Fatal(err)
}
profiles = append(profiles, "caller profile")
profiles[0] = "changed profile"
prompts, err := state.ListPrompts(context.Background())
if err != nil {
t.Fatal(err)
}
prompts[0].ID = "changed prompt"
prompts[0].Version = "changed version"
profiles, err = state.ListProfiles(context.Background())
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(profiles, []string{"profile"}) {
t.Fatalf("caller mutation changed profile inventory: %#v", profiles)
}
prompts, err = state.ListPrompts(context.Background())
if err != nil {
t.Fatal(err)
}
wantPrompts := []domain.PromptIdentity{{ID: "example", Version: "v1"}}
if !reflect.DeepEqual(prompts, wantPrompts) {
t.Fatalf("caller mutation changed prompt inventory: %#v", prompts)
}
}
func TestLoadedListsEmptyInventory(t *testing.T) {
state, err := Load(configkit.Input{Source: configkit.NoSource()}, nil)
if err != nil {
t.Fatal(err)
}
profiles, err := state.ListProfiles(context.Background())
if err != nil || len(profiles) != 0 {
t.Fatalf("empty profile listing = %#v, %v", profiles, err)
}
prompts, err := state.ListPrompts(context.Background())
if err != nil || len(prompts) != 0 {
t.Fatalf("empty prompt listing = %#v, %v", prompts, err)
}
}
func TestLoadedListsHonorCanceledContext(t *testing.T) {
state, err := Load(mapping("identities", map[string]any{
"profiles": map[string]any{"partial": map[string]any{}},
"prompts": map[string]any{"example": map[string]any{"v1": listingPromptDefinition()}},
}), nil)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
profiles, err := state.ListProfiles(ctx)
if !errors.Is(err, context.Canceled) || len(profiles) != 0 {
t.Fatalf("canceled profile listing = %#v, %v", profiles, err)
}
prompts, err := state.ListPrompts(ctx)
if !errors.Is(err, context.Canceled) || len(prompts) != 0 {
t.Fatalf("canceled prompt listing = %#v, %v", prompts, err)
}
}
func listingPromptDefinition() map[string]any {
return map[string]any{
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}
}

View File

@@ -54,10 +54,10 @@ type promptConfig struct {
}
type promptInputConfig struct {
Name string `yaml:"name"`
Required bool `yaml:"required"`
ContentType string `yaml:"content_type"`
Description string `yaml:"description"`
Name string `yaml:"name"`
Required bool `yaml:"required"`
ContentType *string `yaml:"content_type"`
Description *string `yaml:"description"`
}
type promptMessageConfig struct {
@@ -93,10 +93,13 @@ type loaded struct {
result *configkit.Result[document]
backends map[string]map[string]fieldState
profiles map[string]map[string]fieldState
promptMetadata map[string]map[string][]promptInputPresence
prompts map[string]map[string]*promptEntry
ambiguous map[string]bool
registry *backend.Registry
indexedProfiles map[string]domain.ExecutionProfile
profileIDs []string
promptIDs []domain.PromptIdentity
}
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
@@ -112,8 +115,9 @@ func Load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
// load performs one ConfigKit lifecycle over the selected source tiers.
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
state := &loaded{
backends: make(map[string]map[string]fieldState),
profiles: make(map[string]map[string]fieldState),
backends: make(map[string]map[string]fieldState),
profiles: make(map[string]map[string]fieldState),
promptMetadata: make(map[string]map[string][]promptInputPresence),
}
result, err := configkit.LoadLayered(main, configkit.LayeredOptions[document]{
Underlays: underlays,
@@ -122,6 +126,7 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
Resolve: func(config *document, resolution *configkit.Resolution) error {
captureFields(config.Backends, "backends", backendFields, resolution, state.backends)
captureFields(config.Profiles, "profiles", profileFields, resolution, state.profiles)
capturePromptMetadata(config.Prompts, resolution, state.promptMetadata)
resolveBackends(config.Backends, resolution)
resolveProfiles(config.Profiles, resolution)
resolvePrompts(config.Prompts, resolution)
@@ -140,9 +145,53 @@ func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
if err := state.buildExecutionProfiles(); err != nil {
return nil, fmt.Errorf("load configuration: %w", err)
}
state.profileIDs = sortedKeys(state.indexedProfiles)
for _, id := range sortedKeys(state.prompts) {
for _, version := range sortedKeys(state.prompts[id]) {
state.promptIDs = append(state.promptIDs, domain.PromptIdentity{ID: id, Version: version})
}
}
return state, nil
}
type promptInputPresence struct {
contentType fieldState
description fieldState
}
func capturePromptMetadata(prompts map[string]map[string]promptConfig, resolution *configkit.Resolution, destination map[string]map[string][]promptInputPresence) {
for _, id := range sortedKeys(prompts) {
if destination[id] == nil {
destination[id] = make(map[string][]promptInputPresence, len(prompts[id]))
}
for _, version := range sortedKeys(prompts[id]) {
inputs := prompts[id][version].Inputs
states := make([]promptInputPresence, len(inputs))
base := configkit.Keys("prompts", id, version, "inputs")
for index := range inputs {
path := base.Index(index)
states[index] = promptInputPresence{
contentType: captureFieldState(path.Key("content_type"), resolution),
description: captureFieldState(path.Key("description"), resolution),
}
}
destination[id][version] = states
}
}
}
func captureFieldState(path configkit.Path, resolution *configkit.Resolution) fieldState {
info, exists := resolution.InputInfo(path)
switch {
case !exists || !info.Explicit:
return fieldAbsent
case info.Null:
return fieldNull
default:
return fieldValue
}
}
func captureFields[T any](entries map[string]T, collection string, fields []string, resolution *configkit.Resolution, destination map[string]map[string]fieldState) {
for id := range entries {
states := make(map[string]fieldState, len(fields))

View File

@@ -21,11 +21,12 @@ var (
)
type promptEntry struct {
definition domain.PromptDefinition
inspection domain.PromptInspection
hash string
session *template.Template
messages []*template.Template
definition domain.PromptDefinition
inspection domain.PromptInspection
requirementInputs []prompt.InputMetadata
hash string
session *template.Template
messages []*template.Template
}
func resolvePrompts(prompts map[string]map[string]promptConfig, resolution *configkit.Resolution) {
@@ -48,18 +49,24 @@ func resolvePrompts(prompts map[string]map[string]promptConfig, resolution *conf
for index := range definition.Inputs {
input := &definition.Inputs[index]
inputPath := base.Key("inputs").Index(index)
normalizedName := strings.TrimSpace(input.Name)
if normalizedName != input.Name {
configkit.Normalize(resolution, inputPath.Key("name"), &input.Name, normalizedName, "normalize input metadata")
inputsChanged = true
}
for _, field := range []struct {
name string
value *string
value **string
}{
{"name", &input.Name},
{"content_type", &input.ContentType},
{"description", &input.Description},
} {
normalized := strings.TrimSpace(*field.value)
if normalized != *field.value {
configkit.Normalize(resolution, inputPath.Key(field.name), field.value, normalized, "normalize input metadata")
inputsChanged = true
if *field.value != nil {
normalized := strings.TrimSpace(**field.value)
if normalized != **field.value {
configkit.Normalize(resolution, inputPath.Key(field.name), field.value, &normalized, "normalize input metadata")
inputsChanged = true
}
}
}
}
@@ -132,6 +139,10 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo
}
seenInputs := make(map[string]bool, len(raw.Inputs))
definition.Inputs = make([]domain.PromptInput, 0, len(raw.Inputs))
entry := &promptEntry{
messages: make([]*template.Template, 0, len(raw.Messages)),
requirementInputs: make([]prompt.InputMetadata, 0, len(raw.Inputs)),
}
for index, input := range raw.Inputs {
path := base.Key("inputs").Index(index)
if input.Name == "" {
@@ -141,11 +152,27 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo
return nil, fmt.Errorf("%s: input name is duplicated", path.Key("name"))
}
seenInputs[input.Name] = true
contentType := ""
if input.ContentType != nil {
contentType = *input.ContentType
}
description := ""
if input.Description != nil {
description = *input.Description
}
definition.Inputs = append(definition.Inputs, domain.PromptInput{
Name: input.Name, Required: input.Required, ContentType: input.ContentType, Description: input.Description,
Name: input.Name, Required: input.Required, ContentType: contentType, Description: description,
})
presence := state.promptMetadata[id][version][index]
metadata := prompt.InputMetadata{Name: input.Name, Required: input.Required}
if presence.contentType == fieldValue {
metadata.ContentType = stringPointer(contentType)
}
if presence.description == fieldValue {
metadata.Description = stringPointer(description)
}
entry.requirementInputs = append(entry.requirementInputs, metadata)
}
entry := &promptEntry{messages: make([]*template.Template, 0, len(raw.Messages))}
definition.Templates = make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
for index, message := range raw.Messages {
path := base.Key("messages").Index(index)
@@ -233,6 +260,11 @@ func (state *loaded) materializePrompt(id, version string, raw promptConfig, loo
return entry, nil
}
func stringPointer(value string) *string {
copy := value
return &copy
}
func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOrigin func(configkit.Path) (configkit.FieldInfo, bool)) (string, error) {
if strings.TrimSpace(name) == "" {
return "", fmt.Errorf("%s: content_file path must not be blank", fieldPath)
@@ -265,14 +297,14 @@ func (state *loaded) readContent(fieldPath configkit.Path, name string, lookupOr
return string(content), nil
}
func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domain.PromptInspection, error) {
func (state *loaded) lookupPromptEntry(id, version string) (*promptEntry, error) {
if state == nil || strings.TrimSpace(id) == "" || (version != "" && strings.TrimSpace(version) == "") {
return nil, domain.PromptInspection{}, errPromptNotFound
return nil, errPromptNotFound
}
versions := state.prompts[id]
if version == "" {
if state.ambiguous[id] {
return nil, domain.PromptInspection{}, errPromptAmbiguous
return nil, errPromptAmbiguous
}
for selected := range versions {
version = selected
@@ -280,9 +312,12 @@ func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domai
}
entry := versions[version]
if entry == nil {
return nil, domain.PromptInspection{}, errPromptNotFound
return nil, errPromptNotFound
}
definition := entry.definition
return entry, nil
}
func ownPromptDefinition(definition domain.PromptDefinition) *domain.PromptDefinition {
definition.Inputs = append([]domain.PromptInput(nil), definition.Inputs...)
definition.Templates = append([]domain.PromptMessageTemplate(nil), definition.Templates...)
for index := range definition.Templates {
@@ -291,9 +326,27 @@ func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domai
definition.Templates[index].CacheControl = &copy
}
}
inspection := entry.inspection
return &definition
}
func ownPromptInspection(inspection domain.PromptInspection) *domain.PromptInspection {
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
return &definition, inspection, nil
return &inspection
}
func promptLookupError(err error) error {
if errors.Is(err, errPromptNotFound) {
return promptdef.ErrPromptDefinitionNotFound
}
return err
}
func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domain.PromptInspection, error) {
entry, err := state.lookupPromptEntry(id, version)
if err != nil {
return nil, domain.PromptInspection{}, err
}
return ownPromptDefinition(entry.definition), *ownPromptInspection(entry.inspection), nil
}
// GetPromptSnapshot returns an independently owned captured definition and its
@@ -302,14 +355,24 @@ func (state *loaded) GetPromptSnapshot(ctx context.Context, id, version string)
if err := ctx.Err(); err != nil {
return nil, "", err
}
definition, inspection, err := state.prompt(id, version)
if errors.Is(err, errPromptNotFound) {
return nil, "", promptdef.ErrPromptDefinitionNotFound
}
entry, err := state.lookupPromptEntry(id, version)
if err != nil {
return nil, "", err
return nil, "", promptLookupError(err)
}
return definition, inspection.PromptHash, nil
return ownPromptDefinition(entry.definition), entry.hash, nil
}
// GetPromptInspection returns an independently owned inspection of one
// captured prompt without copying its execution templates.
func (state *loaded) GetPromptInspection(ctx context.Context, id, version string) (*domain.PromptInspection, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
entry, err := state.lookupPromptEntry(id, version)
if err != nil {
return nil, promptLookupError(err)
}
return ownPromptInspection(entry.inspection), nil
}
// GetPromptDefinition implements the runner's prompt lookup boundary.
@@ -329,3 +392,33 @@ func (state *loaded) GetCompiledTemplates(id, version string) (prompt.CompiledTe
}
return prompt.CompiledTemplates{Session: entry.session, Messages: entry.messages}, true
}
// GetRequirementSnapshot returns captured templates and input metadata for
// one exact prompt identity. Returned metadata and slices are independently
// owned; the compiled templates are shared immutable construction-time parses.
func (state *loaded) GetRequirementSnapshot(id, version string) (prompt.RequirementSnapshot, bool) {
if state == nil {
return prompt.RequirementSnapshot{}, false
}
entry := state.prompts[id][version]
if entry == nil {
return prompt.RequirementSnapshot{}, false
}
snapshot := prompt.RequirementSnapshot{
Templates: prompt.CompiledTemplates{
Session: entry.session,
Messages: append([]*template.Template(nil), entry.messages...),
},
Inputs: make([]prompt.InputMetadata, len(entry.requirementInputs)),
}
for index, input := range entry.requirementInputs {
snapshot.Inputs[index] = input
if input.ContentType != nil {
snapshot.Inputs[index].ContentType = stringPointer(*input.ContentType)
}
if input.Description != nil {
snapshot.Inputs[index].Description = stringPointer(*input.Description)
}
}
return snapshot, true
}

View File

@@ -1,6 +1,7 @@
package configuration
import (
"context"
"errors"
"io/fs"
"os"
@@ -10,6 +11,7 @@ import (
"testing/fstest"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
)
func TestLoadCapturesInheritedImportedContent(t *testing.T) {
@@ -134,6 +136,94 @@ func TestLoadRejectsMalformedStaticSyntaxAndAmbiguousSelection(t *testing.T) {
}
}
func TestPromptInspectionLookupMatchesSnapshotSelectionAndOwnsInputs(t *testing.T) {
makePrompt := func(content string) map[string]any {
return map[string]any{
"default_profile": "partial",
"inputs": []any{map[string]any{
"name": "document", "required": true,
"content_type": " text/plain ", "description": " source document ",
}},
"messages": []any{map[string]any{"role": "user", "content": content}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}
}
state, err := load(mapping("prompt inspection lookup", map[string]any{
"profiles": map[string]any{"partial": map[string]any{}},
"prompts": map[string]any{
"multi": map[string]any{"v1": makePrompt("first"), "v2": makePrompt("second")},
"single": map[string]any{"v1": makePrompt("only")},
},
}), nil)
if err != nil {
t.Fatal(err)
}
for _, selection := range []struct {
id, version string
}{
{id: "multi", version: "v2"},
{id: "single"},
} {
inspection, err := state.GetPromptInspection(context.Background(), selection.id, selection.version)
if err != nil {
t.Fatalf("inspect %q/%q: %v", selection.id, selection.version, err)
}
definition, hash, err := state.GetPromptSnapshot(context.Background(), selection.id, selection.version)
if err != nil {
t.Fatalf("snapshot %q/%q: %v", selection.id, selection.version, err)
}
if inspection.PromptID != definition.ID || inspection.PromptVersion != definition.Version ||
inspection.PromptHash != hash || inspection.DefaultProfileID != "partial" ||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "document" ||
inspection.Inputs[0].ContentType != "text/plain" || inspection.Inputs[0].Description != "source document" {
t.Fatalf("inspection did not match the selected snapshot: inspection=%+v definition=%+v hash=%q", inspection, definition, hash)
}
if len(definition.Inputs) != len(inspection.Inputs) || definition.Inputs[0].Name != inspection.Inputs[0].Name {
t.Fatalf("definition and inspection inputs diverged: definition=%+v inspection=%+v", definition.Inputs, inspection.Inputs)
}
}
owned, err := state.GetPromptInspection(context.Background(), "single", "")
if err != nil {
t.Fatal(err)
}
owned.Inputs[0].Name = "caller mutation"
owned.Inputs[0].Description = "caller mutation"
again, err := state.GetPromptInspection(context.Background(), "single", "v1")
if err != nil || again.Inputs[0].Name != "document" || again.Inputs[0].Description != "source document" {
t.Fatalf("inspection mutation escaped into captured state: inspection=%+v err=%v", again, err)
}
for _, selection := range []struct {
id, version string
}{
{id: "missing", version: "v1"},
{id: "multi", version: "V2"},
{id: " ", version: "v1"},
} {
inspection, err := state.GetPromptInspection(context.Background(), selection.id, selection.version)
if inspection != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
t.Errorf("inspection selection %q/%q=(%+v, %v), want not found", selection.id, selection.version, inspection, err)
}
definition, _, err := state.GetPromptSnapshot(context.Background(), selection.id, selection.version)
if definition != nil || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
t.Errorf("snapshot selection %q/%q=(%+v, %v), want not found", selection.id, selection.version, definition, err)
}
}
if _, err := state.GetPromptInspection(context.Background(), "multi", ""); !errors.Is(err, errPromptAmbiguous) {
t.Errorf("ambiguous inspection selection error=%v", err)
}
if _, _, err := state.GetPromptSnapshot(context.Background(), "multi", ""); !errors.Is(err, errPromptAmbiguous) {
t.Errorf("ambiguous snapshot selection error=%v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := state.GetPromptInspection(ctx, "single", "v1"); !errors.Is(err, context.Canceled) {
t.Errorf("canceled inspection selection error=%v", err)
}
}
func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T) {
state, err := load(mapping("empty-content", map[string]any{"prompts": map[string]any{" exact.id ": map[string]any{"v1": map[string]any{
"inputs": []any{map[string]any{"name": " note ", "required": true}},
@@ -170,6 +260,113 @@ func TestLoadAcceptsEmptyInlineContentAndRejectsInvalidPromptFields(t *testing.T
}
}
func TestRequirementSnapshotCapturesMetadataPresenceAndNormalizedValues(t *testing.T) {
state, err := load(mapping("requirement-metadata", map[string]any{
"prompts": map[string]any{"example": map[string]any{"v1": map[string]any{
"session_id": "{{.session}}",
"inputs": []any{
map[string]any{"name": "absent"},
map[string]any{"name": "null", "content_type": nil, "description": nil},
map[string]any{"name": "empty", "content_type": "", "description": ""},
map[string]any{"name": "normalized", "content_type": " text/plain ", "description": " final text "},
},
"messages": []any{map[string]any{"role": "user", "content": "{{input \"absent\"}}"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}}},
}), nil)
if err != nil {
var loadError *configkit.LoadError
if errors.As(err, &loadError) {
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
}
t.Fatal(err)
}
snapshot, ok := state.GetRequirementSnapshot("example", "v1")
if !ok || len(snapshot.Inputs) != 4 || len(snapshot.Templates.Messages) != 1 || snapshot.Templates.Session == nil {
t.Fatalf("requirement snapshot was not captured: %+v, %t", snapshot, ok)
}
for _, index := range []int{0, 1} {
if snapshot.Inputs[index].ContentType != nil || snapshot.Inputs[index].Description != nil {
t.Errorf("input %q should not have explicit string metadata: %+v", snapshot.Inputs[index].Name, snapshot.Inputs[index])
}
}
if snapshot.Inputs[2].ContentType == nil || *snapshot.Inputs[2].ContentType != "" ||
snapshot.Inputs[2].Description == nil || *snapshot.Inputs[2].Description != "" {
t.Errorf("explicit empty metadata presence was lost: %+v", snapshot.Inputs[2])
}
if snapshot.Inputs[3].ContentType == nil || *snapshot.Inputs[3].ContentType != "text/plain" ||
snapshot.Inputs[3].Description == nil || *snapshot.Inputs[3].Description != "final text" {
t.Errorf("snapshot did not use normalized final metadata: %+v", snapshot.Inputs[3])
}
definition, inspection, err := state.prompt("example", "v1")
if err != nil {
t.Fatal(err)
}
if len(definition.Inputs) != 4 || len(inspection.Inputs) != 4 || inspection.Inputs[2].ContentType != "" || inspection.Inputs[2].Description != "" ||
inspection.Inputs[3].ContentType != "text/plain" || inspection.Inputs[3].Description != "final text" {
t.Fatalf("existing prompt inspection shape or normalized values changed: %+v", inspection)
}
*snapshot.Inputs[2].ContentType = "caller mutation"
snapshot.Inputs[3].Name = "caller mutation"
snapshot.Templates.Messages[0] = nil
again, ok := state.GetRequirementSnapshot("example", "v1")
if !ok || *again.Inputs[2].ContentType != "" || again.Inputs[3].Name != "normalized" || again.Templates.Messages[0] == nil {
t.Fatalf("snapshot mutation escaped into captured state: %+v, %t", again, ok)
}
if _, ok := state.GetRequirementSnapshot("example", "V1"); ok {
t.Fatal("snapshot lookup unexpectedly changed exact version matching")
}
}
func TestInputMetadataPresenceDoesNotChangePromptHashOrInspection(t *testing.T) {
variants := []struct {
name string
fields map[string]any
}{
{name: "absent"},
{name: "null", fields: map[string]any{"content_type": nil, "description": nil}},
{name: "empty", fields: map[string]any{"content_type": "", "description": ""}},
}
var expectedHash string
for _, variant := range variants {
t.Run(variant.name, func(t *testing.T) {
input := map[string]any{"name": "document"}
for name, value := range variant.fields {
input[name] = value
}
state, err := load(mapping("hash-"+variant.name, map[string]any{
"prompts": map[string]any{"same": map[string]any{"v1": map[string]any{
"inputs": []any{input},
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}}},
}), nil)
if err != nil {
var loadError *configkit.LoadError
if errors.As(err, &loadError) {
t.Fatalf("%v: %+v", err, loadError.Diagnostics())
}
t.Fatal(err)
}
_, inspection, err := state.prompt("same", "v1")
if err != nil {
t.Fatal(err)
}
if len(inspection.Inputs) != 1 || inspection.Inputs[0].ContentType != "" || inspection.Inputs[0].Description != "" {
t.Fatalf("old inspection metadata changed: %+v", inspection.Inputs)
}
if expectedHash == "" {
expectedHash = inspection.PromptHash
} else if inspection.PromptHash != expectedHash {
t.Fatalf("metadata presence changed the definition hash: got %q, want %q", inspection.PromptHash, expectedHash)
}
})
}
}
func promptWithFile(name string) map[string]any {
return map[string]any{"prompts": map[string]any{"example": map[string]any{"v1": map[string]any{
"messages": []any{map[string]any{"role": "user", "content_file": name}},

View File

@@ -147,18 +147,26 @@ func overlayProviderMap(lower, upper map[string]any) (map[string]any, error) {
return merged, nil
}
// GetProfile returns an independently owned indexed profile for the internal
// runner's lookup boundary.
func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
func (state *loaded) lookupProfile(ctx context.Context, id string) (domain.ExecutionProfile, error) {
if err := ctx.Err(); err != nil {
return nil, err
return domain.ExecutionProfile{}, err
}
if state == nil {
return nil, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
return domain.ExecutionProfile{}, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
}
profile, found := state.indexedProfiles[id]
if !found {
return nil, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
return domain.ExecutionProfile{}, fmt.Errorf("%w: %q", profilepkg.ErrProfileNotFound, id)
}
return profile, nil
}
// GetProfile returns an independently owned indexed profile for the internal
// runner's lookup boundary.
func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
profile, err := state.lookupProfile(ctx, id)
if err != nil {
return nil, err
}
target := *profile.ResolvedTarget
params, err := jsonvalue.CopyMap(target.ExtraParams)
@@ -170,6 +178,15 @@ func (state *loaded) GetProfile(ctx context.Context, id string) (*domain.Executi
return &profile, nil
}
// GetProfileIdentity returns the exact lookup ID for a captured profile
// without copying its execution target.
func (state *loaded) GetProfileIdentity(ctx context.Context, id string) (string, error) {
if _, err := state.lookupProfile(ctx, id); err != nil {
return "", err
}
return id, nil
}
// GetBackend returns an independently owned effective backend definition.
func (state *loaded) GetBackend(id string) (domain.Backend, error) {
if state == nil {

View File

@@ -3,6 +3,7 @@ package configuration
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"reflect"
@@ -11,6 +12,7 @@ import (
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
profilepkg "gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
@@ -89,6 +91,18 @@ func TestLoadBuildsEffectiveTargetsAfterInheritance(t *testing.T) {
if partial.ResolvedTarget.Endpoint != "" || partial.ResolvedTarget.Model != "" {
t.Fatalf("partial profile gained a runnable target: %+v", partial.ResolvedTarget)
}
identity, err := state.GetProfileIdentity(context.Background(), "partial")
if err != nil || identity != "partial" {
t.Fatalf("partial profile identity=(%q, %v)", identity, err)
}
if _, err := state.GetProfileIdentity(context.Background(), "missing"); !errors.Is(err, profilepkg.ErrProfileNotFound) {
t.Fatalf("missing profile identity error=%v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := state.GetProfileIdentity(ctx, "partial"); !errors.Is(err, context.Canceled) {
t.Fatalf("canceled profile identity lookup error=%v", err)
}
endpointOnly, _ := state.GetProfile(context.Background(), "endpoint-only")
if endpointOnly.ResolvedTarget.Endpoint != "https://standalone.example/v1" ||
endpointOnly.ResolvedTarget.BackendID != "" || endpointOnly.ResolvedTarget.Model != "" {

View File

@@ -135,6 +135,12 @@ type Artifact struct {
Hash string
}
// PromptIdentity identifies one configured prompt version.
type PromptIdentity struct {
ID string
Version string
}
// PromptDefinition represents a configured prompt execution definition.
type PromptDefinition struct {
ID string `yaml:"id"`

View File

@@ -0,0 +1,58 @@
package domain
// RequirementKind classifies how a prompt may depend on an input or variable.
type RequirementKind string
const (
RequirementRequired RequirementKind = "required"
RequirementConditional RequirementKind = "conditional"
RequirementOptional RequirementKind = "optional"
)
// InputRequirement describes one declared or template-referenced input.
type InputRequirement struct {
Name string
Kind RequirementKind
Declared bool
ContentType *string
Description *string
Explanation string
}
// VariableRequirement describes one caller variable referenced by a prompt.
type VariableRequirement struct {
Name string
Kind RequirementKind
Supplied bool
Explanation string
}
// RequirementIssueCode identifies a limit that may affect requirement
// discovery.
type RequirementIssueCode string
const (
RequirementIssueMissingVariable RequirementIssueCode = "missing_variable"
RequirementIssueInputContent RequirementIssueCode = "input_content"
RequirementIssueDynamicInputName RequirementIssueCode = "dynamic_input_name"
RequirementIssueUnsupportedConstruct RequirementIssueCode = "unsupported_construct"
)
// RequirementIssue describes one limitation on requirement discovery.
type RequirementIssue struct {
Code RequirementIssueCode
InputName string
VariableName string
Description string
}
// RequirementInspection contains the inventories discovered for one prompt.
type RequirementInspection struct {
PromptID string
PromptVersion string
ProfileID string
Inputs []InputRequirement
Variables []VariableRequirement
Complete bool
Issues []RequirementIssue
}

View File

@@ -83,6 +83,12 @@ func (o *Options) SetSchemaFile(path string) error {
type Engine struct {
runner *usecase.Runner
inspection configuration.Inspection
listing identityInventory
}
type identityInventory interface {
ListProfiles(context.Context) ([]string, error)
ListPrompts(context.Context) ([]domain.PromptIdentity, error)
}
// New assembles an engine from facade-selected configuration and collaborators.
@@ -135,6 +141,7 @@ func New(cfg Config, options Options) (*Engine, error) {
return &Engine{
runner: usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries),
inspection: snapshot.Inspection(),
listing: snapshot,
}, nil
}
@@ -178,9 +185,23 @@ func fileSource(name string) (fs.FS, string, string, error) {
func (e *Engine) InspectPrompt(ctx context.Context, id, version string) (*domain.PromptInspection, error) {
return e.runner.InspectPrompt(ctx, id, version)
}
func (e *Engine) InspectRequirements(
ctx context.Context,
request usecase.RequirementInspectionRequest,
) (*domain.RequirementInspection, error) {
return e.runner.InspectRequirements(ctx, request)
}
func (e *Engine) InspectProfile(ctx context.Context, id string) (*domain.ProfileInspection, error) {
return e.runner.InspectProfile(ctx, id)
}
func (e *Engine) ListProfiles(ctx context.Context) ([]string, error) {
return e.listing.ListProfiles(ctx)
}
func (e *Engine) ListPrompts(ctx context.Context) ([]domain.PromptIdentity, error) {
return e.listing.ListPrompts(ctx)
}
func (e *Engine) Prepare(ctx context.Context, request domain.RunRequest) (*domain.PreparedRun, error) {
return e.runner.Prepare(ctx, request)
}

View File

@@ -152,6 +152,11 @@ func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (
payload, err := json.Marshal(wirePayload)
if err != nil {
for key, value := range req.Target.ExtraParams {
if _, extraErr := json.Marshal(value); extraErr != nil {
return nil, fmt.Errorf("%w: extra_params.%s must be JSON-serializable: %v", ErrInvalidRequest, key, extraErr)
}
}
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
}
apiKey := strings.TrimSpace(req.Target.APIKey)
@@ -446,9 +451,6 @@ func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
if IsReservedOpenAIChatRequestField(key) {
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
}
if _, err := json.Marshal(value); err != nil {
return nil, fmt.Errorf("extra_params.%s must be JSON-serializable: %w", key, err)
}
out[key] = value
}

View File

@@ -1113,6 +1113,13 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
}{
{name: "empty key", extraParams: map[string]any{"": "empty"}, want: "key must not be empty"},
{name: "unserializable value", extraParams: map[string]any{"bad": math.Inf(1)}, want: "JSON-serializable"},
{
name: "nested unserializable value",
extraParams: map[string]any{
"nested": map[string]any{"values": []any{"valid", math.NaN()}},
},
want: "extra_params.nested",
},
}
for _, key := range []string{
"model",
@@ -1161,6 +1168,56 @@ func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *te
}
}
func TestOpenAICompatibleClientClassifiesSchemaEncodingFailures(t *testing.T) {
tests := []struct {
name string
extraParams map[string]any
want error
unwanted error
}{
{
name: "serializable extra keeps full-request failure",
extraParams: map[string]any{"client_option": "accepted"},
want: ErrRequestFailed,
unwanted: ErrInvalidRequest,
},
{
name: "invalid extra keeps invalid-request failure",
extraParams: map[string]any{"nested": map[string]any{"bad": math.Inf(1)}},
want: ErrInvalidRequest,
unwanted: ErrRequestFailed,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
provider := newRecordingProvider(t)
client := newProviderClient(t, provider, OpenAICompatibleConfig{})
request := ordinaryGenerateRequest()
request.Target.ExtraParams = tc.extraParams
request.StructuredOutput = &domain.StructuredOutputSpec{
Type: domain.StructuredOutputJSONSchema,
JSONSchema: &domain.StructuredOutputJSONSpec{
Name: "invalid-schema",
Strict: true,
Schema: json.RawMessage("{"),
},
}
_, err := client.Generate(context.Background(), request)
if err == nil || !errors.Is(err, tc.want) {
t.Fatalf("error = %v, want identity %v", err, tc.want)
}
if errors.Is(err, tc.unwanted) {
t.Fatalf("error = %v, unexpectedly has identity %v", err, tc.unwanted)
}
if calls := provider.requestCount(); calls != 0 {
t.Fatalf("provider calls = %d, want 0", calls)
}
})
}
}
func checkConfiguredModelFallback(t *testing.T) {
provider := newRecordingProvider(t)
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "default-model"})

View File

@@ -0,0 +1,119 @@
package llm
import (
"encoding/json"
"runtime"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
var prepareGenerationBenchmarkSink *PreparedGeneration
func BenchmarkPrepareGeneration(b *testing.B) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "https://api.example.test/v1",
Model: "benchmark-model",
})
if err != nil {
b.Fatalf("construct benchmark client: %v", err)
}
prompt := domain.RenderedPrompt{
Messages: []domain.RenderedMessage{{Role: "user", Content: "Summarize this request."}},
}
tests := []struct {
name string
request domain.GenerateRequest
}{
{
name: "no_extras",
request: domain.GenerateRequest{
Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model"},
},
},
{
name: "small_extras",
request: domain.GenerateRequest{
Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
"priority": "low",
"attempt": 3,
"enabled": true,
}},
},
},
{
name: "1MiB_string_extra",
request: domain.GenerateRequest{
Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
"payload": strings.Repeat("x", 1<<20),
}},
},
},
{
name: "nested_json_extras",
request: domain.GenerateRequest{
Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model", ExtraParams: map[string]any{
"options": map[string]any{
"enabled": true,
"limits": []any{2, 4, 8},
"tags": []string{"summary", "offline"},
},
}},
},
},
{
name: "structured_output",
request: domain.GenerateRequest{
Prompt: prompt,
Target: domain.ExecutionTarget{Model: "benchmark-model"},
StructuredOutput: &domain.StructuredOutputSpec{
Type: domain.StructuredOutputJSONSchema,
JSONSchema: &domain.StructuredOutputJSONSpec{
Name: "summary",
Strict: true,
Schema: map[string]any{
"type": "object",
"properties": map[string]any{
"summary": map[string]any{"type": "string"},
"count": map[string]any{"type": "integer"},
},
"required": []string{"summary", "count"},
"additionalProperties": false,
},
},
},
},
},
}
for _, tc := range tests {
b.Run(tc.name, func(b *testing.B) {
prepared, err := client.PrepareGeneration(tc.request)
if err != nil {
b.Fatalf("check prepared request: %v", err)
}
if !json.Valid(prepared.payload) {
b.Fatal("prepared request is not valid JSON")
}
b.ReportAllocs()
b.SetBytes(int64(len(prepared.payload)))
b.ResetTimer()
for range b.N {
prepared, err := client.PrepareGeneration(tc.request)
if err != nil {
b.Fatalf("prepare generation: %v", err)
}
prepareGenerationBenchmarkSink = prepared
}
b.StopTimer()
runtime.KeepAlive(prepareGenerationBenchmarkSink)
})
}
}

View File

@@ -0,0 +1,254 @@
package prompt
import (
"bytes"
"context"
"runtime"
"strconv"
"testing"
"text/template"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
const rendererBenchmarkArtifactSize = 256 << 10
var (
renderedPromptBenchmarkSink *domain.RenderedPrompt
renderedPromptsBenchmarkSink []*domain.RenderedPrompt
)
type rendererBenchmarkIdentity struct {
id string
version string
}
type rendererBenchmarkLookup struct {
identity rendererBenchmarkIdentity
templates CompiledTemplates
}
func (lookup rendererBenchmarkLookup) GetCompiledTemplates(id, version string) (CompiledTemplates, bool) {
if id != lookup.identity.id || version != lookup.identity.version {
return CompiledTemplates{}, false
}
return lookup.templates, true
}
type compiledRenderingBenchmarkCase struct {
name string
renderer Renderer
definition *domain.PromptDefinition
inputs map[string]*domain.Artifact
vars map[string]string
ctx context.Context
outputBytes int
check func(*testing.B, *domain.RenderedPrompt)
}
func BenchmarkCompiledRendering(b *testing.B) {
body := bytes.Repeat([]byte("x"), rendererBenchmarkArtifactSize)
bodyText := string(body)
small := newCompiledRenderingBenchmarkCase(b,
"small_message", "", []string{"Summary for {{.audience}}."},
&domain.PromptDefinition{
ID: "compiled-render-benchmark", Version: "v1",
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "Summary for {{.audience}}."}},
}, nil, map[string]string{"audience": "project leads"},
len("Summary for project leads."),
func(b *testing.B, rendered *domain.RenderedPrompt) {
if rendered.SessionID != "" || len(rendered.Messages) != 1 || rendered.Messages[0].Content != "Summary for project leads." {
b.Fatalf("unexpected small compiled render: %+v", rendered)
}
},
)
artifactInputs := map[string]*domain.Artifact{
"document": {Name: "document", Body: body},
}
largeContent := `{{input "document"}}`
largeDefinition := &domain.PromptDefinition{
ID: "compiled-render-benchmark", Version: "v1",
Inputs: []domain.PromptInput{{Name: "document"}},
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: largeContent}},
}
large := newCompiledRenderingBenchmarkCase(b,
"artifact_backed_256KiB", "", []string{largeContent},
largeDefinition, artifactInputs, nil, len(bodyText),
func(b *testing.B, rendered *domain.RenderedPrompt) {
if len(rendered.Messages) != 1 {
b.Fatalf("artifact render message count=%d, want 1", len(rendered.Messages))
}
if rendered.Messages[0].Content != bodyText {
b.Fatalf("artifact render length=%d, want %d", len(rendered.Messages[0].Content), len(bodyText))
}
},
)
repeatedSession := `document-size={{len (input "document")}}`
repeatedMessages := []string{
`{{input "document"}}`,
`{{input "document"}}|{{input "document"}}`,
}
repeatedDefinition := &domain.PromptDefinition{
ID: "compiled-render-benchmark", Version: "v1", SessionID: repeatedSession,
Inputs: []domain.PromptInput{{Name: "document"}},
Templates: []domain.PromptMessageTemplate{
{Role: "system", Content: repeatedMessages[0]},
{Role: "user", Content: repeatedMessages[1]},
},
}
repeatedOutputBytes := len(bodyText)*3 + 1
repeated := newCompiledRenderingBenchmarkCase(b,
"repeated_across_session_and_messages", repeatedSession, repeatedMessages,
repeatedDefinition, artifactInputs, nil, repeatedOutputBytes,
func(b *testing.B, rendered *domain.RenderedPrompt) {
if rendered.SessionID != "document-size="+strconv.Itoa(len(bodyText)) || len(rendered.Messages) != 2 ||
rendered.Messages[0].Content != bodyText || rendered.Messages[1].Content != bodyText+"|"+bodyText {
b.Fatalf("unexpected repeated compiled render: session=%q message lengths=%v", rendered.SessionID, renderedMessageLengths(rendered))
}
},
)
for _, benchmark := range []compiledRenderingBenchmarkCase{small, large, repeated} {
b.Run(benchmark.name, func(b *testing.B) {
rendered, err := benchmark.renderer.Render(benchmark.ctx, benchmark.definition, benchmark.inputs, benchmark.vars)
if err != nil {
b.Fatalf("check compiled renderer output: %v", err)
}
benchmark.check(b, rendered)
b.ReportAllocs()
b.SetBytes(int64(benchmark.outputBytes))
b.ResetTimer()
for range b.N {
rendered, err := benchmark.renderer.Render(benchmark.ctx, benchmark.definition, benchmark.inputs, benchmark.vars)
if err != nil {
b.Fatalf("render compiled template: %v", err)
}
renderedPromptBenchmarkSink = rendered
}
b.StopTimer()
runtime.KeepAlive(renderedPromptBenchmarkSink)
})
}
}
// BenchmarkRetainedRenderedOutput reports post-GC live heap for a fixed batch
// of rendered outputs above the same fixture's setup-only baseline.
func BenchmarkRetainedRenderedOutput(b *testing.B) {
body := bytes.Repeat([]byte("x"), rendererBenchmarkArtifactSize)
bodyText := string(body)
content := `{{input "document"}}`
definition := &domain.PromptDefinition{
ID: "retained-render-benchmark", Version: "v1",
Inputs: []domain.PromptInput{{Name: "document"}},
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: content}},
}
compiled := CompiledTemplates{Messages: []*template.Template{mustCompileRendererBenchmarkTemplate(b, content)}}
renderer := NewCompiledRenderer(rendererBenchmarkLookup{
identity: rendererBenchmarkIdentity{id: definition.ID, version: definition.Version},
templates: compiled,
})
inputs := map[string]*domain.Artifact{"document": {Name: "document", Body: body}}
ctx := context.Background()
const retainedBatchSize = 4
retained := make([]*domain.RenderedPrompt, retainedBatchSize)
check, err := renderer.Render(ctx, definition, inputs, nil)
if err != nil {
b.Fatalf("check retained-output fixture: %v", err)
}
if len(check.Messages) != 1 || check.Messages[0].Content != bodyText {
b.Fatalf("retained-output fixture returned unexpected content")
}
check = nil
// Repeated -count runs must start from the same setup-only heap baseline.
renderedPromptBenchmarkSink = nil
renderedPromptsBenchmarkSink = nil
runtime.GC()
var setupMemory runtime.MemStats
runtime.ReadMemStats(&setupMemory)
runtime.KeepAlive(renderer)
runtime.KeepAlive(definition)
runtime.KeepAlive(inputs)
b.ReportAllocs()
b.SetBytes(int64(len(bodyText)))
b.ResetTimer()
for iteration := 0; iteration < b.N; iteration++ {
rendered, err := renderer.Render(ctx, definition, inputs, nil)
if err != nil {
b.Fatalf("render retained-output fixture: %v", err)
}
retained[iteration%len(retained)] = rendered
}
b.StopTimer()
for i, rendered := range retained {
if rendered != nil {
continue
}
rendered, err := renderer.Render(ctx, definition, inputs, nil)
if err != nil {
b.Fatalf("fill retained-output fixture: %v", err)
}
retained[i] = rendered
}
renderedPromptsBenchmarkSink = retained
runtime.GC()
var retainedMemory runtime.MemStats
runtime.ReadMemStats(&retainedMemory)
b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
runtime.KeepAlive(retained)
runtime.KeepAlive(renderer)
runtime.KeepAlive(definition)
runtime.KeepAlive(inputs)
}
func newCompiledRenderingBenchmarkCase(
b *testing.B,
name string,
session string,
messages []string,
definition *domain.PromptDefinition,
inputs map[string]*domain.Artifact,
vars map[string]string,
outputBytes int,
check func(*testing.B, *domain.RenderedPrompt),
) compiledRenderingBenchmarkCase {
compiled := CompiledTemplates{}
if session != "" {
compiled.Session = mustCompileRendererBenchmarkTemplate(b, session)
}
for _, content := range messages {
compiled.Messages = append(compiled.Messages, mustCompileRendererBenchmarkTemplate(b, content))
}
return compiledRenderingBenchmarkCase{
name: name,
renderer: NewCompiledRenderer(rendererBenchmarkLookup{
identity: rendererBenchmarkIdentity{id: definition.ID, version: definition.Version},
templates: compiled,
}),
definition: definition, inputs: inputs, vars: vars, ctx: context.Background(),
outputBytes: outputBytes, check: check,
}
}
func mustCompileRendererBenchmarkTemplate(b *testing.B, source string) *template.Template {
b.Helper()
compiled, err := CompileTemplate("benchmark", source)
if err != nil {
b.Fatalf("compile benchmark template: %v", err)
}
return compiled
}
func renderedMessageLengths(rendered *domain.RenderedPrompt) []int {
lengths := make([]int, len(rendered.Messages))
for i, message := range rendered.Messages {
lengths[i] = len(message.Content)
}
return lengths
}

View File

@@ -1,7 +1,6 @@
package prompt
import (
"bytes"
"context"
"errors"
"fmt"
@@ -126,7 +125,7 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
if err := ctx.Err(); err != nil {
return nil, err
}
var buf bytes.Buffer
var buf strings.Builder
executeErr := tmpl.Execute(&buf, vars)
if err := ctx.Err(); err != nil {
return nil, err
@@ -235,7 +234,7 @@ func executeSessionTemplate(ctx context.Context, tmpl *template.Template, vars m
if err := ctx.Err(); err != nil {
return "", err
}
var buf bytes.Buffer
var buf strings.Builder
executeErr := tmpl.Execute(&buf, vars)
if err := ctx.Err(); err != nil {
return "", err

View File

@@ -4,8 +4,11 @@ import (
"bytes"
"context"
"errors"
"fmt"
"strings"
"sync"
"testing"
"text/template"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
@@ -501,6 +504,77 @@ func TestGoRendererArtifactTextLifecycle(t *testing.T) {
}
}
func TestCompiledRendererKeepsConcurrentRequestsIndependent(t *testing.T) {
const (
promptID = "concurrent-render"
version = "v1"
)
sessionTemplate, err := CompileTemplate("session", "session-{{.request}}")
if err != nil {
t.Fatalf("compile session template: %v", err)
}
messageTemplate, err := CompileTemplate("message", `request={{.request}};document={{input "document"}}`)
if err != nil {
t.Fatalf("compile message template: %v", err)
}
renderer := NewCompiledRenderer(compiledRendererTestLookup{
id: promptID,
version: version,
templates: CompiledTemplates{
Session: sessionTemplate,
Messages: []*template.Template{messageTemplate},
},
})
definition := &domain.PromptDefinition{
ID: promptID, Version: version, SessionID: "session-{{.request}}",
Inputs: []domain.PromptInput{{Name: "document"}},
Templates: []domain.PromptMessageTemplate{{Role: "user", Content: `request={{.request}};document={{input "document"}}`}},
}
const requestCount = 24
errs := make(chan error, requestCount)
var wait sync.WaitGroup
wait.Add(requestCount)
for i := 0; i < requestCount; i++ {
go func(i int) {
defer wait.Done()
request := fmt.Sprintf("request-%d", i)
document := fmt.Sprintf("document-%d", i)
rendered, err := renderer.Render(context.Background(), definition,
map[string]*domain.Artifact{"document": {Body: []byte(document)}},
map[string]string{"request": request},
)
if err != nil {
errs <- fmt.Errorf("render %s: %w", request, err)
return
}
if rendered.SessionID != "session-"+request || len(rendered.Messages) != 1 ||
rendered.Messages[0].Content != "request="+request+";document="+document {
errs <- fmt.Errorf("render %s returned session %q and messages %#v", request, rendered.SessionID, rendered.Messages)
}
}(i)
}
wait.Wait()
close(errs)
for err := range errs {
t.Error(err)
}
}
type compiledRendererTestLookup struct {
id, version string
templates CompiledTemplates
}
func (lookup compiledRendererTestLookup) GetCompiledTemplates(id, version string) (CompiledTemplates, bool) {
if id != lookup.id || version != lookup.version {
return CompiledTemplates{}, false
}
return lookup.templates, true
}
// BenchmarkGoRendererArtifactReferences measures NewGoRenderer's per-call
// template-parsing fallback rather than the captured compiled path.
func BenchmarkGoRendererArtifactReferences(b *testing.B) {
body := bytes.Repeat([]byte("document content "), (artifactTextChunkSize*4)/len("document content "))
inputs := map[string]*domain.Artifact{

View File

@@ -0,0 +1,978 @@
package prompt
import (
"context"
"sort"
"text/template"
"text/template/parse"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
type requirementReachability uint8
const (
requirementDefinite requirementReachability = iota
requirementPossible
requirementInactive
)
type dotIdentity uint8
const (
dotRoot dotIdentity = iota
dotNonRoot
dotUnknown
)
type inputEvidence struct {
name string
kind domain.RequirementKind
declared bool
applicable bool
contentType *string
description *string
explanation string
}
type variableEvidence struct {
name string
kind domain.RequirementKind
applicable bool
explanation string
}
type requirementIssueKey struct {
code domain.RequirementIssueCode
inputName string
variableName string
}
type conditionEvaluation struct {
value any
known bool
unsupported bool
inputNameUnknown bool
missingVariables map[string]struct{}
contentInputs map[string]struct{}
}
func (evaluation conditionEvaluation) truth() bool {
truth, ok := template.IsTrue(evaluation.value)
return evaluation.known && ok && truth
}
func (walker *requirementWalker) evaluateCondition(pipe *parse.PipeNode, reach requirementReachability, dot dotIdentity) conditionEvaluation {
if pipe == nil {
return conditionEvaluation{unsupported: true}
}
if len(pipe.Cmds) == 1 && len(pipe.Decl) == 0 && !pipe.IsAssign {
return walker.evaluateCommand(pipe.Cmds[0], reach, dot)
}
if name, ok := directInputPipeline(pipe); ok {
return walker.evaluateInputName(name, reach, dot)
}
before := walker.dependencies
visitReach := requirementPossible
if reach == requirementInactive {
visitReach = requirementInactive
}
if err := walker.visitPipe(pipe, visitReach, dot); err != nil {
return conditionEvaluation{unsupported: true}
}
return conditionEvaluation{unsupported: walker.dependencies != before}
}
func (walker *requirementWalker) evaluateCommand(command *parse.CommandNode, reach requirementReachability, dot dotIdentity) conditionEvaluation {
if command == nil || len(command.Args) == 0 {
return conditionEvaluation{unsupported: true}
}
identifier, isIdentifier := command.Args[0].(*parse.IdentifierNode)
if isIdentifier {
switch identifier.Ident {
case "input":
if reach == requirementInactive {
if len(command.Args) == 2 {
if name, ok := literalStringValue(command.Args[1]); ok {
walker.recordInput(name, requirementInactive)
return conditionEvaluation{}
}
}
for _, argument := range command.Args[1:] {
_ = walker.visit(argument, requirementInactive, dot)
}
return conditionEvaluation{}
}
if len(command.Args) == 2 {
if name, ok := command.Args[1].(*parse.StringNode); ok {
walker.recordInput(name.Text, reach)
return conditionEvaluation{contentInputs: map[string]struct{}{name.Text: {}}}
}
return walker.evaluateInputName(command.Args[1], reach, dot)
}
for _, argument := range command.Args[1:] {
_ = walker.visit(argument, requirementPossible, dot)
}
return walker.unresolvedInputName(conditionEvaluation{unsupported: true})
case "index":
if len(command.Args) == 3 {
if key, ok := command.Args[2].(*parse.StringNode); ok && isRootMap(command.Args[1], dot, walker.rootKnown) {
walker.recordOptionalVariable(key.Text, reach)
value, supplied := walker.vars[key.Text]
if !supplied {
value = ""
}
return conditionEvaluation{value: value, known: true}
}
}
case "not":
if len(command.Args) == 2 {
evaluation := walker.evaluateValue(command.Args[1], reach, dot)
if evaluation.known {
truth, ok := template.IsTrue(evaluation.value)
if ok {
evaluation.value = !truth
} else {
evaluation.known = false
evaluation.unsupported = true
}
}
return evaluation
}
case "eq", "ne":
if len(command.Args) == 3 {
left := walker.evaluateValue(command.Args[1], reach, dot)
right := walker.evaluateValue(command.Args[2], reach, dot)
result := mergeConditionEvaluations(left, right)
if left.known && right.known {
leftString, leftOK := left.value.(string)
rightString, rightOK := right.value.(string)
if leftOK && rightOK {
equal := leftString == rightString
if identifier.Ident == "ne" {
equal = !equal
}
result.value = equal
result.known = true
return result
}
result.unsupported = true
}
return result
}
}
}
if !isIdentifier && len(command.Args) == 1 {
return walker.evaluateValue(command.Args[0], reach, dot)
}
if reach == requirementInactive {
_ = walker.visitCommand(command, reach, dot)
return conditionEvaluation{}
}
before := walker.dependencies
_ = walker.visitCommand(command, requirementPossible, dot)
return conditionEvaluation{unsupported: walker.dependencies != before}
}
func (walker *requirementWalker) evaluateValue(node parse.Node, reach requirementReachability, dot dotIdentity) conditionEvaluation {
switch node := node.(type) {
case *parse.DotNode:
if dot == dotRoot {
return conditionEvaluation{value: walker.vars, known: true}
}
if dot == dotUnknown {
walker.unresolvedVariableScope(reach)
}
return conditionEvaluation{unsupported: true}
case *parse.StringNode:
return conditionEvaluation{value: node.Text, known: true}
case *parse.BoolNode:
return conditionEvaluation{value: node.True, known: true}
case *parse.NilNode:
return conditionEvaluation{value: nil, known: true}
case *parse.FieldNode:
if dot != dotRoot || len(node.Ident) == 0 {
if dot == dotUnknown {
walker.unresolvedVariableScope(reach)
}
return conditionEvaluation{unsupported: true}
}
name := node.Ident[0]
walker.recordVariable(name, reach)
if len(node.Ident) > 1 {
walker.addIssue("", name, "A chained field access is outside the direct-key scope.")
return conditionEvaluation{unsupported: true}
}
value, supplied := walker.vars[name]
if !supplied {
return conditionEvaluation{missingVariables: map[string]struct{}{name: {}}}
}
return conditionEvaluation{value: value, known: true}
case *parse.VariableNode:
if len(node.Ident) == 1 && node.Ident[0] == "$" && walker.rootKnown {
return conditionEvaluation{value: walker.vars, known: true}
}
if len(node.Ident) < 2 || node.Ident[0] != "$" || !walker.rootKnown {
walker.visitVariable(node, reach)
return conditionEvaluation{unsupported: true}
}
name := node.Ident[1]
walker.recordVariable(name, reach)
if len(node.Ident) > 2 {
walker.addIssue("", name, "A chained field access is outside the direct-key scope.")
return conditionEvaluation{unsupported: true}
}
value, supplied := walker.vars[name]
if !supplied {
return conditionEvaluation{missingVariables: map[string]struct{}{name: {}}}
}
return conditionEvaluation{value: value, known: true}
case *parse.CommandNode:
return walker.evaluateCommand(node, reach, dot)
case *parse.PipeNode:
return walker.evaluateCondition(node, reach, dot)
case *parse.ChainNode:
before := walker.dependencies
visitReach := requirementPossible
if reach == requirementInactive {
visitReach = requirementInactive
}
_ = walker.visit(node.Node, visitReach, dot)
return conditionEvaluation{unsupported: walker.dependencies != before}
default:
return conditionEvaluation{unsupported: true}
}
}
func (walker *requirementWalker) evaluateInputName(node parse.Node, reach requirementReachability, dot dotIdentity) conditionEvaluation {
evaluation := walker.evaluateValue(node, reach, dot)
if evaluation.known {
if name, ok := evaluation.value.(string); ok {
walker.recordInput(name, reach)
return conditionEvaluation{contentInputs: map[string]struct{}{name: {}}}
}
evaluation.unsupported = true
}
// Value analysis has already visited recognizable references, including
// the conservative fallback for unsupported expressions. Do not rescan
// the same subtree: nested computed names would multiply that work.
return walker.unresolvedInputName(evaluation)
}
func (walker *requirementWalker) unresolvedInputName(evaluation conditionEvaluation) conditionEvaluation {
for name := range evaluation.missingVariables {
walker.addIssueCode(domain.RequirementIssueMissingVariable, "", name, "A missing variable prevents resolving an input name.")
}
for name := range evaluation.contentInputs {
walker.addIssueCode(domain.RequirementIssueInputContent, name, "", "Captured input content affects an input name.")
}
walker.addIssueCode(domain.RequirementIssueDynamicInputName, "", "", "An input helper name is not statically available.")
walker.unknownInputEffect = true
evaluation.inputNameUnknown = true
return evaluation
}
func directInputPipeline(pipe *parse.PipeNode) (parse.Node, bool) {
if pipe == nil || len(pipe.Cmds) != 2 || len(pipe.Decl) != 0 || pipe.IsAssign {
return nil, false
}
input := pipe.Cmds[1]
if len(input.Args) != 1 {
return nil, false
}
identifier, ok := input.Args[0].(*parse.IdentifierNode)
if !ok || identifier.Ident != "input" {
return nil, false
}
value := pipe.Cmds[0]
if len(value.Args) > 0 {
valueIdentifier, isValueIdentifier := value.Args[0].(*parse.IdentifierNode)
if isValueIdentifier && valueIdentifier.Ident == "input" && len(value.Args) == 2 {
return value, true
}
}
if len(value.Args) == 1 {
return value.Args[0], true
}
if len(value.Args) == 3 {
function, ok := value.Args[0].(*parse.IdentifierNode)
if ok && function.Ident == "index" {
return value, true
}
}
return nil, false
}
func literalStringValue(node parse.Node) (string, bool) {
switch node := node.(type) {
case *parse.StringNode:
return node.Text, true
case *parse.PipeNode:
if len(node.Cmds) == 1 && len(node.Cmds[0].Args) == 1 && len(node.Decl) == 0 && !node.IsAssign {
return literalStringValue(node.Cmds[0].Args[0])
}
}
return "", false
}
func mergeConditionEvaluations(left, right conditionEvaluation) conditionEvaluation {
result := conditionEvaluation{
unsupported: left.unsupported || right.unsupported,
inputNameUnknown: left.inputNameUnknown || right.inputNameUnknown,
missingVariables: make(map[string]struct{}, len(left.missingVariables)+len(right.missingVariables)),
contentInputs: make(map[string]struct{}, len(left.contentInputs)+len(right.contentInputs)),
}
for name := range left.missingVariables {
result.missingVariables[name] = struct{}{}
}
for name := range right.missingVariables {
result.missingVariables[name] = struct{}{}
}
for name := range left.contentInputs {
result.contentInputs[name] = struct{}{}
}
for name := range right.contentInputs {
result.contentInputs[name] = struct{}{}
}
return result
}
func branchReach(parent requirementReachability, known, truth, selected bool) requirementReachability {
if parent == requirementInactive {
return requirementInactive
}
if !known {
return requirementPossible
}
if truth == selected {
if parent == requirementPossible {
return requirementPossible
}
return requirementDefinite
}
return requirementInactive
}
type requirementWalker struct {
ctx context.Context
vars map[string]string
inputs map[string]*inputEvidence
variables map[string]*variableEvidence
issues map[requirementIssueKey]domain.RequirementIssue
complete bool
rootKnown bool
unknownInputEffect bool
unknownVariableEffect bool
dependencies int
templateSet *template.Template
visitedTrees map[*parse.Tree]bool
}
// AnalyzeRequirements collects the prompt dependencies recognizable from its
// captured syntax. Branch bodies remain conditional until their conditions are
// classified; unsupported dependency-bearing syntax makes the result partial.
func AnalyzeRequirements(
ctx context.Context,
definition *domain.PromptDefinition,
snapshot RequirementSnapshot,
vars map[string]string,
sessionBypassed bool,
) (domain.RequirementInspection, error) {
if err := ctx.Err(); err != nil {
return domain.RequirementInspection{}, err
}
result := domain.RequirementInspection{Complete: true}
if definition != nil {
result.PromptID = definition.ID
result.PromptVersion = definition.Version
}
walker := &requirementWalker{
ctx: ctx,
vars: vars,
inputs: make(map[string]*inputEvidence, len(snapshot.Inputs)),
variables: make(map[string]*variableEvidence),
issues: make(map[requirementIssueKey]domain.RequirementIssue),
complete: true,
rootKnown: true,
}
for _, input := range snapshot.Inputs {
if err := ctx.Err(); err != nil {
return domain.RequirementInspection{}, err
}
kind := domain.RequirementOptional
if input.Required {
kind = domain.RequirementRequired
}
walker.inputs[input.Name] = &inputEvidence{
name: input.Name,
kind: kind,
declared: true,
contentType: copyString(input.ContentType),
description: copyString(input.Description),
}
if kind == domain.RequirementOptional {
walker.inputs[input.Name].explanation = "Declared as an optional input."
}
}
for _, message := range snapshot.Templates.Messages {
if err := walker.walkTemplate(message); err != nil {
return domain.RequirementInspection{}, err
}
}
if !sessionBypassed && snapshot.Templates.Session != nil {
if err := walker.walkTemplate(snapshot.Templates.Session); err != nil {
return domain.RequirementInspection{}, err
}
}
if err := ctx.Err(); err != nil {
return domain.RequirementInspection{}, err
}
return walker.result(result), nil
}
func (walker *requirementWalker) walkTemplate(compiled *template.Template) error {
if err := walker.ctx.Err(); err != nil {
return err
}
if compiled == nil || compiled.Tree == nil || compiled.Tree.Root == nil {
walker.addIssue("", "", "A captured template has no parsed root.")
return nil
}
previousSet := walker.templateSet
previousVisited := walker.visitedTrees
walker.templateSet = compiled
walker.visitedTrees = map[*parse.Tree]bool{compiled.Tree: true}
walker.rootKnown = true
err := walker.visit(compiled.Tree.Root, requirementDefinite, dotRoot)
walker.templateSet = previousSet
walker.visitedTrees = previousVisited
return err
}
func (walker *requirementWalker) visit(node parse.Node, reach requirementReachability, dot dotIdentity) error {
if err := walker.ctx.Err(); err != nil {
return err
}
if node == nil {
return nil
}
switch node := node.(type) {
case *parse.ListNode:
if node == nil {
return nil
}
for _, child := range node.Nodes {
if err := walker.visit(child, reach, dot); err != nil {
return err
}
}
case *parse.ActionNode:
return walker.visit(node.Pipe, reach, dot)
case *parse.IfNode:
if reach == requirementInactive {
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
return err
}
if err := walker.visit(node.List, requirementInactive, dot); err != nil {
return err
}
return walker.visit(node.ElseList, requirementInactive, dot)
}
condition := walker.evaluateCondition(node.Pipe, reach, dot)
before := walker.dependencies
thenReach := branchReach(reach, condition.known, condition.truth(), true)
elseReach := branchReach(reach, condition.known, condition.truth(), false)
if err := walker.visit(node.List, thenReach, dot); err != nil {
return err
}
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
return err
}
if !condition.known && walker.dependencies != before {
walker.recordConditionLimitation(condition)
}
return nil
case *parse.WithNode:
if reach == requirementInactive {
if err := walker.visit(node.Pipe, requirementInactive, dot); err != nil {
return err
}
bodyDot := walker.withDotIdentity(node.Pipe, dot, conditionEvaluation{})
if err := walker.visit(node.List, requirementInactive, bodyDot); err != nil {
return err
}
return walker.visit(node.ElseList, requirementInactive, dot)
}
condition := walker.evaluateCondition(node.Pipe, reach, dot)
before := walker.dependencies
thenReach := branchReach(reach, condition.known, condition.truth(), true)
elseReach := branchReach(reach, condition.known, condition.truth(), false)
bodyDot := walker.withDotIdentity(node.Pipe, dot, condition)
if err := walker.visit(node.List, thenReach, bodyDot); err != nil {
return err
}
if err := walker.visit(node.ElseList, elseReach, dot); err != nil {
return err
}
if !condition.known && walker.dependencies != before {
walker.recordConditionLimitation(condition)
}
return nil
case *parse.RangeNode:
if err := walker.visit(node.Pipe, reach, dot); err != nil {
return err
}
before := walker.dependencies
bodyReach := requirementPossible
if reach == requirementInactive {
bodyReach = requirementInactive
}
if err := walker.visit(node.List, bodyReach, dotNonRoot); err != nil {
return err
}
if err := walker.visit(node.ElseList, bodyReach, dot); err != nil {
return err
}
if reach != requirementInactive && walker.dependencies != before {
walker.addIssue("", "", "Dependencies inside a range are only conservatively enumerated.")
}
case *parse.TemplateNode:
pipeReach := requirementPossible
if reach == requirementInactive {
pipeReach = requirementInactive
}
if err := walker.visit(node.Pipe, pipeReach, dot); err != nil {
return err
}
if reach == requirementInactive {
return nil
}
called := walker.templateSet.Lookup(node.Name)
if called == nil || called.Tree == nil || called.Tree.Root == nil {
walker.addIssue("", "", "An invoked named template is unavailable for syntax inspection.")
return nil
}
if walker.visitedTrees[called.Tree] {
walker.addIssue("", "", "An invoked named template is scanned conservatively.")
return nil
}
walker.visitedTrees[called.Tree] = true
rootKnown := walker.rootKnown
walker.rootKnown = false
walker.unknownVariableEffect = true
err := walker.visit(called.Tree.Root, requirementPossible, dotNonRoot)
walker.rootKnown = rootKnown
if err != nil {
return err
}
walker.addIssue("", "", "An invoked named template is scanned conservatively.")
case *parse.PipeNode:
return walker.visitPipe(node, reach, dot)
case *parse.CommandNode:
return walker.visitCommand(node, reach, dot)
case *parse.FieldNode:
if dot == dotRoot && len(node.Ident) > 0 {
walker.recordVariable(node.Ident[0], reach)
if len(node.Ident) > 1 && reach != requirementInactive {
walker.addIssue("", node.Ident[0], "A chained field access is outside the direct-key scope.")
}
}
if dot == dotUnknown {
walker.unresolvedVariableScope(reach)
}
case *parse.DotNode:
if dot == dotUnknown {
walker.unresolvedVariableScope(reach)
}
case *parse.VariableNode:
walker.visitVariable(node, reach)
case *parse.ChainNode:
before := walker.dependencies
chainReach := requirementPossible
if reach == requirementInactive {
chainReach = requirementInactive
}
if err := walker.visit(node.Node, chainReach, dot); err != nil {
return err
}
if reach != requirementInactive && walker.dependencies != before {
walker.addIssue("", "", "A chained expression is not interpreted.")
}
case *parse.BreakNode, *parse.ContinueNode:
if reach != requirementInactive {
walker.addIssue("", "", "Loop control is outside the supported syntax scope.")
}
}
return nil
}
func (walker *requirementWalker) visitPipe(pipe *parse.PipeNode, reach requirementReachability, dot dotIdentity) error {
if pipe == nil {
return nil
}
if len(pipe.Cmds) == 1 && len(pipe.Decl) == 0 && !pipe.IsAssign {
return walker.visitCommand(pipe.Cmds[0], reach, dot)
}
if name, ok := directInputPipeline(pipe); ok {
if reach == requirementInactive {
if literal, known := literalStringValue(name); known {
walker.recordInput(literal, requirementInactive)
return nil
}
return walker.visit(name, requirementInactive, dot)
}
walker.evaluateInputName(name, reach, dot)
return nil
}
before := walker.dependencies
commandReach := requirementPossible
if reach == requirementInactive {
commandReach = requirementInactive
}
for _, command := range pipe.Cmds {
if err := walker.visitCommand(command, commandReach, dot); err != nil {
return err
}
}
if reach != requirementInactive && walker.dependencies != before {
walker.addIssue("", "", "A pipeline or variable assignment is not interpreted.")
}
if reach != requirementInactive {
for _, declaration := range pipe.Decl {
if len(declaration.Ident) > 0 && declaration.Ident[0] == "$" {
walker.rootKnown = false
walker.unknownVariableEffect = true
walker.addIssue("", "", "The root template binding is reassigned.")
}
}
}
return nil
}
func (walker *requirementWalker) visitCommand(command *parse.CommandNode, reach requirementReachability, dot dotIdentity) error {
if command == nil || len(command.Args) == 0 {
return nil
}
identifier, isIdentifier := command.Args[0].(*parse.IdentifierNode)
if isIdentifier && identifier.Ident == "input" {
if reach == requirementInactive {
if len(command.Args) == 2 {
if name, ok := literalStringValue(command.Args[1]); ok {
walker.recordInput(name, reach)
return nil
}
}
for _, argument := range command.Args[1:] {
if err := walker.visit(argument, requirementInactive, dot); err != nil {
return err
}
}
return nil
}
if len(command.Args) == 2 {
if name, ok := command.Args[1].(*parse.StringNode); ok {
walker.recordInput(name.Text, reach)
return nil
}
walker.evaluateInputName(command.Args[1], reach, dot)
return nil
}
for _, argument := range command.Args[1:] {
if err := walker.visit(argument, requirementPossible, dot); err != nil {
return err
}
}
walker.unresolvedInputName(conditionEvaluation{unsupported: true})
return nil
}
if isIdentifier && identifier.Ident == "index" && len(command.Args) == 3 {
if _, ok := command.Args[2].(*parse.StringNode); ok && isRootMap(command.Args[1], dot, walker.rootKnown) {
key := command.Args[2].(*parse.StringNode).Text
walker.recordOptionalVariable(key, reach)
return nil
}
// A local map may alias the caller root even though its value is not
// tracked. A lookup on it must not vanish from complete discovery.
if variable, ok := unwrapSingleValue(command.Args[1]).(*parse.VariableNode); ok && len(variable.Ident) == 1 {
walker.unresolvedVariableScope(reach)
}
}
if !isIdentifier && len(command.Args) == 1 {
return walker.visit(command.Args[0], reach, dot)
}
before := walker.dependencies
for index, argument := range command.Args {
if isIdentifier && index == 0 {
continue
}
argumentReach := requirementPossible
if reach == requirementInactive {
argumentReach = requirementInactive
}
if err := walker.visit(argument, argumentReach, dot); err != nil {
return err
}
}
if reach != requirementInactive && walker.dependencies != before {
walker.addIssue("", "", "A function call or computed expression is not interpreted.")
}
return nil
}
func (walker *requirementWalker) visitVariable(variable *parse.VariableNode, reach requirementReachability) {
if len(variable.Ident) < 2 {
return
}
if variable.Ident[0] != "$" {
walker.unresolvedVariableScope(reach)
return
}
if !walker.rootKnown {
if reach != requirementInactive {
walker.unknownVariableEffect = true
walker.addIssue("", "", "The root template binding is not known.")
}
return
}
walker.recordVariable(variable.Ident[1], reach)
if len(variable.Ident) > 2 && reach != requirementInactive {
walker.addIssue("", variable.Ident[1], "A chained field access is outside the direct-key scope.")
}
}
func (walker *requirementWalker) unresolvedVariableScope(reach requirementReachability) {
if reach != requirementInactive {
walker.unknownVariableEffect = true
walker.addIssue("", "", "A caller-variable scope is not statically known.")
}
}
func (walker *requirementWalker) withDotIdentity(pipe *parse.PipeNode, dot dotIdentity, condition conditionEvaluation) dotIdentity {
if pipe != nil && len(pipe.Cmds) == 1 && len(pipe.Decl) == 0 && !pipe.IsAssign && len(pipe.Cmds[0].Args) == 1 &&
isRootMap(pipe.Cmds[0].Args[0], dot, walker.rootKnown) {
return dotRoot
}
if condition.known {
if _, root := condition.value.(map[string]string); root {
return dotRoot
}
return dotNonRoot
}
if condition.inputNameUnknown ||
!condition.unsupported && (len(condition.missingVariables) > 0 || len(condition.contentInputs) > 0) {
// Supported caller values and input bodies are strings; supported
// comparisons and negation return booleans, never the root map.
return dotNonRoot
}
return dotUnknown
}
func isRootMap(node parse.Node, dot dotIdentity, rootKnown bool) bool {
switch node := unwrapSingleValue(node).(type) {
case *parse.DotNode:
return dot == dotRoot
case *parse.VariableNode:
return rootKnown && len(node.Ident) == 1 && node.Ident[0] == "$"
default:
return false
}
}
func unwrapSingleValue(node parse.Node) parse.Node {
for {
pipe, ok := node.(*parse.PipeNode)
if !ok || len(pipe.Cmds) != 1 || len(pipe.Decl) != 0 || pipe.IsAssign || len(pipe.Cmds[0].Args) != 1 {
return node
}
node = pipe.Cmds[0].Args[0]
}
}
func (walker *requirementWalker) recordInput(name string, reach requirementReachability) {
evidence := walker.inputs[name]
if evidence == nil {
evidence = &inputEvidence{
name: name, kind: domain.RequirementOptional,
explanation: "Declared as an optional input.",
}
walker.inputs[name] = evidence
}
if reach == requirementInactive {
if !evidence.applicable && evidence.kind != domain.RequirementRequired {
evidence.explanation = "This input is referenced only in a skipped template path."
}
} else {
evidence.applicable = true
walker.promoteInput(evidence, kindForReach(reach))
if evidence.kind == domain.RequirementRequired {
evidence.explanation = ""
} else if reach == requirementPossible {
evidence.explanation = "This reference is not known to be reached on every execution path."
}
}
if reach != requirementInactive {
walker.dependencies++
}
}
func (walker *requirementWalker) recordVariable(name string, reach requirementReachability) {
evidence := walker.variables[name]
if evidence == nil {
evidence = &variableEvidence{name: name, kind: domain.RequirementOptional}
walker.variables[name] = evidence
}
if reach == requirementInactive {
if !evidence.applicable {
evidence.explanation = "This variable is referenced only in a skipped template path."
}
} else {
evidence.applicable = true
walker.promoteVariable(evidence, kindForReach(reach))
if evidence.kind == domain.RequirementRequired {
evidence.explanation = ""
} else if reach == requirementPossible {
evidence.explanation = "This reference is not known to be reached on every execution path."
}
}
if reach != requirementInactive {
walker.dependencies++
}
}
func (walker *requirementWalker) recordOptionalVariable(name string, reach requirementReachability) {
evidence := walker.variables[name]
if evidence == nil {
evidence = &variableEvidence{name: name, kind: domain.RequirementOptional}
walker.variables[name] = evidence
}
if reach == requirementInactive {
if !evidence.applicable {
evidence.explanation = "This variable is referenced only in a skipped template path."
}
} else {
evidence.applicable = true
}
if evidence.kind == domain.RequirementOptional && reach != requirementInactive {
evidence.explanation = "This key is read through an optional map lookup."
}
if reach != requirementInactive {
walker.dependencies++
}
}
func (walker *requirementWalker) promoteInput(evidence *inputEvidence, kind domain.RequirementKind) {
if requirementRank(kind) > requirementRank(evidence.kind) {
evidence.kind = kind
}
if evidence.kind == domain.RequirementRequired {
evidence.explanation = ""
}
}
func (walker *requirementWalker) promoteVariable(evidence *variableEvidence, kind domain.RequirementKind) {
if requirementRank(kind) > requirementRank(evidence.kind) {
evidence.kind = kind
}
if evidence.kind == domain.RequirementRequired {
evidence.explanation = ""
}
}
func kindForReach(reach requirementReachability) domain.RequirementKind {
if reach == requirementPossible {
return domain.RequirementConditional
}
return domain.RequirementRequired
}
func requirementRank(kind domain.RequirementKind) int {
switch kind {
case domain.RequirementRequired:
return 3
case domain.RequirementConditional:
return 2
default:
return 1
}
}
func (walker *requirementWalker) addIssue(inputName, variableName, description string) {
walker.addIssueCode(domain.RequirementIssueUnsupportedConstruct, inputName, variableName, description)
}
func (walker *requirementWalker) addIssueCode(code domain.RequirementIssueCode, inputName, variableName, description string) {
walker.complete = false
issue := domain.RequirementIssue{
Code: code,
InputName: inputName,
VariableName: variableName,
Description: description,
}
key := requirementIssueKey{code: issue.Code, inputName: inputName, variableName: variableName}
walker.issues[key] = issue
}
func (walker *requirementWalker) recordConditionLimitation(condition conditionEvaluation) {
for name := range condition.missingVariables {
walker.addIssueCode(domain.RequirementIssueMissingVariable, "", name, "A missing variable prevents selecting a dependency-bearing branch.")
}
for name := range condition.contentInputs {
walker.addIssueCode(domain.RequirementIssueInputContent, name, "", "Captured input content affects branch selection.")
}
if condition.unsupported && !condition.inputNameUnknown ||
(len(condition.missingVariables) == 0 && len(condition.contentInputs) == 0 && !condition.inputNameUnknown) {
walker.addIssue("", "", "A dependency-bearing branch condition is not resolved by the supported syntax analysis.")
}
}
func (walker *requirementWalker) result(result domain.RequirementInspection) domain.RequirementInspection {
result.Complete = walker.complete
result.Inputs = make([]domain.InputRequirement, 0, len(walker.inputs))
for _, evidence := range walker.inputs {
if walker.unknownInputEffect && evidence.kind != domain.RequirementRequired {
evidence.kind = domain.RequirementConditional
evidence.explanation = "An unresolved input name could refer to this input."
}
result.Inputs = append(result.Inputs, domain.InputRequirement{
Name: evidence.name, Kind: evidence.kind, Declared: evidence.declared,
ContentType: copyString(evidence.contentType), Description: copyString(evidence.description),
Explanation: evidence.explanation,
})
}
sort.Slice(result.Inputs, func(i, j int) bool { return result.Inputs[i].Name < result.Inputs[j].Name })
result.Variables = make([]domain.VariableRequirement, 0, len(walker.variables))
for _, evidence := range walker.variables {
if walker.unknownVariableEffect && evidence.kind != domain.RequirementRequired {
evidence.kind = domain.RequirementConditional
evidence.explanation = "An unresolved caller-variable scope could refer to this variable."
}
_, supplied := walker.vars[evidence.name]
result.Variables = append(result.Variables, domain.VariableRequirement{
Name: evidence.name, Kind: evidence.kind, Supplied: supplied, Explanation: evidence.explanation,
})
}
sort.Slice(result.Variables, func(i, j int) bool { return result.Variables[i].Name < result.Variables[j].Name })
result.Issues = make([]domain.RequirementIssue, 0, len(walker.issues))
for _, issue := range walker.issues {
result.Issues = append(result.Issues, issue)
}
sort.Slice(result.Issues, func(i, j int) bool {
left, right := result.Issues[i], result.Issues[j]
if left.Code != right.Code {
return left.Code < right.Code
}
if left.InputName != right.InputName {
return left.InputName < right.InputName
}
return left.VariableName < right.VariableName
})
return result
}
func copyString(value *string) *string {
if value == nil {
return nil
}
copy := *value
return &copy
}

View File

@@ -0,0 +1,671 @@
package prompt
import (
"context"
"errors"
"reflect"
"strings"
"testing"
"text/template"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestAnalyzeRequirementsDoesNotHideCallerMapScopes(t *testing.T) {
for _, source := range []string{
`{{with .}}{{.audience}}{{end}}`,
`{{with $}}{{.audience}}{{end}}`,
`{{with (.)}}{{.audience}}{{end}}`,
`{{$alias := .}}{{with $alias}}{{.audience}}{{end}}`,
`{{with .}}{{index . "audience"}}{{end}}`,
`{{$alias := .}}{{index $alias "audience"}}`,
`{{$alias := .}}{{index ($alias) "audience"}}`,
`{{$alias := .}}{{$alias.audience}}`,
} {
t.Run(source, func(t *testing.T) {
compiled := compileRequirementTemplate(t, "scope", source)
vars := map[string]string{"audience": "visible"}
var rendered strings.Builder
if err := compiled.Execute(&rendered, vars); err != nil || rendered.String() != "visible" {
t.Fatalf("caller-map reference did not render: %q, %v", rendered.String(), err)
}
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
}, vars, false)
if err != nil {
t.Fatal(err)
}
if result.Complete {
if len(result.Variables) != 1 || result.Variables[0].Name != "audience" {
t.Fatalf("complete discovery lost the caller key: %+v", result)
}
} else if len(result.Issues) == 0 {
t.Fatalf("partial discovery did not explain its limitation: %+v", result)
}
})
}
}
func TestAnalyzeRequirementsUnknownScopesAffectOptionalEntriesOnlyWhenActive(t *testing.T) {
for _, access := range []string{`{{with $alias}}{{.audience}}{{end}}`, `{{index $alias "audience"}}`, `{{index ($alias) "audience"}}`, `{{$alias.audience}}`} {
for _, active := range []bool{false, true} {
source := `{{if false}}{{.optional}}{{end}}{{$alias := .}}` + access
if !active {
source = `{{if false}}` + source + `{{end}}`
}
compiled := compileRequirementTemplate(t, "unknown-scope", source)
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
}, map[string]string{"audience": "visible"}, false)
if err != nil {
t.Fatal(err)
}
wantKind := domain.RequirementOptional
if active {
wantKind = domain.RequirementConditional
}
if result.Complete == active || len(result.Variables) != 1 || result.Variables[0].Name != "optional" || result.Variables[0].Kind != wantKind {
t.Fatalf("active=%v, source=%s, result=%+v", active, source, result)
}
if active && (len(result.Issues) == 0 || result.Issues[0].Code != domain.RequirementIssueUnsupportedConstruct) {
t.Fatalf("active unknown scope lacks a structured issue: %+v", result)
}
if !active && len(result.Issues) != 0 {
t.Fatalf("skipped unknown scope introduced uncertainty: %+v", result)
}
}
}
}
func TestRequirementWalkerDoesNotRescanNestedUnsupportedNames(t *testing.T) {
for _, depth := range []int{6, 12} {
expression := ".selector"
for level := 0; level < depth; level++ {
expression = `input (printf "%s" (` + expression + `))`
}
source := "{{" + expression + "}}"
compiled := compileRequirementTemplate(t, "computed-name", source)
walker := &requirementWalker{
ctx: context.Background(), vars: map[string]string{"selector": "binding"},
inputs: make(map[string]*inputEvidence), variables: make(map[string]*variableEvidence),
issues: make(map[requirementIssueKey]domain.RequirementIssue), complete: true, rootKnown: true,
}
if err := walker.walkTemplate(compiled); err != nil {
t.Fatal(err)
}
// A generous source-relative bound detects exponential rescanning, not
// an exact visit count or a production analysis limit.
if walker.dependencies > len(source)*8 {
t.Fatalf("depth=%d: %d accesses for %d bytes of source", depth, walker.dependencies, len(source))
}
result := walker.result(domain.RequirementInspection{})
if result.Complete || len(result.Variables) != 1 || result.Variables[0].Name != "selector" ||
result.Variables[0].Kind != domain.RequirementConditional || len(result.Issues) == 0 {
t.Fatalf("nested unsupported names lost discovery metadata: %+v", result)
}
}
}
func TestAnalyzeRequirementsCollectsDeclaredAndLiteralInputs(t *testing.T) {
description := "retained metadata"
definition := &domain.PromptDefinition{ID: "capture", Version: "v1"}
snapshot := RequirementSnapshot{
Inputs: []InputMetadata{
{Name: "optional", ContentType: stringPointerForTest("text/plain"), Description: &description},
{Name: "unused", Required: true},
{Name: "transcript", Required: true},
},
Templates: CompiledTemplates{Messages: []*template.Template{
compileRequirementTemplate(t, "message", `{{input "transcript"}} {{input "extra"}} {{input "extra"}}`),
}},
}
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, nil, false)
if err != nil {
t.Fatal(err)
}
if result.PromptID != "capture" || result.PromptVersion != "v1" || !result.Complete || len(result.Issues) != 0 {
t.Fatalf("unexpected result identity or completeness: %+v", result)
}
wantNames := []string{"extra", "optional", "transcript", "unused"}
if got := requirementInputNames(result.Inputs); !reflect.DeepEqual(got, wantNames) {
t.Fatalf("input names = %v, want %v", got, wantNames)
}
if got := result.Inputs[0]; got.Kind != domain.RequirementRequired || got.Declared || got.ContentType != nil || got.Description != nil {
t.Fatalf("undeclared literal input was not required independently: %+v", got)
}
if got := result.Inputs[1]; got.Kind != domain.RequirementOptional || !got.Declared || got.ContentType == nil || *got.ContentType != "text/plain" ||
got.Description == nil || *got.Description != description || got.Explanation == "" {
t.Fatalf("optional declaration metadata was not retained: %+v", got)
}
if result.Inputs[2].Kind != domain.RequirementRequired || !result.Inputs[2].Declared || result.Inputs[3].Kind != domain.RequirementRequired {
t.Fatalf("required declarations or literal use were lost: %+v", result.Inputs)
}
*result.Inputs[1].Description = "caller mutation"
if *snapshot.Inputs[0].Description != description {
t.Fatal("result metadata pointer aliases the captured snapshot")
}
}
func TestAnalyzeRequirementsDistinguishesDirectAndOptionalCallerKeys(t *testing.T) {
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Templates: CompiledTemplates{Messages: []*template.Template{
compileRequirementTemplate(t, "message", `{{.missing}} {{$.present}} {{index . "lookup"}}`),
}},
}, map[string]string{"present": ""}, false)
if err != nil {
t.Fatal(err)
}
if !result.Complete || len(result.Issues) != 0 {
t.Fatalf("plain root and literal lookup syntax should be complete: %+v", result)
}
wantNames := []string{"lookup", "missing", "present"}
if got := requirementVariableNames(result.Variables); !reflect.DeepEqual(got, wantNames) {
t.Fatalf("variable names = %v, want %v", got, wantNames)
}
if result.Variables[0].Kind != domain.RequirementOptional || result.Variables[0].Supplied || result.Variables[0].Explanation == "" {
t.Errorf("literal index lookup should be optional and absent: %+v", result.Variables[0])
}
if result.Variables[1].Kind != domain.RequirementRequired || result.Variables[1].Supplied {
t.Errorf("missing direct variable should be required but not supplied: %+v", result.Variables[1])
}
if result.Variables[2].Kind != domain.RequirementRequired || !result.Variables[2].Supplied {
t.Errorf("empty supplied direct variable was not detected by presence: %+v", result.Variables[2])
}
}
func TestAnalyzeRequirementsSeparatesRootAndLocalTemplateValues(t *testing.T) {
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Templates: CompiledTemplates{Messages: []*template.Template{
compileRequirementTemplate(t, "message", `{{$local := "local"}}{{with .context}}{{$local}} {{.child}} {{$.root}}{{end}}`),
}},
}, nil, false)
if err != nil {
t.Fatal(err)
}
wantNames := []string{"context", "root"}
if got := requirementVariableNames(result.Variables); !reflect.DeepEqual(got, wantNames) {
t.Fatalf("root variable names = %v, want %v", got, wantNames)
}
if result.Variables[0].Kind != domain.RequirementRequired || result.Variables[1].Kind != domain.RequirementConditional || result.Complete {
t.Fatalf("with scope and unresolved branch should be conservative: %+v", result)
}
if len(result.Issues) != 1 || result.Issues[0].Code != domain.RequirementIssueMissingVariable || result.Issues[0].VariableName != "context" {
t.Fatalf("conditional root use should have one associated limitation: %+v", result.Issues)
}
}
func TestAnalyzeRequirementsMarksBranchAndUnsupportedScopeReferences(t *testing.T) {
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Templates: CompiledTemplates{Messages: []*template.Template{
compileRequirementTemplate(t, "message", `{{if .enabled}}{{input "maybe"}}{{end}}{{range .items}}{{input "inside"}} {{.local}} {{$.root}}{{end}}`),
}},
}, nil, false)
if err != nil {
t.Fatal(err)
}
wantInputs := []string{"inside", "maybe"}
wantVars := []string{"enabled", "items", "root"}
if got := requirementInputNames(result.Inputs); !reflect.DeepEqual(got, wantInputs) {
t.Fatalf("input names = %v, want %v", got, wantInputs)
}
if got := requirementVariableNames(result.Variables); !reflect.DeepEqual(got, wantVars) {
t.Fatalf("variable names = %v, want %v", got, wantVars)
}
if result.Complete || len(result.Issues) < 2 {
t.Fatalf("branch/range limitations were not reported: %+v", result)
}
for _, input := range result.Inputs {
if input.Kind != domain.RequirementConditional {
t.Errorf("input %q should be conservative conditional: %+v", input.Name, input)
}
}
if result.Variables[0].Kind != domain.RequirementRequired || result.Variables[1].Kind != domain.RequirementRequired ||
result.Variables[2].Kind != domain.RequirementConditional {
t.Errorf("root and loop-scope variables were misclassified: %+v", result.Variables)
}
}
func TestAnalyzeRequirementsExcludesUnusedDefinitionsAndBypassedSession(t *testing.T) {
message := compileRequirementTemplate(t, "message", `{{define "unused"}}{{input "unused"}}{{end}}main`)
session := compileRequirementTemplate(t, "session", `{{input "session_only"}}`)
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Templates: CompiledTemplates{Messages: []*template.Template{message}, Session: session},
}, nil, true)
if err != nil {
t.Fatal(err)
}
if !result.Complete || len(result.Inputs) != 0 {
t.Fatalf("unused definitions or bypassed session were included: %+v", result)
}
result, err = AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Templates: CompiledTemplates{Messages: []*template.Template{message}, Session: session},
}, nil, false)
if err != nil {
t.Fatal(err)
}
if len(result.Inputs) != 1 || result.Inputs[0].Name != "session_only" || result.Inputs[0].Kind != domain.RequirementRequired {
t.Fatalf("applicable session template was not visited: %+v", result)
}
}
func TestAnalyzeRequirementsScansInvokedDefinitionsConservatively(t *testing.T) {
message := compileRequirementTemplate(t, "message", `{{define "unused"}}{{input "unused"}}{{end}}{{define "called"}}{{input "called"}} {{.local}}{{end}}main {{template "called" .}}`)
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Templates: CompiledTemplates{Messages: []*template.Template{message}},
}, nil, false)
if err != nil {
t.Fatal(err)
}
if result.Complete || len(result.Inputs) != 1 || result.Inputs[0].Name != "called" ||
result.Inputs[0].Kind != domain.RequirementConditional || len(result.Variables) != 0 {
t.Fatalf("called definition was not scanned conservatively: %+v", result)
}
if len(result.Issues) < 1 || result.Issues[0].Code != domain.RequirementIssueUnsupportedConstruct {
t.Fatalf("named-template limitation was not reported: %+v", result.Issues)
}
}
func TestAnalyzeRequirementsClassifiesKnownConditions(t *testing.T) {
tests := []struct {
name string
condition string
vars map[string]string
wantActive string
wantSkipped string
}{
{name: "string false is truthy", condition: `.flag`, vars: map[string]string{"flag": "false"}, wantActive: "active", wantSkipped: "inactive"},
{name: "empty direct value is false", condition: `.flag`, vars: map[string]string{"flag": ""}, wantActive: "inactive", wantSkipped: "active"},
{name: "simple equality", condition: `eq .mode "active"`, vars: map[string]string{"mode": "active"}, wantActive: "active", wantSkipped: "inactive"},
{name: "simple inequality", condition: `ne .mode "active"`, vars: map[string]string{"mode": "other"}, wantActive: "active", wantSkipped: "inactive"},
{name: "not direct value", condition: `not .flag`, vars: map[string]string{"flag": ""}, wantActive: "active", wantSkipped: "inactive"},
{name: "literal true", condition: `true`, wantActive: "active", wantSkipped: "inactive"},
{name: "literal false", condition: `false`, wantActive: "inactive", wantSkipped: "active"},
{name: "missing index is empty", condition: `index . "toggle"`, wantActive: "inactive", wantSkipped: "active"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
Role: "user", Content: `{{if ` + test.condition + `}}{{input "active"}}{{else}}{{input "inactive"}}{{end}}`,
}}}
snapshot := RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
}}}
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, test.vars, false)
if err != nil {
t.Fatal(err)
}
if !result.Complete || len(result.Issues) != 0 {
t.Fatalf("known condition should produce a complete result: %+v", result)
}
inputs := make(map[string]domain.InputRequirement, len(result.Inputs))
for _, input := range result.Inputs {
inputs[input.Name] = input
}
if inputs[test.wantActive].Kind != domain.RequirementRequired || inputs[test.wantSkipped].Kind != domain.RequirementOptional || inputs[test.wantSkipped].Explanation == "" {
t.Fatalf("selected/skipped inputs were misclassified: %+v", result.Inputs)
}
if test.condition == `index . "toggle"` {
if len(result.Variables) != 1 || result.Variables[0].Name != "toggle" || result.Variables[0].Kind != domain.RequirementOptional || result.Variables[0].Supplied {
t.Fatalf("missing index key was not retained as optional: %+v", result.Variables)
}
}
artifacts := map[string]*domain.Artifact{
"active": {Name: "active", Body: []byte("active")},
"inactive": {Name: "inactive", Body: []byte("inactive")},
}
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
if err != nil {
t.Fatal(err)
}
selected := "active"
if inputs[selected].Kind != domain.RequirementRequired {
selected = "inactive"
}
if len(rendered.Messages) != 1 || rendered.Messages[0].Content != selected {
t.Fatalf("analysis and renderer disagree: result=%+v rendered=%+v", result, rendered)
}
})
}
}
func TestAnalyzeRequirementsHandlesWithDotAndNestedReachability(t *testing.T) {
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
Role: "user", Content: `{{with .context}}{{input "inside"}} {{.child}} {{$.root}}{{else}}{{input "empty"}}{{end}}`,
}}}
snapshot := RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
}}}
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, map[string]string{"context": "value", "root": "root"}, false)
if err != nil {
t.Fatal(err)
}
if !result.Complete || len(result.Inputs) != 2 || len(result.Variables) != 2 {
t.Fatalf("known with condition should have a complete inventory: %+v", result)
}
if result.Inputs[0].Name != "empty" || result.Inputs[0].Kind != domain.RequirementOptional ||
result.Inputs[1].Name != "inside" || result.Inputs[1].Kind != domain.RequirementRequired {
t.Fatalf("with body and else reachability is incorrect: %+v", result.Inputs)
}
if result.Variables[0].Name != "context" || result.Variables[0].Kind != domain.RequirementRequired ||
result.Variables[1].Name != "root" || result.Variables[1].Kind != domain.RequirementRequired {
t.Fatalf("with dot was confused with the root caller map: %+v", result.Variables)
}
nested := compileRequirementTemplate(t, "nested", `{{if .outer}}{{if .inner}}{{input "inner"}}{{else}}{{input "inner_else"}}{{end}}{{else}}{{input "outer_else"}}{{end}}`)
nestedResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Templates: CompiledTemplates{Messages: []*template.Template{nested}},
}, map[string]string{"outer": "yes", "inner": "yes"}, false)
if err != nil {
t.Fatal(err)
}
if !nestedResult.Complete || nestedResult.Inputs[0].Name != "inner" || nestedResult.Inputs[0].Kind != domain.RequirementRequired ||
nestedResult.Inputs[1].Name != "inner_else" || nestedResult.Inputs[1].Kind != domain.RequirementOptional ||
nestedResult.Inputs[2].Name != "outer_else" || nestedResult.Inputs[2].Kind != domain.RequirementOptional {
t.Fatalf("known nested conditions lost selected or inactive reachability: %+v", nestedResult)
}
}
func TestAnalyzeRequirementsReportsUnresolvedDecisionAndContentConditions(t *testing.T) {
missing := compileRequirementTemplate(t, "missing", `{{if eq .mode "active"}}{{input "yes"}}{{else}}{{input "no"}}{{end}}`)
missingResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Templates: CompiledTemplates{Messages: []*template.Template{missing}},
}, nil, false)
if err != nil {
t.Fatal(err)
}
if missingResult.Complete || len(missingResult.Issues) != 1 || missingResult.Issues[0].Code != domain.RequirementIssueMissingVariable ||
missingResult.Issues[0].VariableName != "mode" || missingResult.Inputs[0].Kind != domain.RequirementConditional ||
missingResult.Inputs[1].Kind != domain.RequirementConditional {
t.Fatalf("missing decision value did not produce conditional dependencies: %+v", missingResult)
}
content := compileRequirementTemplate(t, "content", `{{if input "selector"}}{{input "yes"}}{{else}}{{input "no"}}{{end}}`)
contentResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Templates: CompiledTemplates{Messages: []*template.Template{content}},
}, nil, false)
if err != nil {
t.Fatal(err)
}
contentInputs := make(map[string]domain.InputRequirement, len(contentResult.Inputs))
for _, input := range contentResult.Inputs {
contentInputs[input.Name] = input
}
if contentResult.Complete || len(contentResult.Issues) != 1 || contentResult.Issues[0].Code != domain.RequirementIssueInputContent ||
contentResult.Issues[0].InputName != "selector" || contentInputs["selector"].Kind != domain.RequirementRequired ||
contentInputs["yes"].Kind != domain.RequirementConditional || contentInputs["no"].Kind != domain.RequirementConditional {
t.Fatalf("content-dependent condition did not preserve known dependencies: %+v", contentResult)
}
textOnly := compileRequirementTemplate(t, "text-only", `{{if .missing}}plain text{{else}}also plain text{{end}}`)
textResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Templates: CompiledTemplates{Messages: []*template.Template{textOnly}},
}, nil, false)
if err != nil || !textResult.Complete || len(textResult.Issues) != 0 {
t.Fatalf("unresolved text-only branch should not affect completeness: %+v, %v", textResult, err)
}
}
func TestAnalyzeRequirementsRetainsInactiveEntriesAndPromotesActiveUses(t *testing.T) {
description := "declared for a reason"
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
Role: "user", Content: `{{if false}}{{input "shared"}} {{.shared_var}} {{input "declared"}}{{input .dynamic}}{{and .left .right}}{{end}}{{input "shared"}} {{.shared_var}}`,
}}}
snapshot := RequirementSnapshot{
Inputs: []InputMetadata{{Name: "declared", Required: true, Description: &description}},
Templates: CompiledTemplates{Messages: []*template.Template{
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
}},
}
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, map[string]string{"shared_var": "present"}, false)
if err != nil {
t.Fatal(err)
}
if !result.Complete || len(result.Issues) != 0 {
t.Fatalf("skipped unsupported expressions introduced uncertainty: %+v", result)
}
inputs := make(map[string]domain.InputRequirement, len(result.Inputs))
for _, input := range result.Inputs {
inputs[input.Name] = input
}
if len(inputs) != 2 || inputs["shared"].Kind != domain.RequirementRequired || inputs["shared"].Explanation != "" ||
inputs["declared"].Kind != domain.RequirementRequired || inputs["declared"].Description == nil || *inputs["declared"].Description != description {
t.Fatalf("active or declared requirements did not override inactivity: %+v", result.Inputs)
}
variables := make(map[string]domain.VariableRequirement, len(result.Variables))
for _, variable := range result.Variables {
variables[variable.Name] = variable
}
for _, name := range []string{"dynamic", "left", "right", "shared_var"} {
if _, ok := variables[name]; !ok {
t.Errorf("inactive caller key %q was not inventoried", name)
}
}
if variables["shared_var"].Kind != domain.RequirementRequired || variables["shared_var"].Explanation != "" ||
variables["dynamic"].Kind != domain.RequirementOptional || variables["dynamic"].Explanation == "" {
t.Fatalf("inactive caller metadata was not promoted or explained: %+v", result.Variables)
}
allOptional := compileRequirementTemplate(t, "optional", `{{index . "lookup"}}`)
optionalResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Inputs: []InputMetadata{{Name: "optional"}}, Templates: CompiledTemplates{Messages: []*template.Template{allOptional}},
}, nil, false)
if err != nil || !optionalResult.Complete || len(optionalResult.Issues) != 0 || optionalResult.Inputs[0].Kind != domain.RequirementOptional ||
optionalResult.Variables[0].Kind != domain.RequirementOptional {
t.Fatalf("all-optional inventory should be complete without artifacts: %+v, %v", optionalResult, err)
}
}
func TestAnalyzeRequirementsResolvesSimpleInputNames(t *testing.T) {
tests := []struct {
name string
expression string
vars map[string]string
kind domain.RequirementKind
supplied bool
}{
{name: "direct dot", expression: `{{input .name}}`, vars: map[string]string{"name": " chosen "}, kind: domain.RequirementRequired, supplied: true},
{name: "explicit root", expression: `{{input $.name}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementRequired, supplied: true},
{name: "parenthesized direct", expression: `{{input (.name)}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementRequired, supplied: true},
{name: "literal lookup", expression: `{{input (index . "name")}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementOptional, supplied: true},
{name: "direct pipeline", expression: `{{.name | input}}`, vars: map[string]string{"name": "chosen"}, kind: domain.RequirementRequired, supplied: true},
{name: "empty supplied value", expression: `{{input .name}}`, vars: map[string]string{"name": ""}, kind: domain.RequirementRequired, supplied: true},
{name: "missing lookup has empty zero value", expression: `{{input (index . "name")}}`, vars: nil, kind: domain.RequirementOptional, supplied: false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{Role: "user", Content: test.expression}}}
snapshot := RequirementSnapshot{Templates: CompiledTemplates{Messages: []*template.Template{
compileRequirementTemplate(t, "message", test.expression),
}}}
result, err := AnalyzeRequirements(context.Background(), definition, snapshot, test.vars, false)
if err != nil {
t.Fatal(err)
}
inputName := "chosen"
if test.name == "direct dot" {
inputName = " chosen "
}
if test.name == "empty supplied value" || test.name == "missing lookup has empty zero value" {
inputName = ""
}
if !result.Complete || len(result.Issues) != 0 || len(result.Inputs) != 1 || result.Inputs[0].Name != inputName ||
result.Inputs[0].Kind != domain.RequirementRequired {
t.Fatalf("simple input name was not resolved exactly: %+v", result)
}
if len(result.Variables) != 1 || result.Variables[0].Name != "name" || result.Variables[0].Kind != test.kind || result.Variables[0].Supplied != test.supplied {
t.Fatalf("name-source caller key classification is incorrect: %+v", result.Variables)
}
artifacts := map[string]*domain.Artifact{inputName: {Name: inputName, Body: []byte("resolved")}}
rendered, err := NewGoRenderer().Render(context.Background(), definition, artifacts, test.vars)
if err != nil {
t.Fatal(err)
}
if len(rendered.Messages) != 1 || rendered.Messages[0].Content != "resolved" {
t.Fatalf("analyzed input name differs from renderer: %+v", rendered)
}
})
}
}
func TestAnalyzeRequirementsReportsUnresolvedInputNamesAndPromotesUnknownEffects(t *testing.T) {
definition := &domain.PromptDefinition{Templates: []domain.PromptMessageTemplate{{
Role: "user", Content: `{{input .missing}}`,
}}}
result, err := AnalyzeRequirements(context.Background(), definition, RequirementSnapshot{
Inputs: []InputMetadata{{Name: "optional"}, {Name: "required", Required: true}},
Templates: CompiledTemplates{Messages: []*template.Template{
compileRequirementTemplate(t, "message", definition.Templates[0].Content),
}},
}, nil, false)
if err != nil {
t.Fatal(err)
}
if result.Complete || len(result.Issues) != 2 || result.Issues[0].Code != domain.RequirementIssueDynamicInputName ||
result.Issues[1].Code != domain.RequirementIssueMissingVariable {
t.Fatalf("missing dynamic name did not report its limitations: %+v", result)
}
if result.Inputs[0].Name != "optional" || result.Inputs[0].Kind != domain.RequirementConditional || result.Inputs[0].Explanation == "" ||
result.Inputs[1].Name != "required" || result.Inputs[1].Kind != domain.RequirementRequired || len(result.Variables) != 1 ||
result.Variables[0].Name != "missing" || result.Variables[0].Kind != domain.RequirementRequired {
t.Fatalf("unknown name effects lost known classifications: %+v", result)
}
computed := compileRequirementTemplate(t, "computed", `{{input (printf "%s" .prefix)}}`)
computedResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Inputs: []InputMetadata{{Name: "declared"}}, Templates: CompiledTemplates{Messages: []*template.Template{computed}},
}, map[string]string{"prefix": "SENSITIVE-VALUE"}, false)
if err != nil {
t.Fatal(err)
}
if computedResult.Complete || len(computedResult.Inputs) != 1 || computedResult.Inputs[0].Name != "declared" ||
computedResult.Inputs[0].Kind != domain.RequirementConditional || len(computedResult.Variables) != 1 ||
computedResult.Variables[0].Name != "prefix" || computedResult.Variables[0].Kind != domain.RequirementConditional {
t.Fatalf("computed-name uncertainty did not affect optional entries: %+v", computedResult)
}
for _, issue := range computedResult.Issues {
if strings.Contains(issue.Description, "SENSITIVE-VALUE") || issue.InputName == "SENSITIVE-VALUE" || issue.VariableName == "SENSITIVE-VALUE" {
t.Fatalf("input-name source value leaked into diagnostics: %+v", issue)
}
}
for _, input := range computedResult.Inputs {
if strings.Contains(input.Explanation, "SENSITIVE-VALUE") {
t.Fatalf("input-name source value leaked into an explanation: %+v", input)
}
}
for _, variable := range computedResult.Variables {
if strings.Contains(variable.Explanation, "SENSITIVE-VALUE") {
t.Fatalf("input-name source value leaked into an explanation: %+v", variable)
}
}
for _, expression := range []string{`{{input (input "selector")}}`, `{{input "selector" | input}}`} {
t.Run(expression, func(t *testing.T) {
contentName := compileRequirementTemplate(t, "content-name", expression)
contentResult, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Templates: CompiledTemplates{Messages: []*template.Template{contentName}},
}, nil, false)
if err != nil {
t.Fatal(err)
}
if contentResult.Complete || len(contentResult.Issues) != 2 ||
contentResult.Issues[0].Code != domain.RequirementIssueDynamicInputName ||
contentResult.Issues[1].Code != domain.RequirementIssueInputContent || contentResult.Issues[1].InputName != "selector" || len(contentResult.Inputs) != 1 ||
contentResult.Inputs[0].Name != "selector" || contentResult.Inputs[0].Kind != domain.RequirementRequired {
t.Fatalf("input-body-derived name did not preserve the known source dependency: %+v", contentResult)
}
})
}
}
func TestAnalyzeRequirementsDoesNotResolveSkippedComputedInputNames(t *testing.T) {
compiled := compileRequirementTemplate(t, "inactive-name", `{{if false}}{{input .name}}{{input (printf "%s" .other)}}{{"piped-literal" | input}}{{template "missing" .arg}}{{end}}`)
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
}, nil, false)
if err != nil {
t.Fatal(err)
}
if !result.Complete || len(result.Issues) != 0 || len(result.Inputs) != 1 || result.Inputs[0].Name != "piped-literal" ||
result.Inputs[0].Kind != domain.RequirementOptional || result.Inputs[0].Explanation == "" || len(result.Variables) != 3 {
t.Fatalf("skipped computed expressions introduced uncertainty or names: %+v", result)
}
for _, variable := range result.Variables {
if variable.Kind != domain.RequirementOptional || variable.Explanation == "" {
t.Errorf("skipped variable %q was not retained as inactive optional: %+v", variable.Name, variable)
}
}
}
func TestAnalyzeRequirementsPromotesOptionalVariablesWhenRootScopeIsUnknown(t *testing.T) {
compiled := compileRequirementTemplate(t, "scope", `{{index . "optional"}} {{.required}} {{template "called" .}}{{define "called"}}{{$.hidden}}{{end}}`)
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
}, map[string]string{"required": "value"}, false)
if err != nil {
t.Fatal(err)
}
if result.Complete || len(result.Variables) != 2 || result.Variables[0].Name != "optional" ||
result.Variables[0].Kind != domain.RequirementConditional || result.Variables[0].Explanation == "" ||
result.Variables[1].Name != "required" || result.Variables[1].Kind != domain.RequirementRequired {
t.Fatalf("unknown caller scope did not affect only optional known entries: %+v", result)
}
}
func TestAnalyzeRequirementsLeavesAssignmentDerivedNamesPartial(t *testing.T) {
compiled := compileRequirementTemplate(t, "assignment", `{{$alias := .source}}{{input $alias}}`)
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Inputs: []InputMetadata{{Name: "known-optional"}}, Templates: CompiledTemplates{Messages: []*template.Template{compiled}},
}, nil, false)
if err != nil {
t.Fatal(err)
}
if result.Complete || len(result.Issues) == 0 || len(result.Inputs) != 1 ||
result.Inputs[0].Name != "known-optional" || result.Inputs[0].Kind != domain.RequirementConditional ||
len(result.Variables) != 1 || result.Variables[0].Name != "source" || result.Variables[0].Kind != domain.RequirementConditional {
t.Fatalf("assignment-derived input name was interpreted or lost: %+v", result)
}
}
func TestAnalyzeRequirementsSupportsEmptyTemplatesAndCancellation(t *testing.T) {
result, err := AnalyzeRequirements(context.Background(), &domain.PromptDefinition{}, RequirementSnapshot{
Templates: CompiledTemplates{Messages: []*template.Template{compileRequirementTemplate(t, "message", "plain text")}},
}, nil, false)
if err != nil || !result.Complete || len(result.Inputs) != 0 || len(result.Variables) != 0 {
t.Fatalf("input-free template should have a complete empty inventory: %+v, %v", result, err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := AnalyzeRequirements(ctx, &domain.PromptDefinition{}, RequirementSnapshot{}, nil, false); !errors.Is(err, context.Canceled) {
t.Fatalf("canceled context returned %v, want context.Canceled", err)
}
}
func compileRequirementTemplate(t *testing.T, name, source string) *template.Template {
t.Helper()
compiled, err := CompileTemplate(name, source)
if err != nil {
t.Fatalf("compile template: %v", err)
}
return compiled
}
func requirementInputNames(inputs []domain.InputRequirement) []string {
names := make([]string, len(inputs))
for index, input := range inputs {
names[index] = input.Name
}
return names
}
func requirementVariableNames(variables []domain.VariableRequirement) []string {
names := make([]string, len(variables))
for index, variable := range variables {
names[index] = variable.Name
}
return names
}
func stringPointerForTest(value string) *string {
return &value
}

View File

@@ -9,6 +9,22 @@ type CompiledTemplates struct {
Messages []*template.Template
}
// InputMetadata is the captured declaration metadata for one prompt input.
// Nil metadata pointers mean the corresponding field was absent or null.
type InputMetadata struct {
Name string
Required bool
ContentType *string
Description *string
}
// RequirementSnapshot provides the captured syntax and declarations needed
// for requirement inspection without reopening prompt resources.
type RequirementSnapshot struct {
Templates CompiledTemplates
Inputs []InputMetadata
}
// TemplateLookup provides parsed templates for a captured prompt identity.
type TemplateLookup interface {
GetCompiledTemplates(id, version string) (CompiledTemplates, bool)

View File

@@ -50,10 +50,12 @@ func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*
return nil, err
}
executionSnapshot, err := clonePreparedRun(operation.run)
executionSnapshot := operation.run
extraParams, err := jsonvalue.CopyMap(executionSnapshot.EffectiveModelParams.ExtraParams)
if err != nil {
return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err)
}
executionSnapshot.EffectiveModelParams.ExtraParams = extraParams
details, err := clonePreparedRun(executionSnapshot)
if err != nil {

View File

@@ -631,6 +631,124 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
}
}
func TestRunnerPreparedExecutionDetachesTransferredSnapshotSources(t *testing.T) {
newExtras := func(value string) map[string]any {
return map[string]any{"provider_options": map[string]any{
"provider": map[string]any{"settings": map[string]any{"labels": []any{value}}},
}}
}
extraLabel := func(extraParams map[string]any) string {
return extraParams["provider_options"].(map[string]any)["provider"].(map[string]any)["settings"].(map[string]any)["labels"].([]any)[0].(string)
}
mutateExtra := func(extraParams map[string]any, value string) {
extraParams["provider_options"].(map[string]any)["provider"].(map[string]any)["settings"].(map[string]any)["labels"].([]any)[0] = value
}
for _, test := range []struct {
name string
requestOverride bool
}{
{name: "profile target"},
{name: "request override", requestOverride: true},
} {
t.Run(test.name, func(t *testing.T) {
profileExtras := newExtras("profile source")
requestExtras := newExtras("request source")
profile := defaultExecutionProfile()
profile.ResolvedTarget.ExtraParams = profileExtras
schemaDocument := map[string]any{
"type": "object",
"properties": map[string]any{"value": map[string]any{"type": "string"}},
}
definition := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
definition.Validation.SchemaPath = "schema.json"
control := &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}
rendered := &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{
Role: domain.RoleUser, Content: "captured rendering", CacheControl: control,
}}}
renderer := &fakeRenderer{rendered: rendered}
sourceArtifact := &domain.Artifact{Body: []byte("captured input"), Hash: "captured-input-hash"}
reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"source": sourceArtifact}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"value":"ok"}`}}
validator := &recordingValidationPreparer{
plan: &recordingPreparedValidation{schemaDocument: schemaDocument},
}
runner := NewRunner(
&fakePromptRepo{def: definition},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
reader, renderer, llmClient, validator, nil,
)
request := domain.RunRequest{
PromptID: "p", PromptVersion: "1", ProfileID: "exec",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "source"},
},
}
wantLabel := "profile source"
if test.requestOverride {
request.Execution = &domain.ExecutionTargetOverride{ExtraParams: requestExtras}
wantLabel = "request source"
}
prepared, err := runner.PrepareExecution(context.Background(), request)
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
mutateExtra(profileExtras, "mutated profile source")
mutateExtra(requestExtras, "mutated request source")
rendered.Messages[0].Content = "mutated renderer source"
control.TTL = "mutated renderer cache metadata"
schemaDocument["properties"].(map[string]any)["value"].(map[string]any)["type"] = "integer"
sourceArtifact.Hash = "mutated source hash"
details := prepared.Details()
if details == nil || len(details.Messages) != 1 || details.Messages[0].Content != "captured rendering" ||
details.Messages[0].CacheControl == nil || details.Messages[0].CacheControl.TTL != "1h" ||
details.InputHashes["transcript"] != "captured-input-hash" ||
extraLabel(details.EffectiveModelParams.ExtraParams) != wantLabel ||
details.StructuredOutput == nil || details.StructuredOutput.JSONSchema == nil ||
details.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" {
t.Fatalf("prepared details retained a source alias: %#v", details)
}
result, err := runner.RunPrepared(context.Background(), prepared)
if err != nil || result == nil || !result.Validation.IsValid || result.InputHashes["transcript"] != "captured-input-hash" {
t.Fatalf("run prepared snapshot=(%+v, %v)", result, err)
}
if len(llmClient.lastReq.Prompt.Messages) != 1 || llmClient.lastReq.StructuredOutput == nil ||
llmClient.lastReq.StructuredOutput.JSONSchema == nil ||
extraLabel(llmClient.lastReq.Target.ExtraParams) != wantLabel ||
llmClient.lastReq.Prompt.Messages[0].Content != "captured rendering" ||
llmClient.lastReq.Prompt.Messages[0].CacheControl == nil || llmClient.lastReq.Prompt.Messages[0].CacheControl.TTL != "1h" ||
llmClient.lastReq.StructuredOutput.JSONSchema.Schema.(map[string]any)["properties"].(map[string]any)["value"].(map[string]any)["type"] != "string" {
t.Fatalf("execution snapshot retained a source alias: %#v", llmClient.lastReq)
}
if completedDetails := prepared.Details(); completedDetails == nil ||
len(completedDetails.Messages) != 1 || completedDetails.Messages[0].Content != "captured rendering" ||
extraLabel(completedDetails.EffectiveModelParams.ExtraParams) != wantLabel {
t.Fatalf("completed details changed after source mutation: %#v", completedDetails)
}
})
}
}
func TestRunnerPrepareExecutionClassifiesExtraCopyFailureAsInvalidRequest(t *testing.T) {
profile := defaultExecutionProfile()
profile.ResolvedTarget.ExtraParams = map[string]any{"unsupported": func() {}}
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil,
)
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
PromptID: "p", ProfileID: "exec",
})
if prepared != nil || !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("prepared execution copy failure=(%v, %v), want ErrInvalidRequest", prepared, err)
}
}
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},

View File

@@ -0,0 +1,237 @@
package usecase
import (
"context"
"fmt"
"reflect"
"runtime"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
)
var preparedSnapshotHandleSink *PreparedExecution
const preparedSnapshotRetainedBatchSize = 4
type preparedSnapshotBenchmarkClient struct{}
func (preparedSnapshotBenchmarkClient) Generate(
context.Context,
domain.GenerateRequest,
) (*domain.GenerateResponse, error) {
return &domain.GenerateResponse{Content: "ok"}, nil
}
type preparedSnapshotBenchmarkValidation struct{ schema any }
func (validation *preparedSnapshotBenchmarkValidation) Validate(
context.Context,
*domain.Artifact,
) (domain.ValidationResult, error) {
return domain.ValidationResult{Status: domain.ValidationPassed, IsValid: true}, nil
}
func (validation *preparedSnapshotBenchmarkValidation) SchemaDocument() any {
return validation.schema
}
type preparedSnapshotBenchmarkValidator struct{ schema any }
func (validator preparedSnapshotBenchmarkValidator) Validate(
context.Context,
*domain.Artifact,
domain.OutputContract,
) (domain.ValidationResult, error) {
return domain.ValidationResult{Status: domain.ValidationPassed, IsValid: true}, nil
}
func (validator preparedSnapshotBenchmarkValidator) PrepareValidation(
context.Context,
domain.OutputContract,
) (validate.PreparedValidation, error) {
return &preparedSnapshotBenchmarkValidation{schema: validator.schema}, nil
}
func preparedSnapshotBenchmarkSchema(properties int) map[string]any {
fields := make(map[string]any, properties)
for index := range properties {
fields[fmt.Sprintf("property-%03d", index)] = map[string]any{"type": "string"}
}
return map[string]any{
"type": "object", "properties": fields, "additionalProperties": false,
}
}
func preparedSnapshotBenchmarkExtras(entries int) map[string]any {
providers := make(map[string]any, entries)
for index := range entries {
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
"enabled": true,
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
"limits": map[string]any{"burst": index + 1, "period": "minute"},
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
}
}
return map[string]any{"provider_options": providers}
}
func newPreparedSnapshotBenchmarkRunner(
mutable bool,
) (*Runner, domain.RunRequest, any) {
messageCount := 1
extraEntries := 0
schemaProperties := 1
if mutable {
messageCount = 8
extraEntries = 64
schemaProperties = 256
}
messages := make([]domain.RenderedMessage, messageCount)
for index := range messages {
messages[index] = domain.RenderedMessage{
Role: domain.RoleUser, Content: fmt.Sprintf("rendered message %d", index),
CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"},
}
}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{
SessionID: "snapshot-session", Messages: messages,
}}
profile := defaultExecutionProfile()
if extraEntries > 0 {
profile.ResolvedTarget.ExtraParams = preparedSnapshotBenchmarkExtras(extraEntries)
}
profileRepository := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}}
definition := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
definition.Validation.SchemaPath = "snapshot-schema.json"
schema := preparedSnapshotBenchmarkSchema(schemaProperties)
validator := preparedSnapshotBenchmarkValidator{schema: schema}
runner := NewRunner(
&fakePromptRepo{def: definition},
profileRepository,
staticArtifactReader{},
renderer,
preparedSnapshotBenchmarkClient{},
validator,
nil,
)
request := domain.RunRequest{
PromptID: "p", PromptVersion: "1", ProfileID: "exec",
Inputs: singleInputRef(), Vars: map[string]string{"name": "Ada"},
AppendedMessages: []domain.RenderedMessage{{
Role: domain.RoleAssistant, Content: "appended message",
CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"},
}},
}
return runner, request, schema
}
func checkPreparedSnapshotBenchmarkFixture(b *testing.B, runner *Runner, request domain.RunRequest, schema any) {
b.Helper()
prepared, err := runner.PrepareExecution(context.Background(), request)
if err != nil {
b.Fatal(err)
}
details := prepared.Details()
if details == nil || details.PromptID != request.PromptID || len(details.Messages) < 2 ||
details.Messages[0].Content != "rendered message 0" || details.Messages[len(details.Messages)-1].Content != "appended message" ||
details.InputHashes["transcript"] == "" || details.StructuredOutput == nil || details.StructuredOutput.JSONSchema == nil {
prepared.Discard()
b.Fatalf("prepared snapshot fixture is incomplete: %#v", details)
}
if got := details.StructuredOutput.JSONSchema.Schema; !reflect.DeepEqual(got, schema) {
prepared.Discard()
b.Fatalf("prepared schema differs from fixture: %#v", got)
}
result, err := runner.RunPrepared(context.Background(), prepared)
if err != nil || result == nil || result.RawOutput != "ok" || !result.Validation.IsValid {
b.Fatalf("completed fixture=(%+v, %v)", result, err)
}
completedDetails := prepared.Details()
if completedDetails == nil || completedDetails.Messages[0].Content != "rendered message 0" ||
!reflect.DeepEqual(completedDetails.StructuredOutput.JSONSchema.Schema, schema) {
b.Fatalf("completed handle did not retain its details: %#v", completedDetails)
}
}
func BenchmarkPreparedSnapshot(b *testing.B) {
for _, fixture := range []struct {
name string
mutable bool
}{
{name: "small"},
{name: "mutable_payload", mutable: true},
} {
b.Run(fixture.name, func(b *testing.B) {
runner, request, schema := newPreparedSnapshotBenchmarkRunner(fixture.mutable)
checkPreparedSnapshotBenchmarkFixture(b, runner, request, schema)
b.ReportAllocs()
b.ResetTimer()
for b.Loop() {
prepared, err := runner.PrepareExecution(context.Background(), request)
if err != nil {
b.Fatal(err)
}
prepared.Discard()
preparedSnapshotHandleSink = prepared
}
b.StopTimer()
preparedSnapshotHandleSink = nil
})
}
}
func BenchmarkRetainedPreparedHandles(b *testing.B) {
for _, state := range []string{"ready", "completed", "discarded"} {
b.Run(state, func(b *testing.B) {
runner, request, schema := newPreparedSnapshotBenchmarkRunner(true)
checkPreparedSnapshotBenchmarkFixture(b, runner, request, schema)
retained := make([]*PreparedExecution, preparedSnapshotRetainedBatchSize)
runtime.GC()
var setupMemory runtime.MemStats
runtime.ReadMemStats(&setupMemory)
runtime.KeepAlive(runner)
runtime.KeepAlive(request)
runtime.KeepAlive(retained)
retainHandle := func() *PreparedExecution {
prepared, err := runner.PrepareExecution(context.Background(), request)
if err != nil {
b.Fatal(err)
}
switch state {
case "completed":
result, err := runner.RunPrepared(context.Background(), prepared)
if err != nil || result == nil || result.RawOutput != "ok" || !result.Validation.IsValid {
b.Fatalf("complete retained handle=(%+v, %v)", result, err)
}
case "discarded":
prepared.Discard()
}
return prepared
}
b.ReportAllocs()
b.ResetTimer()
for iteration := 0; iteration < b.N; iteration++ {
retained[iteration%len(retained)] = retainHandle()
}
b.StopTimer()
for index, prepared := range retained {
if prepared == nil {
retained[index] = retainHandle()
}
}
runtime.GC()
var retainedMemory runtime.MemStats
runtime.ReadMemStats(&retainedMemory)
b.ReportMetric(float64(int64(retainedMemory.HeapAlloc)-int64(setupMemory.HeapAlloc)), "retained-B")
runtime.KeepAlive(runner)
runtime.KeepAlive(request)
runtime.KeepAlive(retained)
})
}
}

View File

@@ -15,16 +15,27 @@ type resolvedProfileSelection struct {
profile *domain.ExecutionProfile
}
type profileIdentityRepository interface {
GetProfileIdentity(context.Context, string) (string, error)
}
func (r *Runner) profileLookupID(profileID string) (string, error) {
if strings.TrimSpace(profileID) == "" {
return "", fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
}
if r == nil || r.profiles == nil {
return "", fmt.Errorf("%w: profile repository is not configured", ErrProfileLoad)
}
return profileID, nil
}
func (r *Runner) resolveProfileSelection(
ctx context.Context,
profileID string,
) (*resolvedProfileSelection, error) {
lookupID := profileID
if strings.TrimSpace(lookupID) == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
}
if r == nil || r.profiles == nil {
return nil, fmt.Errorf("%w: profile repository is not configured", ErrProfileLoad)
lookupID, err := r.profileLookupID(profileID)
if err != nil {
return nil, err
}
selectedProfile, err := r.profiles.GetProfile(ctx, lookupID)
if err != nil {
@@ -45,6 +56,25 @@ func (r *Runner) resolveProfileSelection(
}, nil
}
func (r *Runner) resolveProfileIdentity(ctx context.Context, profileID string) (string, error) {
lookupID, err := r.profileLookupID(profileID)
if err != nil {
return "", err
}
if indexed, ok := r.profiles.(profileIdentityRepository); ok {
identity, err := indexed.GetProfileIdentity(ctx, lookupID)
if err != nil {
return "", fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
return identity, nil
}
selection, err := r.resolveProfileSelection(ctx, lookupID)
if err != nil {
return "", err
}
return selection.id, nil
}
func normalizeResolvedExecutionTarget(target domain.ExecutionTarget) (domain.ExecutionTarget, error) {
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(target.Endpoint)
if err != nil {

View File

@@ -13,6 +13,10 @@ type resolvedPromptDefinition struct {
hash string
}
type promptInspectionRepository interface {
GetPromptInspection(context.Context, string, string) (*domain.PromptInspection, error)
}
func (r *Runner) resolvePromptDefinition(
ctx context.Context,
promptID string,
@@ -51,6 +55,43 @@ func (r *Runner) resolvePromptDefinition(
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
}
func (r *Runner) resolvePromptInspection(
ctx context.Context,
promptID string,
promptVersion string,
) (*domain.PromptInspection, error) {
if strings.TrimSpace(promptID) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
}
if r == nil || r.promptDefs == nil {
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
}
if indexed, ok := r.promptDefs.(promptInspectionRepository); ok {
inspection, err := indexed.GetPromptInspection(ctx, promptID, promptVersion)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
}
if inspection == nil {
return nil, fmt.Errorf("%w: prompt repository returned nil inspection", ErrPromptLoad)
}
return inspection, nil
}
selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion)
if err != nil {
return nil, err
}
definition := selection.definition
return &domain.PromptInspection{
PromptID: definition.ID,
PromptVersion: definition.Version,
PromptHash: selection.hash,
DefaultProfileID: definition.DefaultProfile,
Inputs: append([]domain.PromptInput(nil), definition.Inputs...),
OutputContract: definition.Validation,
}, nil
}
// InspectPrompt resolves one explicit prompt without execution work.
func (r *Runner) InspectPrompt(
ctx context.Context,
@@ -66,19 +107,5 @@ func (r *Runner) InspectPrompt(
default:
}
selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion)
if err != nil {
return nil, err
}
inputs := make([]domain.PromptInput, len(selection.definition.Inputs))
copy(inputs, selection.definition.Inputs)
return &domain.PromptInspection{
PromptID: selection.definition.ID,
PromptVersion: selection.definition.Version,
PromptHash: selection.hash,
DefaultProfileID: selection.definition.DefaultProfile,
Inputs: inputs,
OutputContract: selection.definition.Validation,
}, nil
return r.resolvePromptInspection(ctx, promptID, promptVersion)
}

View File

@@ -18,6 +18,13 @@ type inspectionPromptRepository struct {
version string
}
type promptMetadataRepository struct {
inspection domain.PromptInspection
inspectionErr error
lookupID string
lookupVersion string
}
func (r *inspectionPromptRepository) GetPromptDefinition(
_ context.Context,
id string,
@@ -32,6 +39,22 @@ func (r *inspectionPromptRepository) GetPromptDefinition(
return r.definition, nil
}
func (r *promptMetadataRepository) GetPromptDefinition(context.Context, string, string) (*domain.PromptDefinition, error) {
return nil, errors.New("full prompt definition should not be requested")
}
func (r *promptMetadataRepository) GetPromptInspection(
_ context.Context,
id string,
version string,
) (*domain.PromptInspection, error) {
r.lookupID = id
r.lookupVersion = version
inspection := r.inspection
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
return &inspection, r.inspectionErr
}
func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) {
definition := &domain.PromptDefinition{
ID: "normalized.prompt",
@@ -81,6 +104,42 @@ func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t
}
}
func TestRunnerInspectPromptUsesOwnedMetadataWhenRepositorySupportsIt(t *testing.T) {
repository := &promptMetadataRepository{inspection: domain.PromptInspection{
PromptID: "captured.prompt", PromptVersion: "v2", PromptHash: "cached-prompt-hash",
DefaultProfileID: "partial-profile",
Inputs: []domain.PromptInput{{Name: "document", Required: true, ContentType: "text/plain", Description: "Captured."}},
OutputContract: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSON},
}}
runner := &Runner{promptDefs: repository}
inspection, err := runner.InspectPrompt(context.Background(), " requested-id ", " requested-version ")
if err != nil || repository.lookupID != " requested-id " || repository.lookupVersion != " requested-version " ||
inspection.PromptID != "captured.prompt" || inspection.PromptVersion != "v2" ||
inspection.PromptHash != "cached-prompt-hash" || inspection.DefaultProfileID != "partial-profile" ||
!reflect.DeepEqual(inspection.Inputs, repository.inspection.Inputs) || inspection.OutputContract != repository.inspection.OutputContract {
t.Fatalf("metadata inspection=(%+v, %v), lookup=(%q,%q)", inspection, err, repository.lookupID, repository.lookupVersion)
}
inspection.Inputs[0].Name = "caller mutation"
inspection.OutputContract.SchemaPath = "caller mutation"
second, err := runner.InspectPrompt(context.Background(), "requested-id", "requested-version")
if err != nil || second.Inputs[0].Name != "document" || second.OutputContract.SchemaPath != "" {
t.Fatalf("metadata inspection was not caller-owned: (%+v, %v)", second, err)
}
}
func TestRunnerInspectPromptUsesCapturedHashWhenMetadataCapabilityIsAbsent(t *testing.T) {
repository := &requirementInspectionPromptRepository{
definition: &domain.PromptDefinition{
ID: "captured.prompt", Version: "v1", Inputs: []domain.PromptInput{{Name: "document"}},
},
}
inspection, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " requested-id ", " requested-version ")
if err != nil || inspection.PromptID != "captured.prompt" || inspection.PromptVersion != "v1" ||
inspection.PromptHash != "captured-hash" || inspection.Inputs[0].Name != "document" {
t.Fatalf("captured-hash fallback inspection=(%+v, %v)", inspection, err)
}
}
func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
t.Run("blank ID", func(t *testing.T) {
repository := &inspectionPromptRepository{}

View File

@@ -0,0 +1,80 @@
package usecase
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
)
// RequirementInspectionRequest selects a captured prompt and supplies the
// context used for dependency classification.
type RequirementInspectionRequest struct {
PromptID string
PromptVersion string
ProfileID string
Vars map[string]string
SessionID string
}
type requirementSnapshotRepository interface {
GetRequirementSnapshot(id, version string) (prompt.RequirementSnapshot, bool)
}
// InspectRequirements discovers prompt inputs and caller variables without
// loading artifacts or performing execution preparation.
func (r *Runner) InspectRequirements(
ctx context.Context,
req RequirementInspectionRequest,
) (*domain.RequirementInspection, error) {
if strings.TrimSpace(req.PromptID) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
}
directSessionID, err := domain.NormalizeSessionID(req.SessionID)
if err != nil {
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
}
if err := ctx.Err(); err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
}
selection, err := r.resolvePromptInspection(ctx, req.PromptID, req.PromptVersion)
if err != nil {
return nil, err
}
if err := ctx.Err(); err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
selectedProfileID := req.ProfileID
if strings.TrimSpace(selectedProfileID) == "" {
selectedProfileID = selection.DefaultProfileID
}
if strings.TrimSpace(selectedProfileID) == "" {
return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired)
}
profileIdentity, err := r.resolveProfileIdentity(ctx, selectedProfileID)
if err != nil {
return nil, err
}
if err := ctx.Err(); err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
}
repository, ok := r.promptDefs.(requirementSnapshotRepository)
if !ok {
return nil, fmt.Errorf("%w: prompt repository does not provide captured requirement snapshots", ErrPromptLoad)
}
snapshot, ok := repository.GetRequirementSnapshot(selection.PromptID, selection.PromptVersion)
if !ok {
return nil, fmt.Errorf("%w: captured requirement snapshot is unavailable", ErrPromptLoad)
}
definition := &domain.PromptDefinition{ID: selection.PromptID, Version: selection.PromptVersion}
inspection, err := prompt.AnalyzeRequirements(ctx, definition, snapshot, req.Vars, directSessionID != "")
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
}
inspection.ProfileID = profileIdentity
return &inspection, nil
}

View File

@@ -0,0 +1,372 @@
package usecase
import (
"context"
"errors"
"testing"
"text/template"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
)
type requirementInspectionPromptRepository struct {
definition *domain.PromptDefinition
snapshot prompt.RequirementSnapshot
lookupErr error
snapshotFound bool
lookupCalls int
snapshotCalls int
lookupID string
lookupVersion string
snapshotID string
snapshotVersion string
}
type requirementPromptMetadataRepository struct {
*requirementInspectionPromptRepository
inspection domain.PromptInspection
lookupID string
lookupVersion string
}
type inspectionProfileIdentityRepository struct {
*inspectionProfileRepository
identity string
identityErr error
identityID string
identityCalls int
}
func (repository *inspectionProfileIdentityRepository) GetProfileIdentity(
_ context.Context,
id string,
) (string, error) {
repository.identityCalls++
repository.identityID = id
return repository.identity, repository.identityErr
}
func (repository *requirementInspectionPromptRepository) GetPromptDefinition(
context.Context,
string,
string,
) (*domain.PromptDefinition, error) {
return repository.definition, repository.lookupErr
}
func (repository *requirementPromptMetadataRepository) GetPromptInspection(
_ context.Context,
id string,
version string,
) (*domain.PromptInspection, error) {
repository.lookupID = id
repository.lookupVersion = version
inspection := repository.inspection
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
return &inspection, nil
}
func (repository *requirementInspectionPromptRepository) GetPromptSnapshot(
_ context.Context,
id string,
version string,
) (*domain.PromptDefinition, string, error) {
repository.lookupCalls++
repository.lookupID = id
repository.lookupVersion = version
return repository.definition, "captured-hash", repository.lookupErr
}
func (repository *requirementInspectionPromptRepository) GetRequirementSnapshot(
id string,
version string,
) (prompt.RequirementSnapshot, bool) {
repository.snapshotCalls++
repository.snapshotID = id
repository.snapshotVersion = version
return repository.snapshot, repository.snapshotFound
}
func TestRunnerInspectRequirementsUsesCapturedSelectionWithoutExecutionWork(t *testing.T) {
message, err := prompt.CompileTemplate("message", `{{input "message-input"}} {{.caller}}`)
if err != nil {
t.Fatal(err)
}
session, err := prompt.CompileTemplate("session", `{{input "session-only"}}`)
if err != nil {
t.Fatal(err)
}
definition := &domain.PromptDefinition{ID: "selected", Version: "v2", DefaultProfile: "default-profile"}
repository := &requirementInspectionPromptRepository{
definition: definition,
snapshotFound: true,
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{
Messages: []*template.Template{message}, Session: session,
}},
}
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "partial-profile", ResolvedTarget: &domain.ExecutionTarget{Endpoint: ""},
}}
runner := &Runner{promptDefs: repository, profiles: profiles}
inspection, err := runner.InspectRequirements(context.Background(), RequirementInspectionRequest{
PromptID: " requested ", PromptVersion: " exact-version ", ProfileID: "partial-profile",
Vars: map[string]string{"caller": ""}, SessionID: " direct-session ",
})
if err != nil {
t.Fatalf("inspect requirements: %v", err)
}
if repository.lookupCalls != 1 || repository.lookupID != " requested " || repository.lookupVersion != " exact-version " ||
repository.snapshotCalls != 1 || repository.snapshotID != definition.ID || repository.snapshotVersion != definition.Version {
t.Fatalf("selection lookup=(%d,%q,%q), snapshot=(%d,%q,%q)", repository.lookupCalls, repository.lookupID,
repository.lookupVersion, repository.snapshotCalls, repository.snapshotID, repository.snapshotVersion)
}
if profiles.calls != 1 || profiles.id != "partial-profile" || inspection.PromptID != definition.ID ||
inspection.PromptVersion != definition.Version || inspection.ProfileID != "partial-profile" || !inspection.Complete ||
len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" || len(inspection.Variables) != 1 ||
inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied {
t.Fatalf("unexpected requirement inspection: %+v", inspection)
}
}
func TestRunnerInspectRequirementsUsesPromptMetadataAndExactRequirementSnapshot(t *testing.T) {
message, err := prompt.CompileTemplate("message", `{{input "message-input"}} {{.caller}}`)
if err != nil {
t.Fatal(err)
}
session, err := prompt.CompileTemplate("session", `{{input "session-only"}}`)
if err != nil {
t.Fatal(err)
}
baseRepository := &requirementInspectionPromptRepository{
snapshotFound: true,
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{
Messages: []*template.Template{message}, Session: session,
}},
}
promptRepository := &requirementPromptMetadataRepository{
requirementInspectionPromptRepository: baseRepository,
inspection: domain.PromptInspection{
PromptID: "captured-id", PromptVersion: "captured-version", PromptHash: "captured-hash",
DefaultProfileID: "default-profile",
},
}
profiles := &inspectionProfileIdentityRepository{
inspectionProfileRepository: &inspectionProfileRepository{},
identity: "default-profile",
}
inspection, err := (&Runner{promptDefs: promptRepository, profiles: profiles}).InspectRequirements(
context.Background(), RequirementInspectionRequest{
PromptID: " requested-id ", PromptVersion: " requested-version ",
Vars: map[string]string{"caller": "Ada"}, SessionID: " direct-session ",
},
)
if err != nil || promptRepository.lookupID != " requested-id " || promptRepository.lookupVersion != " requested-version " ||
baseRepository.snapshotID != "captured-id" || baseRepository.snapshotVersion != "captured-version" ||
profiles.identityID != "default-profile" || inspection.PromptID != "captured-id" ||
inspection.PromptVersion != "captured-version" || inspection.ProfileID != "default-profile" ||
!inspection.Complete || len(inspection.Inputs) != 1 || inspection.Inputs[0].Name != "message-input" ||
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "caller" || !inspection.Variables[0].Supplied {
t.Fatalf("metadata-based requirement inspection=(%+v, %v), prompt lookup=(%q,%q), snapshot=(%q,%q), profile=%q",
inspection, err, promptRepository.lookupID, promptRepository.lookupVersion,
baseRepository.snapshotID, baseRepository.snapshotVersion, profiles.identityID)
}
}
func TestRunnerInspectRequirementsUsesProfileIdentityCapabilityOrFallback(t *testing.T) {
message, err := prompt.CompileTemplate("message", `{{.name}}`)
if err != nil {
t.Fatal(err)
}
definition := &domain.PromptDefinition{ID: "selected", Version: "v1"}
makePromptRepository := func() *requirementInspectionPromptRepository {
return &requirementInspectionPromptRepository{
definition: definition, snapshotFound: true,
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{message}}},
}
}
requestedID := " literal-profile "
t.Run("optional capability", func(t *testing.T) {
profiles := &inspectionProfileIdentityRepository{
inspectionProfileRepository: &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "repository-profile", ResolvedTarget: &domain.ExecutionTarget{},
}},
identity: requestedID,
}
inspection, err := (&Runner{promptDefs: makePromptRepository(), profiles: profiles}).InspectRequirements(
context.Background(), RequirementInspectionRequest{
PromptID: "selected", ProfileID: requestedID, Vars: map[string]string{"name": "Ada"},
},
)
if err != nil || inspection.ProfileID != requestedID || profiles.identityCalls != 1 ||
profiles.identityID != requestedID || profiles.calls != 0 {
t.Fatalf("identity capability inspection=(%+v, %v), capability calls=%d id=%q full lookups=%d",
inspection, err, profiles.identityCalls, profiles.identityID, profiles.calls)
}
})
t.Run("full-profile fallback", func(t *testing.T) {
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "repository-profile", ResolvedTarget: &domain.ExecutionTarget{},
}}
inspection, err := (&Runner{promptDefs: makePromptRepository(), profiles: profiles}).InspectRequirements(
context.Background(), RequirementInspectionRequest{
PromptID: "selected", ProfileID: requestedID, Vars: map[string]string{"name": "Ada"},
},
)
if err != nil || inspection.ProfileID != requestedID || profiles.calls != 1 || profiles.id != requestedID {
t.Fatalf("fallback inspection=(%+v, %v), full lookup=(%d,%q)", inspection, err, profiles.calls, profiles.id)
}
})
t.Run("capability failure", func(t *testing.T) {
lookupErr := errors.New("profile identity lookup failed")
profiles := &inspectionProfileIdentityRepository{
inspectionProfileRepository: &inspectionProfileRepository{},
identityErr: lookupErr,
}
_, err := (&Runner{promptDefs: makePromptRepository(), profiles: profiles}).InspectRequirements(
context.Background(), RequirementInspectionRequest{PromptID: "selected", ProfileID: requestedID},
)
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, lookupErr) || profiles.identityCalls != 1 || profiles.calls != 0 {
t.Fatalf("capability failure=(%v), capability calls=%d full lookups=%d", err, profiles.identityCalls, profiles.calls)
}
})
}
func TestRunnerInspectRequirementsUsesPromptDefaultAndIncludesSessionWhenNotBypassed(t *testing.T) {
message, err := prompt.CompileTemplate("message", `{{input "message-input"}}`)
if err != nil {
t.Fatal(err)
}
session, err := prompt.CompileTemplate("session", `{{input "session-input"}}`)
if err != nil {
t.Fatal(err)
}
definition := &domain.PromptDefinition{ID: "selected", Version: "v1", DefaultProfile: "default-profile"}
repository := &requirementInspectionPromptRepository{
definition: definition, snapshotFound: true,
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{message}, Session: session}},
}
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "default-profile", ResolvedTarget: &domain.ExecutionTarget{},
}}
inspection, err := (&Runner{promptDefs: repository, profiles: profiles}).InspectRequirements(
context.Background(), RequirementInspectionRequest{PromptID: "selected"},
)
if err != nil {
t.Fatalf("inspect requirements: %v", err)
}
if profiles.id != "default-profile" || inspection.ProfileID != "default-profile" || len(inspection.Inputs) != 2 ||
inspection.Inputs[0].Name != "message-input" || inspection.Inputs[1].Name != "session-input" {
t.Fatalf("prompt-default profile or session template was not selected: %+v", inspection)
}
}
func TestRunnerInspectRequirementsReturnsIncompleteDiscoveryAsSuccess(t *testing.T) {
compiled, err := prompt.CompileTemplate("message", `{{if .missing}}{{input "left"}}{{else}}{{input "right"}}{{end}}`)
if err != nil {
t.Fatal(err)
}
repository := &requirementInspectionPromptRepository{
definition: &domain.PromptDefinition{ID: "selected", Version: "v1", DefaultProfile: "profile"},
snapshotFound: true,
snapshot: prompt.RequirementSnapshot{Templates: prompt.CompiledTemplates{Messages: []*template.Template{compiled}}},
}
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile", ResolvedTarget: &domain.ExecutionTarget{}}}
inspection, err := (&Runner{promptDefs: repository, profiles: profiles}).InspectRequirements(
context.Background(), RequirementInspectionRequest{PromptID: "selected"},
)
if err != nil || inspection == nil || inspection.Complete || len(inspection.Issues) != 1 ||
inspection.Issues[0].Code != domain.RequirementIssueMissingVariable {
t.Fatalf("incomplete discovery returned (%+v, %v), want successful partial result", inspection, err)
}
}
func TestRunnerInspectRequirementsClassifiesSelectionAndSnapshotFailures(t *testing.T) {
t.Run("blank prompt", func(t *testing.T) {
repository := &requirementInspectionPromptRepository{}
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: " \t "})
if !errors.Is(err, ErrInvalidRequest) || repository.lookupCalls != 0 {
t.Fatalf("blank prompt error=%v lookups=%d", err, repository.lookupCalls)
}
})
t.Run("invalid session", func(t *testing.T) {
repository := &requirementInspectionPromptRepository{}
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{
PromptID: "selected", SessionID: string([]byte{0xff}),
})
if !errors.Is(err, ErrInvalidRequest) || repository.lookupCalls != 0 {
t.Fatalf("invalid session error=%v lookups=%d", err, repository.lookupCalls)
}
})
t.Run("cancellation before lookup", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
repository := &requirementInspectionPromptRepository{}
_, err := (&Runner{promptDefs: repository}).InspectRequirements(ctx, RequirementInspectionRequest{PromptID: "selected"})
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, context.Canceled) || repository.lookupCalls != 0 {
t.Fatalf("canceled inspection=%v lookups=%d", err, repository.lookupCalls)
}
})
t.Run("no profile selection", func(t *testing.T) {
repository := &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected"}}
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
if !errors.Is(err, ErrInvalidRequest) || !errors.Is(err, ErrProfileRequired) {
t.Fatalf("missing profile error=%v", err)
}
})
t.Run("ambiguous prompt selection", func(t *testing.T) {
ambiguous := errors.New("ambiguous omitted version")
repository := &requirementInspectionPromptRepository{lookupErr: ambiguous}
_, err := (&Runner{promptDefs: repository}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, ambiguous) || repository.lookupVersion != "" {
t.Fatalf("ambiguous selection error=%v version=%q", err, repository.lookupVersion)
}
})
t.Run("captured snapshot unavailable", func(t *testing.T) {
repository := &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}}
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile", ResolvedTarget: &domain.ExecutionTarget{}}}
_, err := (&Runner{promptDefs: repository, profiles: profiles}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
if !errors.Is(err, ErrPromptLoad) || repository.snapshotCalls != 1 {
t.Fatalf("missing snapshot error=%v calls=%d", err, repository.snapshotCalls)
}
})
t.Run("repository lacks snapshot capability", func(t *testing.T) {
_, err := (&Runner{promptDefs: &inspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}},
profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile", ResolvedTarget: &domain.ExecutionTarget{}}},
}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"})
if !errors.Is(err, ErrPromptLoad) {
t.Fatalf("missing capability error=%v", err)
}
})
if _, err := (*Runner)(nil).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrPromptLoad) {
t.Fatalf("nil runner error=%v", err)
}
if _, err := (&Runner{}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrPromptLoad) {
t.Fatalf("nil repository error=%v", err)
}
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected"}}}).InspectRequirements(
context.Background(), RequirementInspectionRequest{PromptID: "selected", ProfileID: "explicit"},
); !errors.Is(err, ErrProfileLoad) {
t.Fatalf("missing profile error=%v", err)
}
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{definition: &domain.PromptDefinition{ID: "selected", DefaultProfile: "profile"}},
profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile"}},
}).InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "selected"}); !errors.Is(err, ErrProfileLoad) {
t.Fatalf("unresolved profile error=%v", err)
}
if _, err := (&Runner{promptDefs: &requirementInspectionPromptRepository{lookupErr: promptdef.ErrPromptDefinitionNotFound}}).InspectRequirements(
context.Background(), RequirementInspectionRequest{PromptID: "missing"},
); !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
t.Fatalf("missing prompt error=%v", err)
}
}

View File

@@ -188,7 +188,7 @@ func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract d
res.IsValid = true
return res, nil
case domain.ValidationBasic:
empty := strings.TrimSpace(string(artifact.Body)) == ""
empty := len(bytes.TrimSpace(artifact.Body)) == 0
if err := ctx.Err(); err != nil {
return domain.ValidationResult{}, err
}

View File

@@ -1,6 +1,7 @@
package validate
import (
"bytes"
"context"
"encoding/json"
"net/url"
@@ -35,32 +36,51 @@ func TestStandardValidatorNoneSkipped(t *testing.T) {
func TestStandardValidatorBasicSuccess(t *testing.T) {
v := NewStandardValidator("")
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte("hello")}, domain.OutputContract{
ValidationMode: domain.ValidationBasic,
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.Status != domain.ValidationPassed || !res.IsValid {
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
for _, tc := range []struct {
name string
body []byte
}{
{name: "text", body: []byte("hello")},
{name: "invalid UTF-8 byte", body: []byte{0xff}},
} {
t.Run(tc.name, func(t *testing.T) {
res, err := v.Validate(context.Background(), &domain.Artifact{Body: tc.body}, domain.OutputContract{
ValidationMode: domain.ValidationBasic,
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.Status != domain.ValidationPassed || !res.IsValid {
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
}
})
}
}
func TestStandardValidatorBasicFailureEmpty(t *testing.T) {
v := NewStandardValidator("")
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(" \n\t ")}, domain.OutputContract{
ValidationMode: domain.ValidationBasic,
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.Status != domain.ValidationFailed || res.IsValid {
t.Fatalf("expected failed/invalid, got status=%q valid=%v", res.Status, res.IsValid)
}
if len(res.Errors) == 0 {
t.Fatal("expected validation errors")
for _, tc := range []struct {
name string
body []byte
}{
{name: "empty bytes", body: []byte{}},
{name: "ASCII whitespace", body: []byte(" \n\t ")},
{name: "Unicode whitespace", body: []byte("\u2003\u00a0\u3000")},
} {
t.Run(tc.name, func(t *testing.T) {
res, err := v.Validate(context.Background(), &domain.Artifact{Body: tc.body}, domain.OutputContract{
ValidationMode: domain.ValidationBasic,
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.Status != domain.ValidationFailed || res.IsValid {
t.Fatalf("expected failed/invalid, got status=%q valid=%v", res.Status, res.IsValid)
}
if len(res.Errors) == 0 {
t.Fatal("expected validation errors")
}
})
}
}
@@ -917,6 +937,49 @@ func BenchmarkJSONValidation(b *testing.B) {
}
}
var basicValidationBenchmarkResult domain.ValidationResult
func BenchmarkBasicValidation(b *testing.B) {
largeText := bytes.Repeat([]byte("x"), 1<<20)
largeWhitespace := bytes.Repeat([]byte(" \t\n"), (1<<20)/3)
unicodeWhitespace := []byte("\u2003\u00a0\u3000")
benchmarks := []struct {
name string
body []byte
wantValid bool
}{
{name: "short_text", body: []byte("A concise response."), wantValid: true},
{name: "1MiB_nonempty_text", body: largeText, wantValid: true},
{name: "whitespace_only_text", body: largeWhitespace},
{name: "unicode_whitespace", body: unicodeWhitespace},
}
validator := NewStandardValidator("")
contract := domain.OutputContract{ValidationMode: domain.ValidationBasic}
ctx := context.Background()
for _, benchmark := range benchmarks {
b.Run(benchmark.name, func(b *testing.B) {
artifact := &domain.Artifact{Body: benchmark.body}
result, err := validator.Validate(ctx, artifact, contract)
if err != nil || result.IsValid != benchmark.wantValid {
b.Fatalf("validate basic output: result=%+v error=%v", result, err)
}
b.ReportAllocs()
// Nonempty input can be classified from its boundary bytes without
// scanning the body, so report latency rather than bytes per second.
b.ResetTimer()
for range b.N {
result, err := validator.Validate(ctx, artifact, contract)
if err != nil || result.IsValid != benchmark.wantValid {
b.Fatalf("validate basic output: result=%+v error=%v", result, err)
}
basicValidationBenchmarkResult = result
}
})
}
}
func TestJSONSchemaDialectIsDraft2020(t *testing.T) {
tests := []struct {
name string

67
listing.go Normal file
View File

@@ -0,0 +1,67 @@
package promptkit
import (
"context"
"fmt"
)
// PromptIdentity identifies one prompt ID and exact version in the effective
// configuration captured when an Engine is constructed. It has no stable JSON
// representation.
type PromptIdentity struct {
ID string
Version string
}
// ListProfiles returns profile IDs in the effective configuration captured
// when the engine was constructed. Maintained and consumer profiles are
// included; an inherited profile retains its own ID. IDs are preserved exactly
// and sorted lexically. The returned slice is caller-owned and safe to mutate
// while the engine is used concurrently. Empty inventories succeed.
//
// Listing reports membership, not execution readiness. It does not reload
// configuration or perform resource, credential, provider, or execution work.
// Profile IDs are application data and may be sensitive. A nil or zero-value
// engine returns an error matching ErrInvalidConfig. Cancellation or deadline
// expiration returns the context error and no partial result.
func (e *Engine) ListProfiles(ctx context.Context) ([]string, error) {
if e == nil || e.internal == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
profiles, err := e.internal.ListProfiles(ctx)
if err != nil {
return nil, mapPublicError(err)
}
return profiles, nil
}
// ListPrompts returns every prompt ID and exact version in the effective
// configuration captured when the engine was constructed, sorted lexically by
// ID and then version. Versions are not interpreted or prioritized. The
// returned slice is caller-owned and safe to mutate while the engine is used
// concurrently. Empty inventories succeed.
//
// Listing reports membership, not execution readiness. It does not reload
// configuration or perform resource, credential, provider, or execution work.
// Prompt identities are application data and may be sensitive. A nil or
// zero-value engine returns an error matching ErrInvalidConfig. Cancellation
// or deadline expiration returns the context error and no partial result.
func (e *Engine) ListPrompts(ctx context.Context) ([]PromptIdentity, error) {
if e == nil || e.internal == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
identities, err := e.internal.ListPrompts(ctx)
if err != nil {
return nil, mapPublicError(err)
}
public := make([]PromptIdentity, len(identities))
for i, identity := range identities {
public[i] = PromptIdentity{ID: identity.ID, Version: identity.Version}
}
if err := ctx.Err(); err != nil {
return nil, mapPublicError(err)
}
return public, nil
}

379
listing_contract_test.go Normal file
View File

@@ -0,0 +1,379 @@
package promptkit_test
import (
"context"
"errors"
"fmt"
"reflect"
"sync"
"testing"
"testing/fstest"
"time"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestListIdentitiesBasicContract(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("listing", map[string]any{
"profiles": map[string]any{"configured-profile": map[string]any{"model": "example-model"}},
"prompts": map[string]any{"example-prompt": map[string]any{
"2": listingContractPrompt(),
"10": listingContractPrompt(),
}},
}),
}}})
if err != nil {
t.Fatal(err)
}
profiles, err := engine.ListProfiles(context.Background())
if err != nil {
t.Fatal(err)
}
if !containsListingProfile(profiles, "configured-profile") {
t.Fatalf("profile listing %#v does not include configured profile", profiles)
}
prompts, err := engine.ListPrompts(context.Background())
if err != nil {
t.Fatal(err)
}
want := []promptkit.PromptIdentity{
{ID: "example-prompt", Version: "10"},
{ID: "example-prompt", Version: "2"},
}
if len(prompts) != len(want) {
t.Fatalf("prompt listing = %#v, want %#v", prompts, want)
}
for i := range want {
if prompts[i] != want[i] {
t.Fatalf("prompt listing = %#v, want %#v", prompts, want)
}
}
profile, err := engine.InspectProfile(context.Background(), "configured-profile")
if err != nil || profile == nil || profile.ProfileID != "configured-profile" {
t.Fatalf("listed profile could not be inspected: (%+v, %v)", profile, err)
}
prompt, err := engine.InspectPrompt(context.Background(), prompts[0].ID, prompts[0].Version)
if err != nil || prompt == nil || prompt.PromptID != prompts[0].ID || prompt.PromptVersion != prompts[0].Version {
t.Fatalf("listed prompt could not be inspected: (%+v, %v)", prompt, err)
}
}
func TestListIdentitiesInvalidEngine(t *testing.T) {
var nilEngine *promptkit.Engine
var zeroEngine promptkit.Engine
for _, engine := range []*promptkit.Engine{nilEngine, &zeroEngine} {
if _, err := engine.ListProfiles(context.Background()); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Errorf("ListProfiles on invalid engine: %v", err)
}
if _, err := engine.ListPrompts(context.Background()); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Errorf("ListPrompts on invalid engine: %v", err)
}
}
}
func TestListIdentitiesPreserveContextErrorsWithoutPartialResults(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("listing", map[string]any{
"profiles": map[string]any{"profile": map[string]any{}},
"prompts": map[string]any{"example": map[string]any{"v1": listingContractPrompt()}},
}),
}}})
if err != nil {
t.Fatal(err)
}
canceled, cancel := context.WithCancel(context.Background())
cancel()
expired, expire := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
defer expire()
for _, test := range []struct {
name string
ctx context.Context
want error
}{
{name: "canceled", ctx: canceled, want: context.Canceled},
{name: "expired", ctx: expired, want: context.DeadlineExceeded},
} {
t.Run(test.name, func(t *testing.T) {
profiles, err := engine.ListProfiles(test.ctx)
if !errors.Is(err, test.want) || len(profiles) != 0 {
t.Errorf("ListProfiles = %#v, %v; want no results and %v", profiles, err, test.want)
}
prompts, err := engine.ListPrompts(test.ctx)
if !errors.Is(err, test.want) || len(prompts) != 0 {
t.Errorf("ListPrompts = %#v, %v; want no results and %v", prompts, err, test.want)
}
})
}
}
func TestListIdentitiesUseEffectiveLayeredMembership(t *testing.T) {
underlay := configkit.Input{Source: configkit.Mapping("consumer defaults", map[string]any{
"profiles": map[string]any{
"base/profile": map[string]any{"endpoint": "https://example.test/v1", "model": "base-model"},
"child/profile": map[string]any{"extends": "base/profile"},
"overlay/profile": map[string]any{"endpoint": "https://example.test/v1", "model": "old-model"},
},
"prompts": map[string]any{
"shared/prompt": map[string]any{
"v1": listingContractPrompt(),
"v2": listingContractPrompt(),
},
"underlay-only": map[string]any{"v1": listingContractPrompt()},
},
})}
main := configkit.Input{Source: configkit.Mapping("application overlay", map[string]any{
"profiles": map[string]any{
"overlay/profile": map[string]any{"model": "effective-model"},
},
"prompts": map[string]any{
"shared/prompt": map[string]any{"v1": listingContractPrompt()},
"main-only": map[string]any{"v3": listingContractPrompt()},
},
})}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{underlay}}},
Main: &main,
}})
if err != nil {
t.Fatal(err)
}
profiles, err := engine.ListProfiles(context.Background())
if err != nil {
t.Fatal(err)
}
for _, id := range []string{
"base/profile", "child/profile", "overlay/profile",
"promptkit-openrouter-standard", "promptkit-openrouter-auto",
} {
if !containsListingProfile(profiles, id) {
t.Errorf("profile listing %#v does not include %q", profiles, id)
}
}
for _, id := range []string{"base/profile", "child/profile", "overlay/profile"} {
if got := countListingProfile(profiles, id); got != 1 {
t.Errorf("profile %q appears %d times, want 1", id, got)
}
}
overlay, err := engine.InspectProfile(context.Background(), "overlay/profile")
if err != nil || overlay == nil || overlay.EffectiveModelParams.Model != "effective-model" {
t.Fatalf("overlay profile did not resolve to its effective definition: (%+v, %v)", overlay, err)
}
child, err := engine.InspectProfile(context.Background(), "child/profile")
if err != nil || child == nil || child.EffectiveModelParams.Model != "base-model" {
t.Fatalf("inherited child profile did not resolve its effective definition: (%+v, %v)", child, err)
}
prompts, err := engine.ListPrompts(context.Background())
if err != nil {
t.Fatal(err)
}
wantPrompts := []promptkit.PromptIdentity{
{ID: "main-only", Version: "v3"},
{ID: "shared/prompt", Version: "v1"},
{ID: "shared/prompt", Version: "v2"},
{ID: "underlay-only", Version: "v1"},
}
if !reflect.DeepEqual(prompts, wantPrompts) {
t.Fatalf("effective prompt identities = %#v, want %#v", prompts, wantPrompts)
}
if got := countListingPrompt(prompts, "shared/prompt", "v1"); got != 1 {
t.Fatalf("overlaid prompt identity appears %d times, want 1", got)
}
}
func TestListIdentitiesEmptyEffectiveConfiguration(t *testing.T) {
main := configkit.Input{Source: configkit.Mapping("clear effective identities", map[string]any{
"profiles": nil,
"prompts": nil,
})}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}})
if err != nil {
t.Fatal(err)
}
profiles, err := engine.ListProfiles(context.Background())
if err != nil || len(profiles) != 0 {
t.Fatalf("cleared profile listing = %#v, %v", profiles, err)
}
prompts, err := engine.ListPrompts(context.Background())
if err != nil || len(prompts) != 0 {
t.Fatalf("cleared prompt listing = %#v, %v", prompts, err)
}
}
func TestListIdentitiesAreOwnedAndSafeForConcurrentUse(t *testing.T) {
engine, err := newListingContractEngine(t, map[string]any{
"profiles": map[string]any{"profile": map[string]any{"model": "partial-model"}},
"prompts": map[string]any{"prompt": map[string]any{"v1": listingContractPrompt()}},
})
if err != nil {
t.Fatal(err)
}
profiles, err := engine.ListProfiles(context.Background())
if err != nil {
t.Fatal(err)
}
prompts, err := engine.ListPrompts(context.Background())
if err != nil {
t.Fatal(err)
}
for i := range profiles {
if profiles[i] == "profile" {
profiles[i] = "caller mutation"
}
}
for i := range prompts {
if prompts[i].ID == "prompt" {
prompts[i].ID = "caller mutation"
prompts[i].Version = "caller mutation"
}
}
profiles, err = engine.ListProfiles(context.Background())
if err != nil || !containsListingProfile(profiles, "profile") {
t.Fatalf("profile result mutation escaped into engine: %#v, %v", profiles, err)
}
prompts, err = engine.ListPrompts(context.Background())
if err != nil || countListingPrompt(prompts, "prompt", "v1") != 1 {
t.Fatalf("prompt result mutation escaped into engine: %#v, %v", prompts, err)
}
const workers = 4
const iterations = 25
start := make(chan struct{})
results := make(chan error, workers)
var wait sync.WaitGroup
wait.Add(workers)
for worker := 0; worker < workers; worker++ {
go func() {
defer wait.Done()
<-start
for iteration := 0; iteration < iterations; iteration++ {
profiles, err := engine.ListProfiles(context.Background())
if err != nil {
results <- fmt.Errorf("list profiles: %w", err)
return
}
if len(profiles) == 0 {
results <- errors.New("profile listing is empty")
return
}
profiles[0] = "worker mutation"
prompts, err := engine.ListPrompts(context.Background())
if err != nil {
results <- fmt.Errorf("list prompts: %w", err)
return
}
if len(prompts) == 0 {
results <- errors.New("prompt listing is empty")
return
}
prompts[0].ID = "worker mutation"
prompts[0].Version = "worker mutation"
if _, err := engine.InspectProfile(context.Background(), "profile"); err != nil {
results <- fmt.Errorf("inspect profile: %w", err)
return
}
if _, err := engine.InspectPrompt(context.Background(), "prompt", "v1"); err != nil {
results <- fmt.Errorf("inspect prompt: %w", err)
return
}
}
}()
}
close(start)
wait.Wait()
close(results)
for err := range results {
t.Error(err)
}
}
func TestListIdentitiesUseCapturedSourcesWithoutExecutionWork(t *testing.T) {
files := fstest.MapFS{
"config/listing.yaml": &fstest.MapFile{Data: []byte("profiles:\n captured-partial:\n model: partial-model\nprompts:\n captured-prompt:\n v1:\n messages:\n - role: user\n content: Hello.\n output:\n format: text\n validation_mode: basic\n")},
}
inputs, err := promptkit.FilesystemInputs("application listing", files, "config")
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: inputs}},
}}, promptkit.WithLLMClient(rejectingListingLLMClient{}), promptkit.WithArtifactReader(rejectingListingArtifactReader{}))
if err != nil {
t.Fatal(err)
}
files["config/listing.yaml"].Data = []byte("profiles:\n replacement-profile: {}\nprompts:\n replacement-prompt:\n v2:\n messages:\n - role: user\n content: Changed.\n output:\n format: text\n validation_mode: basic\n")
profiles, err := engine.ListProfiles(context.Background())
if err != nil || !containsListingProfile(profiles, "captured-partial") || containsListingProfile(profiles, "replacement-profile") {
t.Fatalf("profile listing did not use captured source: %#v, %v", profiles, err)
}
prompts, err := engine.ListPrompts(context.Background())
if err != nil || countListingPrompt(prompts, "captured-prompt", "v1") != 1 || countListingPrompt(prompts, "replacement-prompt", "v2") != 0 {
t.Fatalf("prompt listing did not use captured source: %#v, %v", prompts, err)
}
}
func listingContractPrompt() map[string]any {
return map[string]any{
"messages": []any{map[string]any{"role": "user", "content": "Hello."}},
"output": map[string]any{"format": "text", "validation_mode": "basic"},
}
}
func containsListingProfile(profiles []string, id string) bool {
for _, profile := range profiles {
if profile == id {
return true
}
}
return false
}
func countListingProfile(profiles []string, id string) int {
count := 0
for _, profile := range profiles {
if profile == id {
count++
}
}
return count
}
func countListingPrompt(prompts []promptkit.PromptIdentity, id, version string) int {
count := 0
for _, prompt := range prompts {
if prompt.ID == id && prompt.Version == version {
count++
}
}
return count
}
func newListingContractEngine(t *testing.T, values map[string]any, options ...promptkit.Option) (*promptkit.Engine, error) {
t.Helper()
input := configkit.Input{Source: configkit.Mapping("listing contract", values)}
return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &input}}, options...)
}
type rejectingListingLLMClient struct{}
func (rejectingListingLLMClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
return nil, errors.New("unexpected model generation")
}
type rejectingListingArtifactReader struct{}
func (rejectingListingArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
return nil, errors.New("unexpected artifact read")
}

29
releasekit.json Normal file
View File

@@ -0,0 +1,29 @@
{
"product": "Promptkit",
"remote": "origin",
"validation": {
"go": {
"race": true,
"full_vet": true
},
"go_format": true,
"markdown": {},
"module_distribution": {
"workspace_files": true,
"local_replacements": true
}
},
"checks": [
{
"args": ["python3", "./scripts/check-examples.py"],
"env": {
"GOTOOLCHAIN": "local",
"GOWORK": "off",
"GOFLAGS": "-mod=readonly",
"GOPROXY": "off",
"GONOPROXY": "none",
"GOSUMDB": "off"
}
}
]
}

240
requirements.go Normal file
View File

@@ -0,0 +1,240 @@
package promptkit
import (
"context"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
)
// RequirementInspectionRequest selects a prompt and supplies caller context
// for requirement discovery. It requires no artifact bindings or credentials.
type RequirementInspectionRequest struct {
// PromptID is the exact, case-sensitive configured prompt ID.
PromptID string
// PromptVersion selects an exact version. An empty value uses the prompt
// selection rule that requires exactly one configured version.
PromptVersion string
// ProfileID selects the exact effective profile identity. If empty, the
// prompt's default profile is used; the profile need not be ready for
// execution.
ProfileID string
// Vars supplies string values for caller-variable lookups and simple
// conditions. Presence is determined by exact map key, including an empty
// value. Inspection reports caller keys and presence, not their values.
// A supplied value may nevertheless appear as a resolved input identifier.
Vars map[string]string
// SessionID, when nonblank after the normal session-ID normalization,
// bypasses the prompt's session template during discovery.
SessionID string
}
// RequirementKind describes how a selected prompt may depend on an input or
// caller variable for the supplied context. It is independent of whether a
// caller-variable value is present.
type RequirementKind string
const (
// RequirementRequired identifies an input declared required or a dependency
// reached by a recognized definite access.
RequirementRequired RequirementKind = "required"
// RequirementConditional identifies a dependency whose reachability or
// identity remains unresolved for the supplied context.
RequirementConditional RequirementKind = "conditional"
// RequirementOptional identifies a dependency known not to be required,
// including a nonrequired declaration, optional lookup, or reference used
// only on a known skipped path.
RequirementOptional RequirementKind = "optional"
)
// InputRequirement describes a declared or template-referenced prompt input.
type InputRequirement struct {
// Name is the exact input identifier.
Name string
// Kind is the strongest classification established for this name.
Kind RequirementKind
// Declared reports whether the name appears in the effective input
// declarations; helper-only references have Declared false.
Declared bool
// ContentType preserves effective metadata presence: nil means absent or
// null, while a non-nil pointer may refer to an empty string.
ContentType *string
// Description preserves effective metadata presence independently of
// Explanation, including an explicitly empty string.
Description *string
// Explanation is safe human-readable context, such as why a reference is
// optional because it occurs only on a known skipped path. It is not a
// machine-readable contract.
Explanation string
}
// VariableRequirement describes a caller-map key referenced by a prompt.
type VariableRequirement struct {
// Name is the exact caller-map key, not a template-local name.
Name string
// Kind is the strongest classification established for this key.
Kind RequirementKind
// Supplied reports exact-key presence in the request Vars map. An empty
// string is supplied; the value itself is not returned.
Supplied bool
// Explanation is safe human-readable context and is not a machine-readable
// contract.
Explanation string
}
// RequirementIssueCode identifies a limitation that can affect discovery.
type RequirementIssueCode string
const (
// RequirementIssueMissingVariable means an absent caller value prevents
// resolving a dependency decision or input name.
RequirementIssueMissingVariable RequirementIssueCode = "missing_variable"
// RequirementIssueInputContent means unavailable input content affects
// dependency discovery.
RequirementIssueInputContent RequirementIssueCode = "input_content"
// RequirementIssueDynamicInputName means a referenced input name cannot be
// determined for the supplied context.
RequirementIssueDynamicInputName RequirementIssueCode = "dynamic_input_name"
// RequirementIssueUnsupportedConstruct means a dependency-relevant
// expression or scope is outside the supported discovery analysis.
RequirementIssueUnsupportedConstruct RequirementIssueCode = "unsupported_construct"
)
// RequirementIssue describes one discovery limitation using safe summary
// text. Descriptions are not intended as machine-readable details.
type RequirementIssue struct {
// Code is the machine-readable limitation category.
Code RequirementIssueCode
// InputName identifies an affected input when one is known; empty means no
// particular input was identified.
InputName string
// VariableName identifies an affected caller key when one is known; empty
// means no particular key was identified.
VariableName string
// Description is a safe human-readable summary, not a string to parse.
Description string
}
// RequirementInspection contains discovered dependencies for one selected
// prompt context. Input and variable entries are sorted by exact name. Issues
// are deduplicated by code and associated names, then sorted by that tuple.
// Their descriptions and entry explanations are safe summaries, not
// machine-readable details.
//
// Complete describes the supported analysis's discovery coverage, not whether
// requirements are supplied, templates will render, artifacts or schemas are
// valid, or execution is ready. A complete result can still contain required
// inputs or missing required variables. Valid templates using dependency-
// relevant constructs outside the deliberately small recognition scope may
// produce useful partial inventories with Complete false; this is not a
// template-definition error or an authoring prohibition.
type RequirementInspection struct {
// PromptID is the selected exact prompt ID.
PromptID string
// PromptVersion is the selected exact prompt version.
PromptVersion string
// ProfileID is the selected profile identity; it does not indicate
// execution readiness.
ProfileID string
// Inputs contains declared inputs and recognizable template references,
// including entries known to be optional.
Inputs []InputRequirement
// Variables contains recognizable caller-map keys, including keys used
// only on known skipped paths.
Variables []VariableRequirement
// Complete reports discovery coverage, not requirement satisfaction or
// execution readiness.
Complete bool
// Issues contains deduplicated, structured discovery limitations.
Issues []RequirementIssue
}
// InspectRequirements discovers inputs and caller variables for one selected
// prompt before its input artifacts exist. It uses exact, case-sensitive
// prompt selection and requires a profile ID either in the request or the
// selected prompt's default. The selected profile establishes identity only;
// it need not be execution-ready. A normalized nonblank SessionID bypasses the
// captured session template. Vars are copied before analysis, and returned
// slices and metadata pointers belong to the caller.
//
// Discovery recognizes literal input names, direct caller-map keys, literal-
// key root-map lookups, simple caller-derived input names, and if/with
// conditions based on direct values, literals, negation, or two-string
// equality/inequality. It classifies recognizable references on known skipped
// paths as optional. Other dependency-relevant expressions or scopes may leave
// a useful partial result with Complete false. The returned inventories do
// not reject otherwise valid templates or prohibit their use.
//
// Results exclude artifact payloads, prompt contents, and unrelated supplied
// values. Resolved input identifiers may equal the caller values that supply
// them; those identifiers are intentionally included in the inventory.
//
// This method does not perform artifact, schema, credential, rendering,
// validation, capacity, observation, provider, or execution work. Complete
// describes dependency discovery, not whether every dependency is supplied or
// the prompt is ready to execute. Discovery limitations return a partial
// inspection and no error; selection and operational failures return no
// inspection. Blank prompt IDs, invalid session IDs, and absent profile
// selection match ErrInvalidRequest; an absent profile selection also matches
// ErrProfileRequired. Missing prompt/profile IDs match ErrPromptNotFound or
// ErrProfileNotFound. Other selection/snapshot failures match ErrPromptLoad or
// ErrProfileLoad. When cancellation prevents lookup or analysis, no inspection
// is returned and the context error remains discoverable with errors.Is. A nil
// Engine matches ErrInvalidConfig.
func (e *Engine) InspectRequirements(
ctx context.Context,
req RequirementInspectionRequest,
) (*RequirementInspection, error) {
if e == nil || e.internal == nil {
return nil, ErrInvalidConfig
}
inspection, err := e.internal.InspectRequirements(ctx, usecase.RequirementInspectionRequest{
PromptID: req.PromptID, PromptVersion: req.PromptVersion, ProfileID: req.ProfileID,
Vars: copyStringMap(req.Vars), SessionID: req.SessionID,
})
if err != nil {
return nil, mapPublicError(err)
}
return fromDomainRequirementInspection(inspection), nil
}
func fromDomainRequirementInspection(inspection *domain.RequirementInspection) *RequirementInspection {
if inspection == nil {
return nil
}
result := &RequirementInspection{
PromptID: inspection.PromptID, PromptVersion: inspection.PromptVersion,
ProfileID: inspection.ProfileID, Complete: inspection.Complete,
Inputs: make([]InputRequirement, len(inspection.Inputs)),
Variables: make([]VariableRequirement, len(inspection.Variables)),
Issues: make([]RequirementIssue, len(inspection.Issues)),
}
for index, input := range inspection.Inputs {
result.Inputs[index] = InputRequirement{
Name: input.Name, Kind: RequirementKind(input.Kind), Declared: input.Declared,
ContentType: copyRequirementString(input.ContentType), Description: copyRequirementString(input.Description),
Explanation: input.Explanation,
}
}
for index, variable := range inspection.Variables {
result.Variables[index] = VariableRequirement{
Name: variable.Name, Kind: RequirementKind(variable.Kind), Supplied: variable.Supplied,
Explanation: variable.Explanation,
}
}
for index, issue := range inspection.Issues {
result.Issues[index] = RequirementIssue{
Code: RequirementIssueCode(issue.Code), InputName: issue.InputName,
VariableName: issue.VariableName, Description: issue.Description,
}
}
return result
}
func copyRequirementString(value *string) *string {
if value == nil {
return nil
}
copy := *value
return &copy
}

View File

@@ -0,0 +1,72 @@
package promptkit_test
import (
"context"
"fmt"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
var inspectedRequirements *promptkit.RequirementInspection
func requirementInspectionExtras(entries int) map[string]any {
providers := make(map[string]any, entries)
for index := range entries {
providers[fmt.Sprintf("provider-%03d", index)] = map[string]any{
"enabled": true,
"labels": []any{fmt.Sprintf("tier-%d", index%4), fmt.Sprintf("region-%d", index%3)},
"limits": map[string]any{"burst": index + 1, "period": "minute"},
"routes": []any{map[string]any{"weight": index + 1, "fallback": true}},
}
}
return map[string]any{"provider_options": providers}
}
func BenchmarkRequirementInspection(b *testing.B) {
profiles := map[string]any{
"small": map[string]any{
"model": "inspection-model", "extra_params": requirementInspectionExtras(2),
},
"large": map[string]any{
"model": "inspection-model", "extra_params": requirementInspectionExtras(64),
},
}
definition := contractPrompt("small", "Hello {{.name}} {{input \"transcript\"}}", []any{
map[string]any{"name": "transcript", "required": true},
}, "", noValidationOutput)
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("requirement inspection benchmark", map[string]any{
"prompts": oneVersionPrompt("inspection", "1", definition),
"profiles": profiles,
}),
}}})
if err != nil {
b.Fatal(err)
}
ctx := context.Background()
for _, profileID := range []string{"small", "large"} {
request := promptkit.RequirementInspectionRequest{
PromptID: "inspection", ProfileID: profileID, Vars: map[string]string{"name": "Ada"},
}
inspection, err := engine.InspectRequirements(ctx, request)
if err != nil || inspection.PromptID != "inspection" || inspection.PromptVersion != "1" ||
inspection.ProfileID != profileID || !inspection.Complete || len(inspection.Inputs) != 1 ||
inspection.Inputs[0].Name != "transcript" || inspection.Inputs[0].Kind != promptkit.RequirementRequired ||
len(inspection.Variables) != 1 || inspection.Variables[0].Name != "name" || !inspection.Variables[0].Supplied {
b.Fatalf("requirement inspection fixture for %q is invalid: inspection=%+v err=%v", profileID, inspection, err)
}
selectedRequest := request
b.Run(profileID+"_extra", func(b *testing.B) {
b.ReportAllocs()
for b.Loop() {
inspection, err := engine.InspectRequirements(ctx, selectedRequest)
if err != nil {
b.Fatal(err)
}
inspectedRequirements = inspection
}
})
}
}

View File

@@ -0,0 +1,370 @@
package promptkit_test
import (
"context"
"errors"
"fmt"
"strings"
"sync"
"sync/atomic"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
func requirementsEngine(t *testing.T, prompts, profiles map[string]any, options ...promptkit.Option) *promptkit.Engine {
t.Helper()
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("requirements contract", map[string]any{
"prompts": prompts, "profiles": profiles,
}),
}}}, options...)
if err != nil {
t.Fatalf("construct requirements engine: %v", err)
}
return engine
}
type requirementCountingArtifactReader struct{ calls atomic.Int64 }
func (reader *requirementCountingArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
reader.calls.Add(1)
return nil, errors.New("artifact reader must not be called")
}
func contractPrompt(defaultProfile string, content string, inputs []any, session string, output map[string]any) map[string]any {
definition := map[string]any{
"messages": []any{map[string]any{"role": "user", "content": content}},
"output": output,
}
if defaultProfile != "" {
definition["default_profile"] = defaultProfile
}
if inputs != nil {
definition["inputs"] = inputs
}
if session != "" {
definition["session_id"] = session
}
return definition
}
func oneVersionPrompt(id, version string, definition map[string]any) map[string]any {
return map[string]any{id: map[string]any{version: definition}}
}
var noValidationOutput = map[string]any{"format": "text", "validation_mode": "none"}
func TestInspectRequirementsEnumeratesCapturedDependenciesAndInactiveNames(t *testing.T) {
var modelCalls atomic.Int64
reader := &requirementCountingArtifactReader{}
definition := contractPrompt("partial", `TEMPLATE_PRIVATE_MARKER {{if eq .mode "brief"}}{{input "brief"}}{{else}}{{input "long"}}{{end}} {{input "undeclared"}} {{.required}} {{if false}}{{input "inactive"}}{{.inactive_var}}{{end}}`, []any{
map[string]any{"name": "transcript", "required": true, "content_type": "audio/wav", "description": "Transcript source."},
map[string]any{"name": "optional", "content_type": "text/plain", "description": ""},
}, `{{input "session-only"}} {{.session_variable}}`, map[string]any{
"format": "json", "validation_mode": "json_schema", "schema_path": "not-loaded.json",
})
engine := requirementsEngine(t, oneVersionPrompt("requirements", "v1", definition), map[string]any{
"partial": map[string]any{"model": "partial-model", "api_key_required": true},
}, promptkit.WithArtifactReader(reader), promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
PromptID: "requirements", Vars: map[string]string{"mode": "brief", "unrelated": "VARIABLE_PRIVATE_MARKER"},
SessionID: "SESSION_PRIVATE_MARKER",
})
if err != nil {
t.Fatalf("inspect before artifacts, credentials, and schemas exist: %v", err)
}
if inspection.PromptID != "requirements" || inspection.PromptVersion != "v1" || inspection.ProfileID != "partial" || !inspection.Complete {
t.Fatalf("selected context or completeness is wrong: %+v", inspection)
}
inputs := make(map[string]promptkit.InputRequirement, len(inspection.Inputs))
for _, input := range inspection.Inputs {
inputs[input.Name] = input
}
if len(inputs) != 6 || inputs["transcript"].Kind != promptkit.RequirementRequired || !inputs["transcript"].Declared ||
inputs["transcript"].ContentType == nil || *inputs["transcript"].ContentType != "audio/wav" ||
inputs["transcript"].Description == nil || *inputs["transcript"].Description != "Transcript source." {
t.Fatalf("required declaration metadata or discovered inputs are wrong: %+v", inspection.Inputs)
}
if inputs["brief"].Kind != promptkit.RequirementRequired || inputs["long"].Kind != promptkit.RequirementOptional || inputs["long"].Explanation == "" ||
inputs["inactive"].Kind != promptkit.RequirementOptional || inputs["inactive"].Explanation == "" ||
inputs["undeclared"].Kind != promptkit.RequirementRequired || inputs["undeclared"].Declared ||
inputs["optional"].Kind != promptkit.RequirementOptional || !inputs["optional"].Declared ||
inputs["optional"].Description == nil || *inputs["optional"].Description != "" {
t.Fatalf("selected, inactive, or helper-only input classifications are wrong: %+v", inspection.Inputs)
}
if _, found := inputs["session-only"]; found {
t.Fatalf("bypassed session input appeared in inventory: %+v", inspection.Inputs)
}
variables := make(map[string]promptkit.VariableRequirement, len(inspection.Variables))
for _, variable := range inspection.Variables {
variables[variable.Name] = variable
}
if len(variables) != 3 || variables["mode"].Kind != promptkit.RequirementRequired || !variables["mode"].Supplied ||
variables["required"].Kind != promptkit.RequirementRequired || variables["required"].Supplied ||
variables["inactive_var"].Kind != promptkit.RequirementOptional || variables["inactive_var"].Explanation == "" {
t.Fatalf("caller-variable inventory conflated need and presence: %+v", inspection.Variables)
}
if _, found := variables["unrelated"]; found {
t.Fatalf("unreferenced caller key entered the inventory: %+v", inspection.Variables)
}
formatted := fmt.Sprintf("%+v", inspection)
for _, private := range []string{
"TEMPLATE_PRIVATE_MARKER", "VARIABLE_PRIVATE_MARKER", "SESSION_PRIVATE_MARKER",
} {
if strings.Contains(formatted, private) {
t.Fatalf("inspection disclosed %q: %s", private, formatted)
}
}
if reader.calls.Load() != 0 || modelCalls.Load() != 0 {
t.Fatalf("inspection invoked execution collaborators: artifact reads=%d provider calls=%d", reader.calls.Load(), modelCalls.Load())
}
}
func TestInspectRequirementsSelectionUsesExactVersionsAndProfileIdentity(t *testing.T) {
prompts := map[string]any{
"multi": map[string]any{
"v1": contractPrompt("first", `{{input "one"}}`, nil, "", noValidationOutput),
"v2": contractPrompt("second", `{{input "two"}}`, nil, "", noValidationOutput),
},
"without-default": map[string]any{
"v1": contractPrompt("", "plain", nil, "", noValidationOutput),
},
"direct-required": map[string]any{"v1": contractPrompt("direct", "plain", nil, "", noValidationOutput)},
}
profiles := map[string]any{
"first": map[string]any{"model": "first-model"},
"second": map[string]any{"model": "second-model"},
"direct": map[string]any{"endpoint": "https://example.test/v1", "model": "direct-model", "api_key_required": true},
}
engine := requirementsEngine(t, prompts, profiles)
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi"}); !errors.Is(err, promptkit.ErrPromptLoad) {
t.Fatalf("ambiguous omitted version error=%v", err)
}
first, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "v1"})
if err != nil || first.PromptVersion != "v1" || first.ProfileID != "first" || len(first.Inputs) != 1 || first.Inputs[0].Name != "one" {
t.Fatalf("exact first version/default profile=(%+v, %v)", first, err)
}
explicit, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
PromptID: "multi", PromptVersion: "v2", ProfileID: "first",
})
if err != nil || explicit.PromptVersion != "v2" || explicit.ProfileID != "first" || explicit.Inputs[0].Name != "two" {
t.Fatalf("exact version/request profile=(%+v, %v)", explicit, err)
}
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "Multi", PromptVersion: "v1"}); !errors.Is(err, promptkit.ErrPromptNotFound) {
t.Fatalf("case-folded prompt ID unexpectedly selected: %v", err)
}
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "V1"}); !errors.Is(err, promptkit.ErrPromptNotFound) {
t.Fatalf("case-folded version unexpectedly selected: %v", err)
}
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "v1", ProfileID: "missing"}); !errors.Is(err, promptkit.ErrProfileNotFound) {
t.Fatalf("missing profile error=%v", err)
}
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "without-default"}); !errors.Is(err, promptkit.ErrProfileRequired) || !errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("missing profile selection error=%v", err)
}
if direct, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "direct-required"}); err != nil || direct.ProfileID != "direct" {
t.Fatalf("direct-key-required profile should be inspectable without a key: (%+v, %v)", direct, err)
}
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{}); !errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("blank prompt error=%v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := engine.InspectRequirements(ctx, promptkit.RequirementInspectionRequest{PromptID: "multi", PromptVersion: "v1"}); !errors.Is(err, context.Canceled) {
t.Fatalf("canceled inspection error=%v", err)
}
}
func TestInspectPromptHashMatchesPreparationForCapturedConfiguration(t *testing.T) {
prompts := oneVersionPrompt("hash-equality", "v1", contractPrompt(
"profile", "Hello {{.name}}", nil, "", noValidationOutput,
))
engine := requirementsEngine(t, prompts, map[string]any{
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "inspection-model"},
})
inspection, err := engine.InspectPrompt(context.Background(), "hash-equality", "v1")
if err != nil {
t.Fatalf("inspect captured prompt: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "hash-equality", PromptVersion: "v1", Vars: map[string]string{"name": "Ada"},
})
if err != nil {
t.Fatalf("prepare captured prompt: %v", err)
}
if inspection.PromptHash != prepared.PromptHash {
t.Fatalf("inspection hash=%q, preparation hash=%q", inspection.PromptHash, prepared.PromptHash)
}
}
func TestInspectRequirementsReportsConditionalAndCompleteOptionalInventories(t *testing.T) {
prompts := map[string]any{
"conditional": map[string]any{"v1": contractPrompt("profile",
`{{if .mode}}{{input "left"}}{{else}}{{input "right"}}{{end}}`, nil, "", noValidationOutput)},
"optional": map[string]any{"v1": contractPrompt("profile", `{{index . "lookup"}}`,
[]any{map[string]any{"name": "declared-optional"}}, "", noValidationOutput)},
"empty": map[string]any{"v1": contractPrompt("profile", "plain", nil, "", noValidationOutput)},
}
engine := requirementsEngine(t, prompts, map[string]any{"profile": map[string]any{"model": "partial"}})
conditional, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "conditional"})
if err != nil || conditional.Complete || len(conditional.Issues) != 1 ||
conditional.Issues[0].Code != promptkit.RequirementIssueMissingVariable || len(conditional.Inputs) != 2 ||
conditional.Inputs[0].Kind != promptkit.RequirementConditional || conditional.Inputs[1].Kind != promptkit.RequirementConditional {
t.Fatalf("conditional discovery=(%+v, %v)", conditional, err)
}
optional, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "optional"})
if err != nil || !optional.Complete || len(optional.Issues) != 0 || len(optional.Inputs) != 1 ||
optional.Inputs[0].Kind != promptkit.RequirementOptional || len(optional.Variables) != 1 ||
optional.Variables[0].Name != "lookup" || optional.Variables[0].Kind != promptkit.RequirementOptional {
t.Fatalf("all-optional result should be complete without artifact bindings: (%+v, %v)", optional, err)
}
empty, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "empty"})
if err != nil || !empty.Complete || len(empty.Inputs) != 0 || len(empty.Variables) != 0 {
t.Fatalf("empty complete result=(%+v, %v)", empty, err)
}
}
func TestInspectRequirementsBeforePreparationAndWithDirectSessionBypass(t *testing.T) {
definition := contractPrompt("profile", `Name={{.name}} Transcript={{input "transcript"}}{{if false}}ignored {{input "skipped"}} {{.inactive}}{{end}}`, []any{
map[string]any{"name": "transcript", "required": true},
}, `{{.session_var}} {{input "session-only"}}`, noValidationOutput)
engine := requirementsEngine(t, oneVersionPrompt("ready", "v1", definition), map[string]any{
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline"},
})
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
PromptID: "ready", Vars: map[string]string{"name": "Ada"}, SessionID: "direct-session",
})
if err != nil || inspection == nil || !inspection.Complete || len(inspection.Inputs) != 2 ||
inspection.Inputs[0].Name != "skipped" || inspection.Inputs[0].Kind != promptkit.RequirementOptional ||
inspection.Inputs[1].Name != "transcript" || inspection.Inputs[1].Kind != promptkit.RequirementRequired ||
len(inspection.Variables) != 2 || inspection.Variables[0].Name != "inactive" || inspection.Variables[0].Kind != promptkit.RequirementOptional ||
inspection.Variables[1].Name != "name" || inspection.Variables[1].Kind != promptkit.RequirementRequired {
t.Fatalf("artifact-free inspection=(%+v, %v)", inspection, err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "ready", Vars: map[string]string{"name": "Ada"}, SessionID: "direct-session",
Inputs: map[string]promptkit.ArtifactRef{"transcript": promptkit.Inline("source transcript")},
})
if err != nil {
t.Fatalf("prepare with matching synthetic artifact: %v", err)
}
if prepared.Messages[0].Content != "Name=Ada Transcript=source transcript" || prepared.SessionID != "direct-session" {
t.Fatalf("prepared run did not use the inspected dependencies: %+v", prepared)
}
if _, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
PromptID: "ready", Vars: map[string]string{"name": "Ada"},
}); err != nil {
t.Fatalf("inspect without session bypass: %v", err)
}
}
func TestInspectRequirementsUsesCapturedFileBackedTemplates(t *testing.T) {
files := fstest.MapFS{
"config/prompt.yaml": &fstest.MapFile{Data: []byte(capturedPromptDocument)},
"config/body.tmpl": &fstest.MapFile{Data: []byte(`{{input "captured-input"}}`)},
}
source := &unreadableAfterConstructionFS{FS: files}
engine := capturedPromptEngine(t, source)
source.closed.Store(true)
delete(files, "config/body.tmpl")
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{PromptID: "captured"})
if err != nil || inspection == nil || !inspection.Complete || len(inspection.Inputs) != 1 ||
inspection.Inputs[0].Name != "captured-input" || source.reads.Load() != 0 {
t.Fatalf("captured template inspection=(%+v, %v), post-construction reads=%d", inspection, err, source.reads.Load())
}
}
func TestInspectRequirementsReturnsIndependentValuesAndCanOverlapPreparation(t *testing.T) {
definition := contractPrompt("profile", `{{if .decision}}{{input "conditional"}}{{end}} {{input "declared"}}`, []any{
map[string]any{"name": "declared", "required": true, "description": "captured description"},
}, `{{.session}}`, noValidationOutput)
engine := requirementsEngine(t, oneVersionPrompt("owned", "v1", definition), map[string]any{
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline"},
})
request := promptkit.RequirementInspectionRequest{PromptID: "owned"}
first, err := engine.InspectRequirements(context.Background(), request)
if err != nil {
t.Fatal(err)
}
for index := range first.Inputs {
if first.Inputs[index].Name == "declared" {
first.Inputs[index].Name = "mutated"
if first.Inputs[index].Description != nil {
*first.Inputs[index].Description = "mutated description"
}
}
}
first.Variables[0].Name = "mutated-variable"
first.Issues[0].Description = "mutated issue"
second, err := engine.InspectRequirements(context.Background(), request)
if err != nil {
t.Fatalf("inspect after mutating returned values: %v", err)
}
var description *string
for _, input := range second.Inputs {
if input.Name == "declared" {
description = input.Description
}
}
if second.Inputs[0].Name == "mutated" || second.Variables[0].Name != "decision" ||
description == nil || *description != "captured description" ||
len(second.Issues) != 1 || second.Issues[0].Description == "mutated issue" {
t.Fatalf("result mutation reached captured state or later result: (%+v, %v)", second, err)
}
const count = 12
var wait sync.WaitGroup
errs := make(chan error, count*2)
for index := 0; index < count; index++ {
wait.Add(2)
go func() {
defer wait.Done()
_, err := engine.InspectRequirements(context.Background(), request)
errs <- err
}()
go func(index int) {
defer wait.Done()
_, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "owned", SessionID: fmt.Sprintf("session-%d", index), Vars: map[string]string{"decision": "yes"},
Inputs: map[string]promptkit.ArtifactRef{
"conditional": promptkit.Inline("condition"), "declared": promptkit.Inline("body"),
},
})
errs <- err
}(index)
}
wait.Wait()
close(errs)
for err := range errs {
if err != nil {
t.Fatal(err)
}
}
}
func TestInspectRequirementsDoesNotEchoUnrelatedPrivateValues(t *testing.T) {
t.Setenv("PROMPTKIT_REQUIREMENTS_SECRET", "credential-value-marker")
definition := contractPrompt("profile", `template-body-marker {{input "required-input"}} {{.known}}`, nil, `{{.session}}`, noValidationOutput)
engine := requirementsEngine(t, oneVersionPrompt("privacy", "v1", definition), map[string]any{
"profile": map[string]any{"endpoint": "https://example.test/v1", "model": "offline", "api_key_env": "PROMPTKIT_REQUIREMENTS_SECRET"},
})
inspection, err := engine.InspectRequirements(context.Background(), promptkit.RequirementInspectionRequest{
PromptID: "privacy", Vars: map[string]string{"known": "known-value-marker", "unrelated": "unrelated-value-marker"},
SessionID: "direct-session-marker",
})
if err != nil {
t.Fatal(err)
}
formatted := fmt.Sprintf("%+v", inspection)
for _, marker := range []string{
"template-body-marker", "known-value-marker", "unrelated-value-marker",
"direct-session-marker", "credential-value-marker", "PROMPTKIT_REQUIREMENTS_SECRET",
} {
if strings.Contains(formatted, marker) {
t.Fatalf("inspection disclosed %q: %s", marker, formatted)
}
}
}

15
requirements_test.go Normal file
View File

@@ -0,0 +1,15 @@
package promptkit
import (
"context"
"errors"
"testing"
)
func TestInspectRequirementsRejectsNilEngine(t *testing.T) {
var engine *Engine
inspection, err := engine.InspectRequirements(context.Background(), RequirementInspectionRequest{PromptID: "prompt"})
if inspection != nil || !errors.Is(err, ErrInvalidConfig) {
t.Fatalf("nil engine returned (%+v, %v), want ErrInvalidConfig", inspection, err)
}
}

161
scripts/check-examples.py Normal file
View File

@@ -0,0 +1,161 @@
#!/usr/bin/env python3
"""Run the maintained consumer examples and check their semantic output."""
import json
import os
import subprocess
import sys
from pathlib import Path
REPOSITORY_ROOT = Path(__file__).resolve().parent.parent
def example_environment():
environment = os.environ.copy()
environment["GOTOOLCHAIN"] = "local"
try:
host = subprocess.run(
["go", "env", "GOHOSTOS", "GOHOSTARCH"],
cwd=REPOSITORY_ROOT,
env=environment,
text=True,
capture_output=True,
check=False,
)
except OSError as error:
raise RuntimeError(f"could not run go env: {error}") from error
if host.returncode != 0:
raise RuntimeError(
"go env GOHOSTOS GOHOSTARCH failed "
f"with exit status {host.returncode}: {host.stderr.strip()}"
)
values = host.stdout.splitlines()
if len(values) != 2 or not all(values):
raise RuntimeError(
"go env GOHOSTOS GOHOSTARCH returned unexpected output: "
f"{host.stdout!r}"
)
environment["GOOS"], environment["GOARCH"] = values
return environment
def run_example(name, environment, failures):
command = ["go", "run", f"./examples/go-library/{name}"]
try:
result = subprocess.run(
command,
cwd=REPOSITORY_ROOT,
env=environment,
text=True,
capture_output=True,
check=False,
)
except OSError as error:
failures.append(f"{name}: could not start {command[0]}: {error}")
return None
if result.returncode != 0:
failures.append(f"{name}: command exited with status {result.returncode}")
if result.stderr:
failures.append(f"{name}: stderr:\n{result.stderr.rstrip()}")
if result.stdout:
failures.append(f"{name}: stdout:\n{result.stdout.rstrip()}")
return None
try:
value = json.loads(result.stdout)
except json.JSONDecodeError as error:
failures.append(f"{name}: output is not valid JSON: {error}")
if result.stdout:
failures.append(f"{name}: stdout:\n{result.stdout.rstrip()}")
return None
if not isinstance(value, dict):
failures.append(
f"{name}: expected a JSON object, got {type(value).__name__}"
)
return None
return value
def expect_equal(example_name, value, key, expected, failures):
actual = value.get(key)
matches = actual is expected if isinstance(expected, bool) else actual == expected
if not matches:
failures.append(
f"{example_name}: expected {key}={expected!r}, got {actual!r}"
)
def expect_positive(example_name, value, key, failures):
actual = value.get(key)
try:
matches = actual > 0
except TypeError:
matches = False
if not matches:
failures.append(f"{example_name}: expected {key} to be positive, got {actual!r}")
def main():
try:
environment = example_environment()
except RuntimeError as error:
print(error, file=sys.stderr)
return 1
failures = []
prepared = run_example("prepare", environment, failures)
result = run_example("run", environment, failures)
observed = run_example("observe", environment, failures)
if prepared is not None:
expect_equal("prepare", prepared, "prompt_id", "example.prepare", failures)
expect_equal(
"prepare", prepared, "selected_profile", "offline-example", failures
)
expect_equal("prepare", prepared, "model", "offline-model", failures)
expect_positive("prepare", prepared, "message_count", failures)
if result is not None:
expect_equal(
"run", result, "output", "Ada finished the migration review.", failures
)
expect_equal("run", result, "is_valid", True, failures)
expect_equal("run", result, "validation_status", "passed", failures)
expect_equal("run", result, "model", "offline-model", failures)
expect_equal("run", result, "total_tokens", 18, failures)
if observed is not None:
expect_equal("observe", observed, "proposal_attempts", 2, failures)
expect_equal(
"observe", observed, "proposal_transient_retries", 1, failures
)
expect_equal(
"observe", observed, "proposal_usage_complete", False, failures
)
expect_equal("observe", observed, "assessment_attempts", 1, failures)
expect_equal(
"observe", observed, "assessment_transient_retries", 0, failures
)
expect_equal(
"observe", observed, "assessment_outcome", "accepted", failures
)
if failures:
print("Consumer example checks failed:", file=sys.stderr)
for failure in failures:
print(f"- {failure}", file=sys.stderr)
return 1
print("Consumer example checks passed: prepare, run, observe")
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -1,152 +0,0 @@
#!/bin/sh
# Run local checks without publishing or changing files.
set -eu
cd "$(dirname "$0")/.."
go test -race -vet=off ./...
go vet ./...
python3 - <<'PY'
import json
import subprocess
def example(name):
return json.loads(subprocess.check_output(
["go", "run", "./examples/go-library/" + name], text=True))
prepared = example("prepare")
assert prepared["prompt_id"] == "example.prepare", prepared
assert prepared["selected_profile"] == "offline-example", prepared
assert prepared["model"] == "offline-model", prepared
assert prepared["message_count"] > 0, prepared
result = example("run")
assert result["output"] == "Ada finished the migration review.", result
assert result["is_valid"] is True, result
assert result["validation_status"] == "passed", result
assert result["model"] == "offline-model", result
assert result["total_tokens"] == 18, result
observed = example("observe")
assert observed["proposal_attempts"] == 2, observed
assert observed["proposal_transient_retries"] == 1, observed
assert observed["proposal_usage_complete"] is False, observed
assert observed["assessment_attempts"] == 1, observed
assert observed["assessment_transient_retries"] == 0, observed
assert observed["assessment_outcome"] == "accepted", observed
PY
unformatted=$(
git ls-files '*.go' |
while IFS= read -r go_file
do
gofmt -l "$go_file"
done
)
test -z "$unformatted"
python3 - <<'PY'
from pathlib import Path
import re
import subprocess
import sys
from urllib.parse import unquote
root = Path.cwd().resolve()
markdown_files = [
root / name
for name in subprocess.check_output(
["git", "ls-files", "*.md"], text=True
).splitlines()
]
link_pattern = re.compile(r"!?\[[^]]*\]\(([^)]+)\)")
heading_pattern = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$")
scheme_pattern = re.compile(r"^[a-z][a-z0-9+.-]*:", re.IGNORECASE)
def markdown_lines(path):
in_fence = False
fence = ""
for line in path.read_text(encoding="utf-8").splitlines():
stripped = line.lstrip()
marker = stripped[:3]
if marker in {"```", "~~~"}:
if not in_fence:
in_fence = True
fence = marker
elif marker == fence:
in_fence = False
fence = ""
continue
if not in_fence:
yield line
anchor_cache = {}
def anchors(path):
if path in anchor_cache:
return anchor_cache[path]
found = set()
counts = {}
for line in markdown_lines(path):
match = heading_pattern.match(line)
if not match:
continue
heading = re.sub(r"<[^>]+>", "", match.group(1)).replace("`", "")
base = re.sub(r"[^\w\- ]", "", heading.lower()).replace(" ", "-")
count = counts.get(base, 0)
counts[base] = count + 1
found.add(base if count == 0 else f"{base}-{count}")
anchor_cache[path] = found
return found
failures = []
for source in markdown_files:
text = "\n".join(markdown_lines(source))
for match in link_pattern.finditer(text):
target = match.group(1).strip()
if target.startswith("<") and target.endswith(">"):
target = target[1:-1]
if scheme_pattern.match(target) or target.startswith("//"):
continue
path_text, separator, fragment = target.partition("#")
destination = source if not path_text else source.parent / unquote(path_text)
try:
destination = destination.resolve()
destination.relative_to(root)
except ValueError:
failures.append(f"{source.relative_to(root)}: escapes repository: {target}")
continue
if not destination.exists():
failures.append(f"{source.relative_to(root)}: missing target: {target}")
continue
if separator and destination.suffix.lower() == ".md":
fragment = unquote(fragment).lower()
if fragment not in anchors(destination):
failures.append(f"{source.relative_to(root)}: missing anchor: {target}")
if failures:
print("\n".join(failures), file=sys.stderr)
raise SystemExit(1)
PY
case "$(go env GOWORK)" in
''|off) ;;
*) printf '%s\n' 'an active Go workspace is not allowed' >&2; exit 1 ;;
esac
test -z "$(git ls-files go.work go.work.sum)"
test ! -e vendor
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
then
printf '%s\n' 'go.mod contains a replacement' >&2
exit 1
fi
git diff --check
git diff --cached --check
credential_pattern='-----BEGIN ([A-Z0-9]+ )?PRIV''ATE KEY-----|AKI''A[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{36,}|sk-[A-Za-z0-9]{32,}'
if git grep -nEI -e "$credential_pattern" -- .
then
printf '%s\n' 'possible credential found' >&2
exit 1
fi

View File

@@ -1,57 +1,5 @@
#!/bin/sh
# Validate one committed candidate and publish only its selected annotated tag.
set -eu
cd "$(dirname "$0")/.."
release_version=${1:?usage: scripts/release.sh vMAJOR.MINOR.PATCH}
printf '%s\n' "$release_version" |
grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$'
release_commit=$(git rev-parse --verify 'HEAD^{commit}')
check_candidate() {
test "$(git branch --show-current)" = main
test -z "$(git status --porcelain)"
test "$(git rev-parse HEAD)" = "$release_commit"
git fetch origin main
test "$(git rev-parse refs/remotes/origin/main)" = "$release_commit"
if git show-ref --verify --quiet "refs/tags/$release_version"; then
printf '%s\n' "local tag already exists: $release_version" >&2
exit 1
fi
remote_refs=$(git ls-remote --tags origin "refs/tags/$release_version" "refs/tags/$release_version^{}")
test -z "$remote_refs"
}
check_candidate
python3 - "$release_version" <<'PY'
from pathlib import Path
import sys
version = sys.argv[1]
note = Path("docs/releases") / (version + ".md")
lines = note.read_text(encoding="utf-8").splitlines()
assert lines and lines[0] == "# Promptkit " + version, f"wrong release title: {note}"
headings = set()
fence = None
for line in lines:
marker = line.lstrip()[:3]
if marker in {"```", "~~~"}:
if fence is None:
fence = marker
elif fence == marker:
fence = None
elif fence is None:
headings.add(line)
for section in ("Summary", "Compatibility", "Upgrade", "Changes"):
assert "## " + section in headings, f"missing {section}: {note}"
PY
./scripts/release-check.sh
check_candidate
release_note=$(git show "$release_commit:docs/releases/$release_version.md")
printf '%s\n\nValidated commit: %s\n' "$release_note" "$release_commit" |
git tag --annotate --cleanup=verbatim "$release_version" --file - "$release_commit"
test "$(git cat-file -t "refs/tags/$release_version")" = tag
test "$(git rev-parse "refs/tags/$release_version^{commit}")" = "$release_commit"
git push origin "refs/tags/$release_version:refs/tags/$release_version"
printf '%s\n' "Released $release_version. No CI or module-availability wait is required."
exec ./scripts/releasekit.sh publish "$@"

18
scripts/releasekit.sh Executable file
View File

@@ -0,0 +1,18 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
export GOTOOLCHAIN=local
export GOWORK=off
export GOFLAGS=-mod=readonly
GOOS=$(go env GOHOSTOS)
GOARCH=$(go env GOHOSTARCH)
export GOOS GOARCH
mkdir -p .releasekit/bin
go build -mod=readonly -o .releasekit/bin/releasekit \
"gitea.maximumdirect.net/eric/ReleaseKit/cmd/releasekit"
exec ./.releasekit/bin/releasekit "$@"

476
scripts/test-release.py Normal file
View File

@@ -0,0 +1,476 @@
"""Exercise ReleaseKit publication wiring in disposable repositories."""
import json
import os
import shutil
import subprocess
import sys
import tempfile
import textwrap
from pathlib import Path
REPOSITORY_ROOT = Path(__file__).resolve().parent.parent
RELEASE_VERSION = "v0.0.1"
FAILED_RELEASE_VERSION = "v0.0.2"
RELEASEKIT_MODULE = "gitea.maximumdirect.net/eric/ReleaseKit"
def execute(command, cwd, environment):
try:
return subprocess.run(
command,
cwd=cwd,
env=environment,
text=True,
capture_output=True,
check=False,
)
except OSError as error:
raise RuntimeError(f"could not run {command[0]}: {error}") from error
def run(command, cwd, environment):
result = execute(command, cwd, environment)
if result.returncode != 0:
details = [
f"command failed with status {result.returncode}: {' '.join(command)}"
]
if result.stdout:
details.append(f"stdout:\n{result.stdout.rstrip()}")
if result.stderr:
details.append(f"stderr:\n{result.stderr.rstrip()}")
raise RuntimeError("\n".join(details))
return result.stdout
def write_file(path, contents):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(textwrap.dedent(contents).lstrip("\n"), encoding="utf-8")
def repo_git(repository, environment, *arguments):
return run(
["git", "-C", str(repository), *arguments],
cwd=repository.parent,
environment=environment,
)
def remote_git(remote, temporary_root, environment, *arguments):
return run(
["git", "--git-dir", str(remote), *arguments],
cwd=temporary_root,
environment=environment,
)
def fixture_sources(repository, environment):
write_file(
repository / "fixture.go",
"""
package fixture
func candidateValue() string { return "ready" }
""",
)
write_file(
repository / "fixture_test.go",
f"""
package fixture
import (
"encoding/json"
"os"
"runtime"
"testing"
)
func TestCandidateSuite(t *testing.T) {{
if candidateValue() != "ready" {{
t.Fatal("candidate package is not ready")
}}
marker := os.Getenv("FIXTURE_SUITE_MARKER")
if marker == "" {{
t.Fatal("FIXTURE_SUITE_MARKER is not set")
}}
record := struct {{
GoVersion string `json:"go_version"`
Race bool `json:"race"`
}}{{GoVersion: runtime.Version(), Race: raceDetectorEnabled}}
data, err := json.Marshal(record)
if err != nil {{
t.Fatal(err)
}}
file, err := os.OpenFile(marker, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {{
t.Fatal(err)
}}
if _, err := file.Write(append(data, '\\n')); err != nil {{
file.Close()
t.Fatal(err)
}}
if err := file.Close(); err != nil {{
t.Fatal(err)
}}
}}
""",
)
write_file(
repository / "race_enabled_test.go",
"""
//go:build race
package fixture
const raceDetectorEnabled = true
""",
)
write_file(
repository / "race_disabled_test.go",
"""
//go:build !race
package fixture
const raceDetectorEnabled = false
""",
)
go_files = [
str(repository / filename)
for filename in (
"fixture.go",
"fixture_test.go",
"race_enabled_test.go",
"race_disabled_test.go",
)
]
run(["gofmt", "-w", *go_files], cwd=repository, environment=environment)
def prepare_repository(repository, remote, environment, module_bytes, version):
run(
["git", "init", "--bare", "--initial-branch=main", str(remote)],
cwd=repository.parent,
environment=environment,
)
run(
["git", "init", "--initial-branch=main", str(repository)],
cwd=repository.parent,
environment=environment,
)
repo_git(repository, environment, "config", "user.name", "ReleaseKit Fixture")
repo_git(repository, environment, "config", "user.email", "fixture@example.invalid")
repo_git(repository, environment, "remote", "add", "origin", str(remote))
for name, contents in module_bytes.items():
(repository / name).write_bytes(contents)
write_file(
repository / ".gitignore",
"""
.releasekit/
""",
)
write_file(
repository / "releasekit.json",
(REPOSITORY_ROOT / "releasekit.json").read_text(encoding="utf-8"),
)
scripts = repository / "scripts"
scripts.mkdir(parents=True, exist_ok=True)
shutil.copy2(REPOSITORY_ROOT / "scripts/release.sh", scripts / "release.sh")
shutil.copy2(REPOSITORY_ROOT / "scripts/releasekit.sh", scripts / "releasekit.sh")
write_file(
scripts / "check-examples.py",
"""
import os
from pathlib import Path
import sys
marker = Path(os.environ["FIXTURE_EXAMPLE_MARKER"])
with marker.open("a", encoding="utf-8") as stream:
stream.write("called\\n")
if os.environ.get("FIXTURE_EXAMPLE_FAILURE") == "1":
print("fixture example check rejected candidate", file=sys.stderr)
raise SystemExit(1)
""",
)
write_file(
repository / f"docs/releases/{version}.md",
f"""
# Promptkit {version}
Disposable publication fixture.
""",
)
fixture_sources(repository, environment)
repo_git(repository, environment, "add", "-A")
run(
["git", "-C", str(repository), "commit", "-m", "Add disposable release fixture"],
cwd=repository.parent,
environment=environment,
)
repo_git(repository, environment, "push", "origin", "main")
def verify_publication(repository, remote, temporary_root, environment, module_bytes, selected_go):
selected_releasekit = run(
["go", "list", "-mod=readonly", "-m", "-f", "{{.Version}}", RELEASEKIT_MODULE],
cwd=repository,
environment=environment,
).strip()
publication = repository / "scripts/release.sh"
run(
[str(publication), RELEASE_VERSION],
cwd=temporary_root,
environment=environment,
)
suite_lines = Path(environment["FIXTURE_SUITE_MARKER"]).read_text(
encoding="utf-8"
).splitlines()
if len(suite_lines) != 1:
raise RuntimeError(f"expected one suite marker, found {len(suite_lines)}")
suite_record = json.loads(suite_lines[0])
if suite_record != {"go_version": selected_go, "race": True}:
raise RuntimeError(f"unexpected suite marker: {suite_record!r}")
example_lines = Path(environment["FIXTURE_EXAMPLE_MARKER"]).read_text(
encoding="utf-8"
).splitlines()
if example_lines != ["called"]:
raise RuntimeError(f"expected one additional-check marker, got {example_lines!r}")
binary = repository / ".releasekit/bin/releasekit"
if not binary.is_file():
raise RuntimeError("ReleaseKit bootstrap did not leave its ignored executable")
build_info = run(
["go", "version", "-m", str(binary)],
cwd=temporary_root,
environment=environment,
)
first_line = build_info.splitlines()[0]
if selected_go not in first_line:
raise RuntimeError(f"tool was built with {first_line!r}, expected {selected_go}")
expected_module = f"\tmod\t{RELEASEKIT_MODULE}\t{selected_releasekit}\t"
if expected_module not in build_info:
raise RuntimeError(
f"bootstrap executable does not record selected ReleaseKit {selected_releasekit}"
)
candidate = repo_git(repository, environment, "rev-parse", "HEAD").strip()
if repo_git(repository, environment, "rev-parse", "main").strip() != candidate:
raise RuntimeError("fixture main moved away from the published candidate")
if remote_git(remote, temporary_root, environment, "rev-parse", "refs/heads/main").strip() != candidate:
raise RuntimeError("local origin main does not identify the fixture candidate")
tag_ref = f"refs/tags/{RELEASE_VERSION}"
if repo_git(repository, environment, "cat-file", "-t", tag_ref).strip() != "tag":
raise RuntimeError("local release ref is not an annotated tag")
local_tag = repo_git(repository, environment, "rev-parse", tag_ref).strip()
local_tag_commit = repo_git(
repository, environment, "rev-parse", f"{tag_ref}^{{}}"
).strip()
if local_tag_commit != candidate:
raise RuntimeError("local annotated tag does not identify the fixture candidate")
remote_tag = remote_git(remote, temporary_root, environment, "rev-parse", tag_ref).strip()
remote_tag_commit = remote_git(
remote, temporary_root, environment, "rev-parse", f"{tag_ref}^{{}}"
).strip()
if remote_tag != local_tag or remote_tag_commit != candidate:
raise RuntimeError("remote annotated tag does not identify the fixture candidate")
if remote_git(
remote, temporary_root, environment, "cat-file", "-t", tag_ref
).strip() != "tag":
raise RuntimeError("remote release ref is not an annotated tag")
tag_object = repo_git(repository, environment, "cat-file", "tag", tag_ref)
separator = tag_object.find("\n\n")
if separator < 0:
raise RuntimeError("annotated tag has no message")
tag_message = tag_object[separator + 2 :]
note = (repository / f"docs/releases/{RELEASE_VERSION}.md").read_text(
encoding="utf-8"
)
if tag_message != note:
raise RuntimeError("annotated tag message does not match the committed release note")
for name, contents in module_bytes.items():
if (repository / name).read_bytes() != contents:
raise RuntimeError(f"publication changed {name}")
status = repo_git(
repository, environment, "status", "--porcelain", "--untracked-files=all"
)
if status:
raise RuntimeError(f"fixture checkout is not clean after publication:\n{status}")
repo_git(repository, environment, "check-ignore", "-q", ".releasekit/bin/releasekit")
def verify_validation_failure(
repository, remote, temporary_root, environment, module_bytes, selected_go
):
scenario_environment = environment.copy()
suite_marker = temporary_root / "failure-suite.jsonl"
example_marker = temporary_root / "failure-example.log"
scenario_environment.update(
{
"FIXTURE_SUITE_MARKER": str(suite_marker),
"FIXTURE_EXAMPLE_MARKER": str(example_marker),
"FIXTURE_EXAMPLE_FAILURE": "1",
}
)
prepare_repository(
repository,
remote,
scenario_environment,
module_bytes,
FAILED_RELEASE_VERSION,
)
result = execute(
[str(repository / "scripts/release.sh"), FAILED_RELEASE_VERSION],
cwd=temporary_root,
environment=scenario_environment,
)
if result.returncode != 1:
raise RuntimeError(
"validation failure returned status "
f"{result.returncode}, want 1; stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
)
if "fixture example check rejected candidate" not in result.stderr:
raise RuntimeError(f"failure diagnostic omitted useful context:\n{result.stderr}")
suite_lines = suite_marker.read_text(encoding="utf-8").splitlines()
if len(suite_lines) != 1:
raise RuntimeError(f"expected one failure-case suite marker, found {len(suite_lines)}")
suite_record = json.loads(suite_lines[0])
if suite_record != {"go_version": selected_go, "race": True}:
raise RuntimeError(f"unexpected failure-case suite marker: {suite_record!r}")
if example_marker.read_text(encoding="utf-8").splitlines() != ["called"]:
raise RuntimeError("expected one additional-check invocation before failure")
tag_ref = f"refs/tags/{FAILED_RELEASE_VERSION}"
for description, command in (
(
"local",
["git", "-C", str(repository), "show-ref", "--verify", "--quiet", tag_ref],
),
(
"remote",
["git", "--git-dir", str(remote), "show-ref", "--verify", "--quiet", tag_ref],
),
):
result = execute(command, cwd=temporary_root, environment=scenario_environment)
if result.returncode == 0:
raise RuntimeError(f"validation failure created the {description} tag {tag_ref}")
if result.returncode != 1:
raise RuntimeError(
f"could not verify {description} tag absence: "
f"status {result.returncode}: {result.stderr.strip()}"
)
for name, contents in module_bytes.items():
if (repository / name).read_bytes() != contents:
raise RuntimeError(f"failed publication changed {name}")
status = repo_git(
repository,
scenario_environment,
"status",
"--porcelain",
"--untracked-files=all",
)
if status:
raise RuntimeError(f"failure fixture checkout is not clean:\n{status}")
def verify_usage_classification(binary, temporary_root, environment):
result = execute(
[str(binary), "unknown"], cwd=temporary_root, environment=environment
)
if result.returncode != 2:
raise RuntimeError(
f"unsupported command returned status {result.returncode}, want 2; "
f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
)
if result.stdout:
raise RuntimeError(f"unsupported command wrote to stdout: {result.stdout!r}")
if 'unknown command "unknown"' not in result.stderr or "usage: releasekit" not in result.stderr:
raise RuntimeError(f"unsupported command diagnostic was not useful:\n{result.stderr}")
def main():
environment = os.environ.copy()
environment.update(
{
"GOTOOLCHAIN": "local",
"GOWORK": "off",
"GOPROXY": "off",
"GONOPROXY": "none",
"GOSUMDB": "off",
}
)
selected_go = run(
["go", "env", "GOVERSION"],
cwd=REPOSITORY_ROOT,
environment=environment,
).strip()
if not selected_go:
print("go env GOVERSION returned no version", file=sys.stderr)
return 1
ignored_rule = ".releasekit/"
ignore_lines = (REPOSITORY_ROOT / ".gitignore").read_text(
encoding="utf-8"
).splitlines()
if ignored_rule not in ignore_lines:
print(".gitignore does not contain the ReleaseKit output rule", file=sys.stderr)
return 1
module_bytes = {
name: (REPOSITORY_ROOT / name).read_bytes() for name in ("go.mod", "go.sum")
}
with tempfile.TemporaryDirectory(prefix="promptkit-release-fixture-") as directory:
temporary_root = Path(directory)
repository = temporary_root / "checkout"
remote = temporary_root / "origin.git"
suite_marker = temporary_root / "suite.jsonl"
example_marker = temporary_root / "example.log"
environment["FIXTURE_SUITE_MARKER"] = str(suite_marker)
environment["FIXTURE_EXAMPLE_MARKER"] = str(example_marker)
try:
prepare_repository(
repository, remote, environment, module_bytes, RELEASE_VERSION
)
verify_publication(
repository,
remote,
temporary_root,
environment,
module_bytes,
selected_go,
)
verify_usage_classification(
repository / ".releasekit/bin/releasekit", temporary_root, environment
)
verify_validation_failure(
temporary_root / "failure-checkout",
temporary_root / "failure-origin.git",
temporary_root,
environment,
module_bytes,
selected_go,
)
except (RuntimeError, OSError, json.JSONDecodeError) as error:
print(error, file=sys.stderr)
return 1
print("ReleaseKit publication success and failure fixtures passed")
return 0
if __name__ == "__main__":
raise SystemExit(main())